From bd7e506596018e4a550775402f7b3e094a15b816 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 07:44:30 -0700 Subject: [PATCH 001/209] test(perf): require deterministic synthetic benchmark corpus --- src/performanceCorpusContract.test.ts | 77 +++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 src/performanceCorpusContract.test.ts diff --git a/src/performanceCorpusContract.test.ts b/src/performanceCorpusContract.test.ts new file mode 100644 index 00000000..5c2d6edd --- /dev/null +++ b/src/performanceCorpusContract.test.ts @@ -0,0 +1,77 @@ +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +interface BenchmarkProfileLock { + readonly sections: number; + readonly bytes: number; + readonly sha256: string; +} + +interface BenchmarkCorpusLock { + readonly contractVersion: 1; + readonly synthetic: true; + readonly scripts: readonly [ + 'English', + 'Korean', + 'Japanese', + 'Chinese', + 'Vietnamese', + 'mixed', + ]; + readonly profiles: Readonly< + Record<'small' | 'medium' | 'large' | 'stress', BenchmarkProfileLock> + >; +} + +function runGenerator(outputDirectory: string): BenchmarkCorpusLock { + const script = resolve(process.cwd(), 'benchmarks/generate-corpus.mjs'); + execFileSync(process.execPath, [script, '--output', outputDirectory], { + cwd: process.cwd(), + stdio: ['ignore', 'pipe', 'pipe'], + }); + return JSON.parse( + readFileSync(join(outputDirectory, 'manifest.json'), 'utf8'), + ) as BenchmarkCorpusLock; +} + +describe('deterministic synthetic performance corpus', () => { + it('reproduces the committed corpus lock exactly across independent runs', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-corpus-')); + const first = join(root, 'first'); + const second = join(root, 'second'); + try { + const expected = JSON.parse( + readFileSync( + resolve(process.cwd(), 'benchmarks/corpus.lock.json'), + 'utf8', + ), + ) as BenchmarkCorpusLock; + const firstManifest = runGenerator(first); + const secondManifest = runGenerator(second); + + expect(firstManifest).toEqual(expected); + expect(secondManifest).toEqual(expected); + expect(firstManifest.synthetic).toBe(true); + expect(firstManifest.scripts).toEqual([ + 'English', + 'Korean', + 'Japanese', + 'Chinese', + 'Vietnamese', + 'mixed', + ]); + + for (const profile of ['small', 'medium', 'large', 'stress'] as const) { + const firstBytes = readFileSync(join(first, `${profile}.md`)); + const secondBytes = readFileSync(join(second, `${profile}.md`)); + expect(firstBytes.equals(secondBytes)).toBe(true); + expect(firstBytes.byteLength).toBe(expected.profiles[profile].bytes); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From e913b9e0da7d9314aa989b2ba5a5d1c26badb85b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 07:44:54 -0700 Subject: [PATCH 002/209] feat(perf): add deterministic multilingual corpus generator --- benchmarks/generate-corpus.mjs | 112 +++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 benchmarks/generate-corpus.mjs diff --git a/benchmarks/generate-corpus.mjs b/benchmarks/generate-corpus.mjs new file mode 100644 index 00000000..4f582805 --- /dev/null +++ b/benchmarks/generate-corpus.mjs @@ -0,0 +1,112 @@ +import { createHash } from 'node:crypto'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +const PIXEL_BASE64 = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII='; +const SCRIPT_PARAGRAPHS = [ + 'English: Deterministic authoring keeps document state explicit and reviewable.', + '한국어: 결정적 작성 흐름은 문서 상태와 변경 근거를 명확하게 유지합니다.', + '日本語: 決定的な編集フローは文書状態と変更根拠を明示的に保ちます。', + '中文: 确定性的编辑流程会明确保留文档状态与变更依据。', + 'Tiếng Việt: Luồng biên soạn xác định giữ trạng thái tài liệu và bằng chứng thay đổi rõ ràng.', +]; +const PROFILE_SECTIONS = Object.freeze({ + small: 1, + medium: 8, + large: 32, + stress: 128, +}); +const SCRIPT_LABELS = Object.freeze([ + 'English', + 'Korean', + 'Japanese', + 'Chinese', + 'Vietnamese', + 'mixed', +]); + +function buildSection(index) { + const id = String(index).padStart(4, '0'); + const tableRows = Array.from({ length: 4 }, (_, rowIndex) => { + const row = String(rowIndex + 1).padStart(2, '0'); + return `| ${id}-r${row}c01 | ${id}-r${row}c02 | ${id}-r${row}c03 | ${id}-r${row}c04 | ${id}-r${row}c05 | ${id}-r${row}c06 |`; + }); + return [ + `# Synthetic section ${id}`, + '', + ...SCRIPT_PARAGRAPHS, + '', + `## Nested list ${id}`, + `- item ${id}-a`, + ` - item ${id}-a-1`, + ` - item ${id}-a-2`, + `- item ${id}-b`, + '', + `> Synthetic blockquote ${id}: benchmark text only; no production content.`, + '', + '```text', + `fixture=${id}; authority=none; network=none`, + '```', + '', + `[synthetic safe link ${id}](https://example.invalid/inkspan/${id})`, + '', + '| c01 | c02 | c03 | c04 | c05 | c06 |', + '| --- | --- | --- | --- | --- | --- |', + ...tableRows, + '', + `![synthetic 1x1 raster ${id}](data:image/png;base64,${PIXEL_BASE64})`, + '', + '---', + '', + ].join('\n'); +} + +function buildProfile(profile, sectionCount) { + return [ + `# Inkspan deterministic benchmark fixture: ${profile}`, + '', + 'Synthetic fixture only. No customer, tenant, prompt, credential, or model data.', + 'Scripts: English, Korean, Japanese, Chinese, Vietnamese, and mixed-script structure.', + '', + ...Array.from({ length: sectionCount }, (_, index) => buildSection(index + 1)), + ].join('\n'); +} + +function sha256(bytes) { + return createHash('sha256').update(bytes).digest('hex'); +} + +function resolveOutputDirectory(argv) { + if (argv.length !== 2 || argv[0] !== '--output' || argv[1].length === 0) { + throw new Error('Usage: node benchmarks/generate-corpus.mjs --output '); + } + return resolve(argv[1]); +} + +const outputDirectory = resolveOutputDirectory(process.argv.slice(2)); +mkdirSync(outputDirectory, { recursive: true }); + +const profileManifest = {}; +for (const [profile, sections] of Object.entries(PROFILE_SECTIONS)) { + const body = buildProfile(profile, sections); + const bytes = Buffer.from(body, 'utf8'); + writeFileSync(resolve(outputDirectory, `${profile}.md`), bytes); + profileManifest[profile] = Object.freeze({ + sections, + bytes: bytes.byteLength, + sha256: sha256(bytes), + }); +} + +const manifest = Object.freeze({ + contractVersion: 1, + synthetic: true, + scripts: SCRIPT_LABELS, + profiles: profileManifest, +}); +writeFileSync( + resolve(outputDirectory, 'manifest.json'), + `${JSON.stringify(manifest, null, 2)}\n`, + 'utf8', +); From 0f941418de9bfa90c5ff1edf007017dc814b2605 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 07:45:08 -0700 Subject: [PATCH 003/209] test(perf): lock synthetic corpus identities --- benchmarks/corpus.lock.json | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 benchmarks/corpus.lock.json diff --git a/benchmarks/corpus.lock.json b/benchmarks/corpus.lock.json new file mode 100644 index 00000000..15a68e1c --- /dev/null +++ b/benchmarks/corpus.lock.json @@ -0,0 +1,34 @@ +{ + "contractVersion": 1, + "synthetic": true, + "scripts": [ + "English", + "Korean", + "Japanese", + "Chinese", + "Vietnamese", + "mixed" + ], + "profiles": { + "small": { + "sections": 1, + "bytes": 1577, + "sha256": "768c220ae809f29fd9e20234f55de93f05a532fe3a6844a083b8d9d1d80af2e7" + }, + "medium": { + "sections": 8, + "bytes": 11112, + "sha256": "5d1cebaf4e87374d2627a629dc7afc80cbc67e94f7ca447702eb3f836c6f4f50" + }, + "large": { + "sections": 32, + "bytes": 43799, + "sha256": "24bd29e0f4860d7a3ce44aaef74ddf4e0c6e747254b683000136500fd93487ed" + }, + "stress": { + "sections": 128, + "bytes": 174552, + "sha256": "ae486d9257ca3c227233833feef1d4a2a7f1e0110dd4e1973d5748f7873c53db" + } + } +} From b94f1246d6efebf345ae6cbb8efdcc558e4fc579 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 08:17:15 -0700 Subject: [PATCH 004/209] test(perf): require mixed script and raster size corpus --- src/performanceCorpusContract.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/performanceCorpusContract.test.ts b/src/performanceCorpusContract.test.ts index 5c2d6edd..7ab91cca 100644 --- a/src/performanceCorpusContract.test.ts +++ b/src/performanceCorpusContract.test.ts @@ -70,6 +70,17 @@ describe('deterministic synthetic performance corpus', () => { expect(firstBytes.equals(secondBytes)).toBe(true); expect(firstBytes.byteLength).toBe(expected.profiles[profile].bytes); } + + const smallBody = readFileSync(join(first, 'small.md'), 'utf8'); + expect(smallBody).toContain( + 'Mixed-script: Inkspan review 검증은 日本語と中文 그리고 Tiếng Việt를 한 문단에서 deterministic하게 다룹니다.', + ); + for (const dimensions of ['1x1', '16x16', '64x64']) { + expect(smallBody).toContain(`![synthetic raster ${dimensions} 0001](`); + } + expect(new Set(smallBody.match(/data:image\/png;base64,[A-Za-z0-9+/=]+/g))).toHaveLength( + 3, + ); } finally { rmSync(root, { recursive: true, force: true }); } From 1ff70213a49fb15778d0ae9d74a5dcdef101f87e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 08:18:19 -0700 Subject: [PATCH 005/209] feat(perf): add mixed script and raster size fixtures --- benchmarks/generate-corpus.mjs | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/benchmarks/generate-corpus.mjs b/benchmarks/generate-corpus.mjs index 4f582805..af73290e 100644 --- a/benchmarks/generate-corpus.mjs +++ b/benchmarks/generate-corpus.mjs @@ -2,14 +2,30 @@ import { createHash } from 'node:crypto'; import { mkdirSync, writeFileSync } from 'node:fs'; import { resolve } from 'node:path'; -const PIXEL_BASE64 = - 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII='; +const RASTER_FIXTURES = Object.freeze([ + Object.freeze({ + dimensions: '1x1', + base64: + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNQcEj4DwADBAHAX3ZiygAAAABJRU5ErkJggg==', + }), + Object.freeze({ + dimensions: '16x16', + base64: + 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAGUlEQVR42mNQcEj4TwlmGDVg1IBRA4aLAQDSpr8QG8NsyQAAAABJRU5ErkJggg==', + }), + Object.freeze({ + dimensions: '64x64', + base64: + 'iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAAZklEQVR42u3QQREAAAQAMFFEEUX/EuRw9liBRVbPZyFAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgTct/Bk8ZbOy3oMAAAAAElFTkSuQmCC', + }), +]); const SCRIPT_PARAGRAPHS = [ 'English: Deterministic authoring keeps document state explicit and reviewable.', '한국어: 결정적 작성 흐름은 문서 상태와 변경 근거를 명확하게 유지합니다.', '日本語: 決定的な編集フローは文書状態と変更根拠を明示的に保ちます。', '中文: 确定性的编辑流程会明确保留文档状态与变更依据。', 'Tiếng Việt: Luồng biên soạn xác định giữ trạng thái tài liệu và bằng chứng thay đổi rõ ràng.', + 'Mixed-script: Inkspan review 검증은 日本語と中文 그리고 Tiếng Việt를 한 문단에서 deterministic하게 다룹니다.', ]; const PROFILE_SECTIONS = Object.freeze({ small: 1, @@ -32,6 +48,10 @@ function buildSection(index) { const row = String(rowIndex + 1).padStart(2, '0'); return `| ${id}-r${row}c01 | ${id}-r${row}c02 | ${id}-r${row}c03 | ${id}-r${row}c04 | ${id}-r${row}c05 | ${id}-r${row}c06 |`; }); + const rasterRows = RASTER_FIXTURES.map( + ({ dimensions, base64 }) => + `![synthetic raster ${dimensions} ${id}](data:image/png;base64,${base64})`, + ); return [ `# Synthetic section ${id}`, '', @@ -55,7 +75,7 @@ function buildSection(index) { '| --- | --- | --- | --- | --- | --- |', ...tableRows, '', - `![synthetic 1x1 raster ${id}](data:image/png;base64,${PIXEL_BASE64})`, + ...rasterRows, '', '---', '', From 322dc641173073244e70f861601e5c7df75ffa22 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 08:18:36 -0700 Subject: [PATCH 006/209] test(perf): lock mixed script corpus identities --- benchmarks/corpus.lock.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/benchmarks/corpus.lock.json b/benchmarks/corpus.lock.json index 15a68e1c..fa2fe87c 100644 --- a/benchmarks/corpus.lock.json +++ b/benchmarks/corpus.lock.json @@ -12,23 +12,23 @@ "profiles": { "small": { "sections": 1, - "bytes": 1577, - "sha256": "768c220ae809f29fd9e20234f55de93f05a532fe3a6844a083b8d9d1d80af2e7" + "bytes": 2152, + "sha256": "420d18f2bb9e42d7e7e2cb5f74e67c90dfe15c3748b5d22875b4a6dc38ecbdea" }, "medium": { "sections": 8, - "bytes": 11112, - "sha256": "5d1cebaf4e87374d2627a629dc7afc80cbc67e94f7ca447702eb3f836c6f4f50" + "bytes": 15712, + "sha256": "921092809cc19be790c7a29a5457a7113e75aa7c76642d4ec09784b6096e045c" }, "large": { "sections": 32, - "bytes": 43799, - "sha256": "24bd29e0f4860d7a3ce44aaef74ddf4e0c6e747254b683000136500fd93487ed" + "bytes": 62199, + "sha256": "6ea32c0c8d2b58bf958dd28424a0b6139954fcefe8850943966be9e67a13b392" }, "stress": { "sections": 128, - "bytes": 174552, - "sha256": "ae486d9257ca3c227233833feef1d4a2a7f1e0110dd4e1973d5748f7873c53db" + "bytes": 248152, + "sha256": "5139848dc240863acb95ffdcf549fe7f151451a1002befc39c2d9c3395826928" } } } From 45a74f8f9cb01033d73a111b6a3f602acd96fb37 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 08:20:26 -0700 Subject: [PATCH 007/209] test(perf): assert distinct raster fixture count --- src/performanceCorpusContract.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/performanceCorpusContract.test.ts b/src/performanceCorpusContract.test.ts index 7ab91cca..e92c7f2c 100644 --- a/src/performanceCorpusContract.test.ts +++ b/src/performanceCorpusContract.test.ts @@ -78,9 +78,9 @@ describe('deterministic synthetic performance corpus', () => { for (const dimensions of ['1x1', '16x16', '64x64']) { expect(smallBody).toContain(`![synthetic raster ${dimensions} 0001](`); } - expect(new Set(smallBody.match(/data:image\/png;base64,[A-Za-z0-9+/=]+/g))).toHaveLength( - 3, - ); + expect( + new Set(smallBody.match(/data:image\/png;base64,[A-Za-z0-9+/=]+/g)).size, + ).toBe(3); } finally { rmSync(root, { recursive: true, force: true }); } From 5831e7c76986ee2984468bb7926caa709faa31eb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 09:01:18 -0700 Subject: [PATCH 008/209] test(perf): require deterministic Office benchmark fixtures --- src/performanceOfficeFixtureContract.test.ts | 82 ++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 src/performanceOfficeFixtureContract.test.ts diff --git a/src/performanceOfficeFixtureContract.test.ts b/src/performanceOfficeFixtureContract.test.ts new file mode 100644 index 00000000..d8a9094d --- /dev/null +++ b/src/performanceOfficeFixtureContract.test.ts @@ -0,0 +1,82 @@ +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +interface OfficeFixtureProfileLock { + readonly pages: number; + readonly blocks: number; + readonly bytes: number; + readonly sha256: string; +} + +interface OfficeFixtureLock { + readonly contractVersion: 1; + readonly synthetic: true; + readonly format: 'docx'; + readonly profiles: Readonly< + Record<'small' | 'page120', OfficeFixtureProfileLock> + >; +} + +function runGenerator(outputDirectory: string): OfficeFixtureLock { + const script = resolve(process.cwd(), 'benchmarks/generate-office-fixtures.mjs'); + execFileSync(process.execPath, [script, '--output', outputDirectory], { + cwd: process.cwd(), + stdio: ['ignore', 'pipe', 'pipe'], + }); + return JSON.parse( + readFileSync(join(outputDirectory, 'manifest.json'), 'utf8'), + ) as OfficeFixtureLock; +} + +describe('deterministic synthetic Office performance fixtures', () => { + it('reproduces a schema-shaped DOCX corpus including a 120-page fixture', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-')); + const first = join(root, 'first'); + const second = join(root, 'second'); + try { + const expected = JSON.parse( + readFileSync( + resolve(process.cwd(), 'benchmarks/office-fixtures.lock.json'), + 'utf8', + ), + ) as OfficeFixtureLock; + const firstManifest = runGenerator(first); + const secondManifest = runGenerator(second); + + expect(firstManifest).toEqual(expected); + expect(secondManifest).toEqual(expected); + expect(firstManifest).toEqual({ + contractVersion: 1, + synthetic: true, + format: 'docx', + profiles: expected.profiles, + }); + + for (const profile of ['small', 'page120'] as const) { + const firstBytes = readFileSync(join(first, `${profile}.json`)); + const secondBytes = readFileSync(join(second, `${profile}.json`)); + expect(firstBytes.equals(secondBytes)).toBe(true); + expect(firstBytes.byteLength).toBe(expected.profiles[profile].bytes); + } + + const page120 = JSON.parse( + readFileSync(join(first, 'page120.json'), 'utf8'), + ) as { + format: string; + blocks: Array<{ type: string; text?: string }>; + }; + expect(page120.format).toBe('docx'); + expect(page120.blocks.filter(({ type }) => type === 'heading')).toHaveLength(120); + expect(page120.blocks.filter(({ type }) => type === 'page_break')).toHaveLength(119); + expect(page120.blocks.some(({ text }) => text?.includes('한국어'))).toBe(true); + expect(page120.blocks.some(({ text }) => text?.includes('日本語'))).toBe(true); + expect(page120.blocks.some(({ text }) => text?.includes('中文'))).toBe(true); + expect(page120.blocks.some(({ text }) => text?.includes('Tiếng Việt'))).toBe(true); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 4f702a3d60152a904d6c64f480a34f38e9787d8d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 09:03:35 -0700 Subject: [PATCH 009/209] perf: generate deterministic Office benchmark fixtures --- benchmarks/generate-office-fixtures.mjs | 113 ++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 benchmarks/generate-office-fixtures.mjs diff --git a/benchmarks/generate-office-fixtures.mjs b/benchmarks/generate-office-fixtures.mjs new file mode 100644 index 00000000..76d71a4a --- /dev/null +++ b/benchmarks/generate-office-fixtures.mjs @@ -0,0 +1,113 @@ +import { createHash } from 'node:crypto'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +const PROFILE_PAGES = Object.freeze({ + small: 2, + page120: 120, +}); + +const MULTILINGUAL_PARAGRAPH = + 'English: deterministic Office rendering fixture. 한국어: 합성 성능 문서입니다. 日本語: 合成性能文書です。 中文: 这是合成性能文档。 Tiếng Việt: Đây là tài liệu hiệu năng tổng hợp.'; + +function buildPage(pageNumber) { + const page = String(pageNumber).padStart(3, '0'); + return [ + Object.freeze({ + type: 'heading', + level: 1, + text: `Synthetic page ${page}`, + }), + Object.freeze({ + type: 'paragraph', + text: `${MULTILINGUAL_PARAGRAPH} Page ${page}.`, + alignment: 'justify', + }), + Object.freeze({ + type: 'rich_paragraph', + runs: Object.freeze([ + Object.freeze({ text: `Page ${page} summary: `, bold: true }), + Object.freeze({ text: 'deterministic ', italic: true }), + Object.freeze({ text: 'Office rendering fixture.', underline: true }), + ]), + }), + Object.freeze({ + type: 'bullet_list', + ordered: false, + items: Object.freeze([ + `page ${page} item A`, + `page ${page} item B`, + `page ${page} item C`, + ]), + }), + Object.freeze({ + type: 'table', + headers: Object.freeze(['Page', 'Metric', 'Value']), + rows: Object.freeze([ + Object.freeze([page, 'latency-sample', pageNumber]), + Object.freeze([page, 'memory-sample', pageNumber * 2]), + Object.freeze([page, 'revision-sample', pageNumber * 3]), + Object.freeze([page, 'render-sample', pageNumber * 4]), + ]), + }), + ]; +} + +function buildRequest(profile, pages) { + const blocks = []; + for (let page = 1; page <= pages; page += 1) { + blocks.push(...buildPage(page)); + if (page < pages) { + blocks.push(Object.freeze({ type: 'page_break' })); + } + } + return Object.freeze({ + format: 'docx', + title: `Inkspan synthetic Office benchmark: ${profile}`, + author: 'Inkspan synthetic benchmark', + subject: 'Deterministic synthetic performance fixture', + blocks: Object.freeze(blocks), + }); +} + +function sha256(bytes) { + return createHash('sha256').update(bytes).digest('hex'); +} + +function resolveOutputDirectory(argv) { + if (argv.length !== 2 || argv[0] !== '--output' || argv[1].length === 0) { + throw new Error( + 'Usage: node benchmarks/generate-office-fixtures.mjs --output ', + ); + } + return resolve(argv[1]); +} + +const outputDirectory = resolveOutputDirectory(process.argv.slice(2)); +mkdirSync(outputDirectory, { recursive: true }); + +const profileManifest = {}; +for (const [profile, pages] of Object.entries(PROFILE_PAGES)) { + const request = buildRequest(profile, pages); + const body = `${JSON.stringify(request, null, 2)}\n`; + const bytes = Buffer.from(body, 'utf8'); + writeFileSync(resolve(outputDirectory, `${profile}.json`), bytes); + profileManifest[profile] = Object.freeze({ + pages, + blocks: request.blocks.length, + bytes: bytes.byteLength, + sha256: sha256(bytes), + }); +} + +const manifest = Object.freeze({ + contractVersion: 1, + synthetic: true, + format: 'docx', + profiles: profileManifest, +}); +writeFileSync( + resolve(outputDirectory, 'manifest.json'), + `${JSON.stringify(manifest, null, 2)}\n`, + 'utf8', +); From bf1fe4e8b17ac346bc827e47bf036c80c306e1ff Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 09:04:01 -0700 Subject: [PATCH 010/209] perf: lock deterministic Office benchmark fixtures --- benchmarks/office-fixtures.lock.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 benchmarks/office-fixtures.lock.json diff --git a/benchmarks/office-fixtures.lock.json b/benchmarks/office-fixtures.lock.json new file mode 100644 index 00000000..f3ff4beb --- /dev/null +++ b/benchmarks/office-fixtures.lock.json @@ -0,0 +1,19 @@ +{ + "contractVersion": 1, + "synthetic": true, + "format": "docx", + "profiles": { + "small": { + "pages": 2, + "blocks": 11, + "bytes": 2974, + "sha256": "9255bd3136bd5523169e495c365235a8b7bb7152092145ae41e2867f92dcc71a" + }, + "page120": { + "pages": 120, + "blocks": 719, + "bytes": 169739, + "sha256": "4df660c5ad762a516457d3005537861acf7c17478269744fca747e84fcbed3f9" + } + } +} From 0a67a3eec7035449a4973288d2b12f6e4e35dfa3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 09:11:16 -0700 Subject: [PATCH 011/209] test(perf): require XLSX and PPTX benchmark fixtures --- src/performanceOfficeFixtureContract.test.ts | 93 +++++++++++++++++--- 1 file changed, 79 insertions(+), 14 deletions(-) diff --git a/src/performanceOfficeFixtureContract.test.ts b/src/performanceOfficeFixtureContract.test.ts index d8a9094d..ebc1fa2d 100644 --- a/src/performanceOfficeFixtureContract.test.ts +++ b/src/performanceOfficeFixtureContract.test.ts @@ -5,8 +5,7 @@ import { join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; interface OfficeFixtureProfileLock { - readonly pages: number; - readonly blocks: number; + readonly units: number; readonly bytes: number; readonly sha256: string; } @@ -14,10 +13,17 @@ interface OfficeFixtureProfileLock { interface OfficeFixtureLock { readonly contractVersion: 1; readonly synthetic: true; - readonly format: 'docx'; - readonly profiles: Readonly< - Record<'small' | 'page120', OfficeFixtureProfileLock> - >; + readonly formats: Readonly<{ + docx: Readonly< + Record<'small' | 'page120', OfficeFixtureProfileLock> + >; + xlsx: Readonly< + Record<'small' | 'wide16384', OfficeFixtureProfileLock> + >; + pptx: Readonly< + Record<'small' | 'slide120', OfficeFixtureProfileLock> + >; + }>; } function runGenerator(outputDirectory: string): OfficeFixtureLock { @@ -31,8 +37,20 @@ function runGenerator(outputDirectory: string): OfficeFixtureLock { ) as OfficeFixtureLock; } +function expectDeterministicFixture( + first: string, + second: string, + fileName: string, + expectedBytes: number, +): void { + const firstBytes = readFileSync(join(first, fileName)); + const secondBytes = readFileSync(join(second, fileName)); + expect(firstBytes.equals(secondBytes)).toBe(true); + expect(firstBytes.byteLength).toBe(expectedBytes); +} + describe('deterministic synthetic Office performance fixtures', () => { - it('reproduces a schema-shaped DOCX corpus including a 120-page fixture', () => { + it('reproduces bounded DOCX, XLSX, and PPTX corpora including 100+ unit fixtures', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-')); const first = join(root, 'first'); const second = join(root, 'second'); @@ -51,19 +69,36 @@ describe('deterministic synthetic Office performance fixtures', () => { expect(firstManifest).toEqual({ contractVersion: 1, synthetic: true, - format: 'docx', - profiles: expected.profiles, + formats: expected.formats, }); for (const profile of ['small', 'page120'] as const) { - const firstBytes = readFileSync(join(first, `${profile}.json`)); - const secondBytes = readFileSync(join(second, `${profile}.json`)); - expect(firstBytes.equals(secondBytes)).toBe(true); - expect(firstBytes.byteLength).toBe(expected.profiles[profile].bytes); + expectDeterministicFixture( + first, + second, + `docx-${profile}.json`, + expected.formats.docx[profile].bytes, + ); + } + for (const profile of ['small', 'wide16384'] as const) { + expectDeterministicFixture( + first, + second, + `xlsx-${profile}.json`, + expected.formats.xlsx[profile].bytes, + ); + } + for (const profile of ['small', 'slide120'] as const) { + expectDeterministicFixture( + first, + second, + `pptx-${profile}.json`, + expected.formats.pptx[profile].bytes, + ); } const page120 = JSON.parse( - readFileSync(join(first, 'page120.json'), 'utf8'), + readFileSync(join(first, 'docx-page120.json'), 'utf8'), ) as { format: string; blocks: Array<{ type: string; text?: string }>; @@ -75,6 +110,36 @@ describe('deterministic synthetic Office performance fixtures', () => { expect(page120.blocks.some(({ text }) => text?.includes('日本語'))).toBe(true); expect(page120.blocks.some(({ text }) => text?.includes('中文'))).toBe(true); expect(page120.blocks.some(({ text }) => text?.includes('Tiếng Việt'))).toBe(true); + + const wide = JSON.parse( + readFileSync(join(first, 'xlsx-wide16384.json'), 'utf8'), + ) as { + format: string; + sheets: Array<{ rows: unknown[][]; freeze_panes?: string }>; + }; + expect(wide.format).toBe('xlsx'); + expect(wide.sheets).toHaveLength(1); + expect(wide.sheets[0]?.rows[0]).toHaveLength(16_384); + expect(wide.sheets[0]?.freeze_panes).toBe('XFD1048576'); + + const slide120 = JSON.parse( + readFileSync(join(first, 'pptx-slide120.json'), 'utf8'), + ) as { + format: string; + slides: Array<{ title: string; bullets?: Array }>; + }; + expect(slide120.format).toBe('pptx'); + expect(slide120.slides).toHaveLength(120); + expect(slide120.slides.some(({ title }) => title.includes('한국어'))).toBe(true); + expect( + slide120.slides.some(({ bullets }) => + bullets?.some((bullet) => + typeof bullet === 'string' + ? bullet.includes('日本語') + : bullet.text.includes('日本語'), + ), + ), + ).toBe(true); } finally { rmSync(root, { recursive: true, force: true }); } From f6e795d650433066cc319d220b796c76b987ab94 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 09:12:02 -0700 Subject: [PATCH 012/209] perf: generate deterministic XLSX and PPTX fixtures --- benchmarks/generate-office-fixtures.mjs | 140 +++++++++++++++++++++--- 1 file changed, 123 insertions(+), 17 deletions(-) diff --git a/benchmarks/generate-office-fixtures.mjs b/benchmarks/generate-office-fixtures.mjs index 76d71a4a..40a512df 100644 --- a/benchmarks/generate-office-fixtures.mjs +++ b/benchmarks/generate-office-fixtures.mjs @@ -2,15 +2,21 @@ import { createHash } from 'node:crypto'; import { mkdirSync, writeFileSync } from 'node:fs'; import { resolve } from 'node:path'; -const PROFILE_PAGES = Object.freeze({ +const DOCX_PROFILE_PAGES = Object.freeze({ small: 2, page120: 120, }); +const PPTX_PROFILE_SLIDES = Object.freeze({ + small: 2, + slide120: 120, +}); + +const EXCEL_MAX_COLUMNS = 16_384; const MULTILINGUAL_PARAGRAPH = 'English: deterministic Office rendering fixture. 한국어: 합성 성능 문서입니다. 日本語: 合成性能文書です。 中文: 这是合成性能文档。 Tiếng Việt: Đây là tài liệu hiệu năng tổng hợp.'; -function buildPage(pageNumber) { +function buildDocxPage(pageNumber) { const page = String(pageNumber).padStart(3, '0'); return [ Object.freeze({ @@ -53,23 +59,88 @@ function buildPage(pageNumber) { ]; } -function buildRequest(profile, pages) { +function buildDocxRequest(profile, pages) { const blocks = []; for (let page = 1; page <= pages; page += 1) { - blocks.push(...buildPage(page)); + blocks.push(...buildDocxPage(page)); if (page < pages) { blocks.push(Object.freeze({ type: 'page_break' })); } } return Object.freeze({ format: 'docx', - title: `Inkspan synthetic Office benchmark: ${profile}`, + title: `Inkspan synthetic DOCX benchmark: ${profile}`, author: 'Inkspan synthetic benchmark', subject: 'Deterministic synthetic performance fixture', blocks: Object.freeze(blocks), }); } +function buildXlsxRequest(profile) { + if (profile === 'wide16384') { + const row = Array.from( + { length: EXCEL_MAX_COLUMNS }, + (_, index) => `C${String(index + 1).padStart(5, '0')}`, + ); + return Object.freeze({ + format: 'xlsx', + title: 'Inkspan synthetic XLSX benchmark: wide16384', + author: 'Inkspan synthetic benchmark', + sheets: Object.freeze([ + Object.freeze({ + name: 'Wide16384', + rows: Object.freeze([Object.freeze(row)]), + freeze_panes: 'XFD1048576', + }), + ]), + }); + } + + return Object.freeze({ + format: 'xlsx', + title: 'Inkspan synthetic XLSX benchmark: small', + author: 'Inkspan synthetic benchmark', + sheets: Object.freeze([ + Object.freeze({ + name: 'Synthetic', + header_row: true, + auto_filter: true, + freeze_panes: 'B2', + rows: Object.freeze([ + Object.freeze(['Language', 'Text', 'Latency', 'Memory']), + Object.freeze(['한국어', '합성 성능 문서', 1, 2]), + Object.freeze(['日本語', '合成性能文書', 3, 4]), + Object.freeze(['中文 / Tiếng Việt', '合成文档 / tài liệu tổng hợp', 5, 6]), + ]), + }), + ]), + }); +} + +function buildPptxSlide(slideNumber) { + const slide = String(slideNumber).padStart(3, '0'); + return Object.freeze({ + title: `한국어 합성 슬라이드 ${slide}`, + bullets: Object.freeze([ + `English deterministic slide ${slide}`, + Object.freeze({ text: `日本語 合成スライド ${slide}`, level: 0 }), + Object.freeze({ text: `中文 合成幻灯片 ${slide}`, level: 1 }), + Object.freeze({ text: `Tiếng Việt trang chiếu ${slide}`, level: 1 }), + ]), + }); +} + +function buildPptxRequest(profile, slides) { + return Object.freeze({ + format: 'pptx', + title: `Inkspan synthetic PPTX benchmark: ${profile}`, + author: 'Inkspan synthetic benchmark', + slides: Object.freeze( + Array.from({ length: slides }, (_, index) => buildPptxSlide(index + 1)), + ), + }); +} + function sha256(bytes) { return createHash('sha256').update(bytes).digest('hex'); } @@ -83,28 +154,63 @@ function resolveOutputDirectory(argv) { return resolve(argv[1]); } -const outputDirectory = resolveOutputDirectory(process.argv.slice(2)); -mkdirSync(outputDirectory, { recursive: true }); - -const profileManifest = {}; -for (const [profile, pages] of Object.entries(PROFILE_PAGES)) { - const request = buildRequest(profile, pages); +function writeFixture(outputDirectory, fileName, request, units) { const body = `${JSON.stringify(request, null, 2)}\n`; const bytes = Buffer.from(body, 'utf8'); - writeFileSync(resolve(outputDirectory, `${profile}.json`), bytes); - profileManifest[profile] = Object.freeze({ - pages, - blocks: request.blocks.length, + writeFileSync(resolve(outputDirectory, fileName), bytes); + return Object.freeze({ + units, bytes: bytes.byteLength, sha256: sha256(bytes), }); } +const outputDirectory = resolveOutputDirectory(process.argv.slice(2)); +mkdirSync(outputDirectory, { recursive: true }); + +const docx = {}; +for (const [profile, pages] of Object.entries(DOCX_PROFILE_PAGES)) { + docx[profile] = writeFixture( + outputDirectory, + `docx-${profile}.json`, + buildDocxRequest(profile, pages), + pages, + ); +} + +const xlsx = Object.freeze({ + small: writeFixture( + outputDirectory, + 'xlsx-small.json', + buildXlsxRequest('small'), + 4, + ), + wide16384: writeFixture( + outputDirectory, + 'xlsx-wide16384.json', + buildXlsxRequest('wide16384'), + EXCEL_MAX_COLUMNS, + ), +}); + +const pptx = {}; +for (const [profile, slides] of Object.entries(PPTX_PROFILE_SLIDES)) { + pptx[profile] = writeFixture( + outputDirectory, + `pptx-${profile}.json`, + buildPptxRequest(profile, slides), + slides, + ); +} + const manifest = Object.freeze({ contractVersion: 1, synthetic: true, - format: 'docx', - profiles: profileManifest, + formats: Object.freeze({ + docx: Object.freeze(docx), + xlsx, + pptx: Object.freeze(pptx), + }), }); writeFileSync( resolve(outputDirectory, 'manifest.json'), From 940275af5f0ec3c191ac01dee78f691cc557fe1a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 09:12:45 -0700 Subject: [PATCH 013/209] perf: lock deterministic Office fixture matrix --- benchmarks/office-fixtures.lock.json | 47 +++++++++++++++++++++------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/benchmarks/office-fixtures.lock.json b/benchmarks/office-fixtures.lock.json index f3ff4beb..01f9920e 100644 --- a/benchmarks/office-fixtures.lock.json +++ b/benchmarks/office-fixtures.lock.json @@ -1,19 +1,42 @@ { "contractVersion": 1, "synthetic": true, - "format": "docx", - "profiles": { - "small": { - "pages": 2, - "blocks": 11, - "bytes": 2974, - "sha256": "9255bd3136bd5523169e495c365235a8b7bb7152092145ae41e2867f92dcc71a" + "formats": { + "docx": { + "small": { + "units": 2, + "bytes": 2972, + "sha256": "c356496b106f5348e98b00d3c1e18185165653a2c2bda083acd7456c96b2eab3" + }, + "page120": { + "units": 120, + "bytes": 169737, + "sha256": "e5d90408c6061051ab1674d931b99d478fadb00a7ecbcb7025b4fa478cfbb507" + } }, - "page120": { - "pages": 120, - "blocks": 719, - "bytes": 169739, - "sha256": "4df660c5ad762a516457d3005537861acf7c17478269744fca747e84fcbed3f9" + "xlsx": { + "small": { + "units": 4, + "bytes": 723, + "sha256": "a267a6208de0e24559c200d047404d14e2263581e0a3af845f355817536883bf" + }, + "wide16384": { + "units": 16384, + "bytes": 327941, + "sha256": "0afdb216fda720cd506d7c24284c2740f326167d4859dc1a7b845c4a91b972ec" + } + }, + "pptx": { + "small": { + "units": 2, + "bytes": 970, + "sha256": "6c1a8ae1d2307a278ac97903eb3160a6a0de57b695b2b8772844af2c8cfa2ce1" + }, + "slide120": { + "units": 120, + "bytes": 50061, + "sha256": "f5994ff30752581fd3b3e5210ff59b281692ea268ce4f0b540e278d134dbd6ee" + } } } } From 326e4ded064ecfc66ea0bf3d37f60fd746ddc383 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 11:06:20 -0700 Subject: [PATCH 014/209] test(perf): require deterministic benchmark statistics summary --- ...manceMeasurementStatisticsContract.test.ts | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 src/performanceMeasurementStatisticsContract.test.ts diff --git a/src/performanceMeasurementStatisticsContract.test.ts b/src/performanceMeasurementStatisticsContract.test.ts new file mode 100644 index 00000000..6085aa25 --- /dev/null +++ b/src/performanceMeasurementStatisticsContract.test.ts @@ -0,0 +1,126 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +interface BenchmarkSummary { + readonly contractVersion: 1; + readonly benchmarkId: string; + readonly unit: string; + readonly sampleCount: number; + readonly percentileMethod: 'nearest-rank'; + readonly minimum: number; + readonly p50: number; + readonly p75: number; + readonly p95: number; + readonly maximum: number; +} + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +function writeInput(path: string, samples: readonly number[]): void { + writeFileSync( + path, + `${JSON.stringify( + { + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + samples, + }, + null, + 2, + )}\n`, + 'utf8', + ); +} + +function runSummary(inputPath: string, outputDirectory: string): BenchmarkSummary { + execFileSync( + process.execPath, + [script, '--input', inputPath, '--output', outputDirectory], + { + cwd: process.cwd(), + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return JSON.parse( + readFileSync(join(outputDirectory, 'summary.json'), 'utf8'), + ) as BenchmarkSummary; +} + +describe('deterministic benchmark sample statistics', () => { + it('writes reproducible nearest-rank JSON and human-readable summaries', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-')); + const input = join(root, 'samples.json'); + const first = join(root, 'first'); + const second = join(root, 'second'); + try { + writeInput(input, [20, 10, 40, 30, 50]); + + const firstSummary = runSummary(input, first); + const secondSummary = runSummary(input, second); + expect(firstSummary).toEqual(secondSummary); + expect(firstSummary).toEqual({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sampleCount: 5, + percentileMethod: 'nearest-rank', + minimum: 10, + p50: 30, + p75: 40, + p95: 50, + maximum: 50, + }); + + const expectedText = [ + 'benchmark=markdown-serialization-large', + 'unit=ms', + 'samples=5', + 'percentile_method=nearest-rank', + 'minimum=10', + 'p50=30', + 'p75=40', + 'p95=50', + 'maximum=50', + '', + ].join('\n'); + expect(readFileSync(join(first, 'summary.txt'), 'utf8')).toBe(expectedText); + expect(readFileSync(join(second, 'summary.txt'), 'utf8')).toBe(expectedText); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed on invalid measurement samples without coercion', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-invalid-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + writeInput(input, [1, -1, 3]); + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark samples must be finite non-negative numbers.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 897a97838d544d41ab80f694889a6f22f3b9feb8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 11:06:55 -0700 Subject: [PATCH 015/209] feat(perf): add deterministic benchmark statistics summary --- benchmarks/summarize-samples.mjs | 139 +++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 benchmarks/summarize-samples.mjs diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs new file mode 100644 index 00000000..f695ae03 --- /dev/null +++ b/benchmarks/summarize-samples.mjs @@ -0,0 +1,139 @@ +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +const MAX_INPUT_BYTES = 16 * 1024 * 1024; +const MAX_SAMPLES = 1_000_000; +const BENCHMARK_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; +const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; + +function resolveArguments(argv) { + if ( + argv.length !== 4 || + argv[0] !== '--input' || + argv[1].length === 0 || + argv[2] !== '--output' || + argv[3].length === 0 + ) { + throw new Error( + 'Usage: node benchmarks/summarize-samples.mjs --input --output ', + ); + } + return Object.freeze({ + inputPath: resolve(argv[1]), + outputDirectory: resolve(argv[3]), + }); +} + +function readBoundedJson(path) { + const bytes = readFileSync(path); + if (bytes.byteLength > MAX_INPUT_BYTES) { + throw new Error('Benchmark sample input exceeds the supported size.'); + } + let parsed; + try { + parsed = JSON.parse(bytes.toString('utf8')); + } catch { + throw new Error('Benchmark sample input must be valid JSON.'); + } + return parsed; +} + +function validateInput(value) { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Benchmark sample input must be an object.'); + } + if (value.contractVersion !== 1) { + throw new Error('Benchmark sample contractVersion must be 1.'); + } + if ( + typeof value.benchmarkId !== 'string' || + !BENCHMARK_ID_PATTERN.test(value.benchmarkId) + ) { + throw new Error('Benchmark benchmarkId is invalid.'); + } + if (typeof value.unit !== 'string' || !UNIT_PATTERN.test(value.unit)) { + throw new Error('Benchmark unit is invalid.'); + } + if ( + !Array.isArray(value.samples) || + value.samples.length === 0 || + value.samples.length > MAX_SAMPLES + ) { + throw new Error('Benchmark samples must be a non-empty bounded array.'); + } + if ( + value.samples.some( + (sample) => + typeof sample !== 'number' || !Number.isFinite(sample) || sample < 0, + ) + ) { + throw new Error('Benchmark samples must be finite non-negative numbers.'); + } + return Object.freeze({ + benchmarkId: value.benchmarkId, + unit: value.unit, + samples: Object.freeze([...value.samples]), + }); +} + +function nearestRank(sorted, percentile) { + const index = Math.ceil(percentile * sorted.length) - 1; + return sorted[index]; +} + +function summarize(input) { + const sorted = [...input.samples].sort((left, right) => left - right); + return Object.freeze({ + contractVersion: 1, + benchmarkId: input.benchmarkId, + unit: input.unit, + sampleCount: sorted.length, + percentileMethod: 'nearest-rank', + minimum: sorted[0], + p50: nearestRank(sorted, 0.5), + p75: nearestRank(sorted, 0.75), + p95: nearestRank(sorted, 0.95), + maximum: sorted.at(-1), + }); +} + +function formatSummary(summary) { + return [ + `benchmark=${summary.benchmarkId}`, + `unit=${summary.unit}`, + `samples=${summary.sampleCount}`, + `percentile_method=${summary.percentileMethod}`, + `minimum=${summary.minimum}`, + `p50=${summary.p50}`, + `p75=${summary.p75}`, + `p95=${summary.p95}`, + `maximum=${summary.maximum}`, + '', + ].join('\n'); +} + +function main() { + const { inputPath, outputDirectory } = resolveArguments(process.argv.slice(2)); + const input = validateInput(readBoundedJson(inputPath)); + const summary = summarize(input); + mkdirSync(outputDirectory, { recursive: true }); + writeFileSync( + resolve(outputDirectory, 'summary.json'), + `${JSON.stringify(summary, null, 2)}\n`, + 'utf8', + ); + writeFileSync( + resolve(outputDirectory, 'summary.txt'), + formatSummary(summary), + 'utf8', + ); +} + +try { + main(); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Benchmark summary failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} From 6a4dd666b579e6a25a473d0829fc9ef881fe1d71 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 16:11:48 -0700 Subject: [PATCH 016/209] test(perf): reject benchmark input output alias --- ...manceMeasurementStatisticsContract.test.ts | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/performanceMeasurementStatisticsContract.test.ts b/src/performanceMeasurementStatisticsContract.test.ts index 6085aa25..e229bb8f 100644 --- a/src/performanceMeasurementStatisticsContract.test.ts +++ b/src/performanceMeasurementStatisticsContract.test.ts @@ -1,5 +1,6 @@ import { execFileSync, spawnSync } from 'node:child_process'; import { + existsSync, mkdtempSync, readFileSync, rmSync, @@ -123,4 +124,32 @@ describe('deterministic benchmark sample statistics', () => { rmSync(root, { recursive: true, force: true }); } }); -}); + + it('refuses to overwrite the measurement input with generated evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-alias-')); + const input = join(root, 'summary.json'); + const output = root; + try { + writeInput(input, [10, 20, 30]); + const originalInput = readFileSync(input, 'utf8'); + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark output must not overwrite the sample input.', + ); + expect(readFileSync(input, 'utf8')).toBe(originalInput); + expect(existsSync(join(root, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); \ No newline at end of file From a838dbefb3a410d9899985ae1f011e71af3a9c3d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 16:15:20 -0700 Subject: [PATCH 017/209] fix(perf): preserve benchmark sample inputs --- benchmarks/summarize-samples.mjs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index f695ae03..a20b0d2b 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -114,19 +114,20 @@ function formatSummary(summary) { function main() { const { inputPath, outputDirectory } = resolveArguments(process.argv.slice(2)); + const summaryJsonPath = resolve(outputDirectory, 'summary.json'); + const summaryTextPath = resolve(outputDirectory, 'summary.txt'); + if (inputPath === summaryJsonPath || inputPath === summaryTextPath) { + throw new Error('Benchmark output must not overwrite the sample input.'); + } const input = validateInput(readBoundedJson(inputPath)); const summary = summarize(input); mkdirSync(outputDirectory, { recursive: true }); writeFileSync( - resolve(outputDirectory, 'summary.json'), + summaryJsonPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8', ); - writeFileSync( - resolve(outputDirectory, 'summary.txt'), - formatSummary(summary), - 'utf8', - ); + writeFileSync(summaryTextPath, formatSummary(summary), 'utf8'); } try { From fe613959efed98bd4490192cfffbe44adeab8758 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 20:35:13 -0700 Subject: [PATCH 018/209] test(perf): reject hard-linked summary evidence aliases --- ...manceMeasurementStatisticsContract.test.ts | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/performanceMeasurementStatisticsContract.test.ts b/src/performanceMeasurementStatisticsContract.test.ts index e229bb8f..4e9853f6 100644 --- a/src/performanceMeasurementStatisticsContract.test.ts +++ b/src/performanceMeasurementStatisticsContract.test.ts @@ -1,6 +1,8 @@ import { execFileSync, spawnSync } from 'node:child_process'; import { existsSync, + linkSync, + mkdirSync, mkdtempSync, readFileSync, rmSync, @@ -152,4 +154,36 @@ describe('deterministic benchmark sample statistics', () => { rmSync(root, { recursive: true, force: true }); } }); -}); \ No newline at end of file + + it('refuses a hard-linked output alias without mutating source evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-hardlink-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const summaryJson = join(output, 'summary.json'); + try { + writeInput(input, [10, 20, 30]); + mkdirSync(output, { recursive: true }); + linkSync(input, summaryJson); + const originalInput = readFileSync(input, 'utf8'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark output must not overwrite the sample input.', + ); + expect(readFileSync(input, 'utf8')).toBe(originalInput); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 33d5c45b1dbfa62b886371b9642c12d04687ad6b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 20:35:41 -0700 Subject: [PATCH 019/209] fix(perf): preserve hard-linked benchmark source evidence --- benchmarks/summarize-samples.mjs | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index a20b0d2b..52306ccf 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -1,4 +1,10 @@ -import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { + existsSync, + mkdirSync, + readFileSync, + statSync, + writeFileSync, +} from 'node:fs'; import { resolve } from 'node:path'; const MAX_INPUT_BYTES = 16 * 1024 * 1024; @@ -112,16 +118,28 @@ function formatSummary(summary) { ].join('\n'); } +function refersToSameFile(leftPath, rightPath) { + if (!existsSync(leftPath) || !existsSync(rightPath)) return false; + const left = statSync(leftPath); + const right = statSync(rightPath); + return left.dev === right.dev && left.ino === right.ino; +} + function main() { const { inputPath, outputDirectory } = resolveArguments(process.argv.slice(2)); const summaryJsonPath = resolve(outputDirectory, 'summary.json'); const summaryTextPath = resolve(outputDirectory, 'summary.txt'); - if (inputPath === summaryJsonPath || inputPath === summaryTextPath) { + mkdirSync(outputDirectory, { recursive: true }); + if ( + inputPath === summaryJsonPath || + inputPath === summaryTextPath || + refersToSameFile(inputPath, summaryJsonPath) || + refersToSameFile(inputPath, summaryTextPath) + ) { throw new Error('Benchmark output must not overwrite the sample input.'); } const input = validateInput(readBoundedJson(inputPath)); const summary = summarize(input); - mkdirSync(outputDirectory, { recursive: true }); writeFileSync( summaryJsonPath, `${JSON.stringify(summary, null, 2)}\n`, From 676555f04ec9955e127a9921700a1457c09b8ae1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 21:23:59 -0700 Subject: [PATCH 020/209] test(perf): prove oversized samples preflight whole-file reads --- ...manceMeasurementStatisticsContract.test.ts | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/performanceMeasurementStatisticsContract.test.ts b/src/performanceMeasurementStatisticsContract.test.ts index 4e9853f6..d953ac03 100644 --- a/src/performanceMeasurementStatisticsContract.test.ts +++ b/src/performanceMeasurementStatisticsContract.test.ts @@ -6,10 +6,12 @@ import { mkdtempSync, readFileSync, rmSync, + truncateSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; import { describe, expect, it } from 'vitest'; interface BenchmarkSummary { @@ -127,6 +129,50 @@ describe('deterministic benchmark sample statistics', () => { } }); + it('rejects obviously oversized sample input before whole-file reads', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-size-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const preload = join(root, 'reject-whole-file-read.mjs'); + try { + writeFileSync(input, '', 'utf8'); + truncateSync(input, 16 * 1024 * 1024 + 1); + writeFileSync( + preload, + `import fs from 'node:fs';\nimport { syncBuiltinESMExports } from 'node:module';\nfs.readFileSync = () => { throw new Error('benchmark whole-file read sentinel'); };\nsyncBuiltinESMExports();\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + '--import', + pathToFileURL(preload).href, + script, + '--input', + input, + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input exceeds the supported size.', + ); + expect(result.stderr).not.toContain('benchmark whole-file read sentinel'); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('refuses to overwrite the measurement input with generated evidence', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-alias-')); const input = join(root, 'summary.json'); From 72db249bd0ddc9d65fc2081c374573d0688d6bb2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 21:24:46 -0700 Subject: [PATCH 021/209] fix(perf): bound benchmark sample reads before allocation --- benchmarks/summarize-samples.mjs | 56 ++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 52306ccf..0daddcca 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -1,13 +1,17 @@ import { + closeSync, existsSync, + fstatSync, mkdirSync, - readFileSync, + openSync, + readSync, statSync, writeFileSync, } from 'node:fs'; import { resolve } from 'node:path'; const MAX_INPUT_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000_000; const BENCHMARK_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; @@ -31,17 +35,49 @@ function resolveArguments(argv) { } function readBoundedJson(path) { - const bytes = readFileSync(path); - if (bytes.byteLength > MAX_INPUT_BYTES) { - throw new Error('Benchmark sample input exceeds the supported size.'); - } - let parsed; + const descriptor = openSync(path, 'r'); try { - parsed = JSON.parse(bytes.toString('utf8')); - } catch { - throw new Error('Benchmark sample input must be valid JSON.'); + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error('Benchmark sample input must be a regular file.'); + } + if (metadata.size > MAX_INPUT_BYTES) { + throw new Error('Benchmark sample input exceeds the supported size.'); + } + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= MAX_INPUT_BYTES) { + const remainingBudget = MAX_INPUT_BYTES + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > MAX_INPUT_BYTES) { + throw new Error('Benchmark sample input exceeds the supported size.'); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + + const bytes = Buffer.concat(chunks, totalBytes); + let parsed; + try { + parsed = JSON.parse(bytes.toString('utf8')); + } catch { + throw new Error('Benchmark sample input must be valid JSON.'); + } + return parsed; + } finally { + closeSync(descriptor); } - return parsed; } function validateInput(value) { From 9766bcfbdfeef398a179e5f05ae15ebc991406d9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 22:20:11 -0700 Subject: [PATCH 022/209] test(perf): require immutable benchmark provenance --- ...manceMeasurementStatisticsContract.test.ts | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/src/performanceMeasurementStatisticsContract.test.ts b/src/performanceMeasurementStatisticsContract.test.ts index d953ac03..3c477543 100644 --- a/src/performanceMeasurementStatisticsContract.test.ts +++ b/src/performanceMeasurementStatisticsContract.test.ts @@ -18,6 +18,11 @@ interface BenchmarkSummary { readonly contractVersion: 1; readonly benchmarkId: string; readonly unit: string; + readonly sourceCommitSha: string; + readonly artifactSha256: string; + readonly documentProfile: 'small' | 'medium' | 'large' | 'stress'; + readonly runtimeId: string; + readonly referenceHardwareId: string; readonly sampleCount: number; readonly percentileMethod: 'nearest-rank'; readonly minimum: number; @@ -28,6 +33,8 @@ interface BenchmarkSummary { } const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); function writeInput(path: string, samples: readonly number[]): void { writeFileSync( @@ -37,6 +44,11 @@ function writeInput(path: string, samples: readonly number[]): void { contractVersion: 1, benchmarkId: 'markdown-serialization-large', unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', samples, }, null, @@ -61,7 +73,7 @@ function runSummary(inputPath: string, outputDirectory: string): BenchmarkSummar } describe('deterministic benchmark sample statistics', () => { - it('writes reproducible nearest-rank JSON and human-readable summaries', () => { + it('writes reproducible nearest-rank JSON and human-readable summaries with provenance metadata', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-')); const input = join(root, 'samples.json'); const first = join(root, 'first'); @@ -76,6 +88,11 @@ describe('deterministic benchmark sample statistics', () => { contractVersion: 1, benchmarkId: 'markdown-serialization-large', unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', sampleCount: 5, percentileMethod: 'nearest-rank', minimum: 10, @@ -88,6 +105,11 @@ describe('deterministic benchmark sample statistics', () => { const expectedText = [ 'benchmark=markdown-serialization-large', 'unit=ms', + `source_commit_sha=${SOURCE_COMMIT_SHA}`, + `artifact_sha256=${ARTIFACT_SHA256}`, + 'document_profile=large', + 'runtime_id=node-22.18.0', + 'reference_hardware_id=github-actions-ubuntu-24.04-x64', 'samples=5', 'percentile_method=nearest-rank', 'minimum=10', @@ -104,6 +126,42 @@ describe('deterministic benchmark sample statistics', () => { } }); + it('fails closed when immutable provenance metadata is missing', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-metadata-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + writeFileSync( + input, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + samples: [1, 2, 3], + })}\n`, + 'utf8', + ); + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sourceCommitSha must be a lowercase 40-character commit SHA.', + ); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('fails closed on invalid measurement samples without coercion', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-invalid-')); const input = join(root, 'samples.json'); From 738d3e7bc67905bed03fddc06af84f8f8c0cbd89 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 22:21:15 -0700 Subject: [PATCH 023/209] feat(perf): bind summaries to immutable provenance --- benchmarks/summarize-samples.mjs | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 0daddcca..1c82a1fc 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -15,6 +15,10 @@ const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000_000; const BENCHMARK_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const EVIDENCE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); function resolveArguments(argv) { if ( @@ -96,6 +100,40 @@ function validateInput(value) { if (typeof value.unit !== 'string' || !UNIT_PATTERN.test(value.unit)) { throw new Error('Benchmark unit is invalid.'); } + if ( + typeof value.sourceCommitSha !== 'string' || + !SHA1_PATTERN.test(value.sourceCommitSha) + ) { + throw new Error( + 'Benchmark sourceCommitSha must be a lowercase 40-character commit SHA.', + ); + } + if ( + typeof value.artifactSha256 !== 'string' || + !SHA256_PATTERN.test(value.artifactSha256) + ) { + throw new Error( + 'Benchmark artifactSha256 must be a lowercase 64-character SHA-256 digest.', + ); + } + if ( + typeof value.documentProfile !== 'string' || + !DOCUMENT_PROFILES.has(value.documentProfile) + ) { + throw new Error('Benchmark documentProfile is invalid.'); + } + if ( + typeof value.runtimeId !== 'string' || + !EVIDENCE_ID_PATTERN.test(value.runtimeId) + ) { + throw new Error('Benchmark runtimeId is invalid.'); + } + if ( + typeof value.referenceHardwareId !== 'string' || + !EVIDENCE_ID_PATTERN.test(value.referenceHardwareId) + ) { + throw new Error('Benchmark referenceHardwareId is invalid.'); + } if ( !Array.isArray(value.samples) || value.samples.length === 0 || @@ -114,6 +152,11 @@ function validateInput(value) { return Object.freeze({ benchmarkId: value.benchmarkId, unit: value.unit, + sourceCommitSha: value.sourceCommitSha, + artifactSha256: value.artifactSha256, + documentProfile: value.documentProfile, + runtimeId: value.runtimeId, + referenceHardwareId: value.referenceHardwareId, samples: Object.freeze([...value.samples]), }); } @@ -129,6 +172,11 @@ function summarize(input) { contractVersion: 1, benchmarkId: input.benchmarkId, unit: input.unit, + sourceCommitSha: input.sourceCommitSha, + artifactSha256: input.artifactSha256, + documentProfile: input.documentProfile, + runtimeId: input.runtimeId, + referenceHardwareId: input.referenceHardwareId, sampleCount: sorted.length, percentileMethod: 'nearest-rank', minimum: sorted[0], @@ -143,6 +191,11 @@ function formatSummary(summary) { return [ `benchmark=${summary.benchmarkId}`, `unit=${summary.unit}`, + `source_commit_sha=${summary.sourceCommitSha}`, + `artifact_sha256=${summary.artifactSha256}`, + `document_profile=${summary.documentProfile}`, + `runtime_id=${summary.runtimeId}`, + `reference_hardware_id=${summary.referenceHardwareId}`, `samples=${summary.sampleCount}`, `percentile_method=${summary.percentileMethod}`, `minimum=${summary.minimum}`, From 864fa8b9a6bd69312ca30b05dff1b7f6ae6ac4cd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 00:33:13 -0700 Subject: [PATCH 024/209] test(perf): require strict UTF-8 benchmark evidence --- ...performanceMeasurementUtf8Contract.test.ts | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 src/performanceMeasurementUtf8Contract.test.ts diff --git a/src/performanceMeasurementUtf8Contract.test.ts b/src/performanceMeasurementUtf8Contract.test.ts new file mode 100644 index 00000000..45ceec9f --- /dev/null +++ b/src/performanceMeasurementUtf8Contract.test.ts @@ -0,0 +1,63 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +describe('benchmark evidence UTF-8 contract', () => { + it('rejects malformed UTF-8 before parsing or generating summary evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-utf8-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + const prefix = [ + '{"contractVersion":1,', + '"benchmarkId":"markdown-serialization-large",', + '"unit":"ms",', + `"sourceCommitSha":"${SOURCE_COMMIT_SHA}",`, + `"artifactSha256":"${ARTIFACT_SHA256}",`, + '"documentProfile":"large",', + '"runtimeId":"node-22.18.0",', + '"referenceHardwareId":"github-actions-ubuntu-24.04-x64",', + '"samples":[1,2,3],', + '"untrustedNote":"', + ].join(''); + writeFileSync( + input, + Buffer.concat([ + Buffer.from(prefix, 'utf8'), + Buffer.from([0x80]), + Buffer.from('"}\n', 'utf8'), + ]), + ); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input must be valid UTF-8 JSON.', + ); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From bc887c0095fb2d97d13c86be3d4b8bf67cccd2cb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 00:36:02 -0700 Subject: [PATCH 025/209] fix(perf): reject malformed UTF-8 benchmark evidence --- benchmarks/summarize-samples.mjs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 1c82a1fc..39b6a139 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -19,6 +19,7 @@ const SHA1_PATTERN = /^[0-9a-f]{40}$/u; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; const EVIDENCE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true }); function resolveArguments(argv) { if ( @@ -72,9 +73,15 @@ function readBoundedJson(path) { } const bytes = Buffer.concat(chunks, totalBytes); + let text; + try { + text = UTF8_DECODER.decode(bytes); + } catch { + throw new Error('Benchmark sample input must be valid UTF-8 JSON.'); + } let parsed; try { - parsed = JSON.parse(bytes.toString('utf8')); + parsed = JSON.parse(text); } catch { throw new Error('Benchmark sample input must be valid JSON.'); } From 596c48e6055e0c717b09af05a4d40d84db34e7ba Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 00:43:41 -0700 Subject: [PATCH 026/209] test(perf): preflight invalid summary destinations --- ...rformanceMeasurementOutputContract.test.ts | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 src/performanceMeasurementOutputContract.test.ts diff --git a/src/performanceMeasurementOutputContract.test.ts b/src/performanceMeasurementOutputContract.test.ts new file mode 100644 index 00000000..aca7928e --- /dev/null +++ b/src/performanceMeasurementOutputContract.test.ts @@ -0,0 +1,61 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +describe('benchmark summary output contract', () => { + it('rejects an invalid second destination before publishing the first summary', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-output-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const summaryJson = join(output, 'summary.json'); + const summaryText = join(output, 'summary.txt'); + try { + writeFileSync( + input, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [1, 2, 3], + })}\n`, + 'utf8', + ); + mkdirSync(summaryText, { recursive: true }); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output paths must be regular files.', + ); + expect(existsSync(summaryJson)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From f072e74983c70375af66f3718b71ff48b76ddc7d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 00:48:25 -0700 Subject: [PATCH 027/209] fix(perf): preflight benchmark output destinations --- benchmarks/summarize-samples.mjs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 39b6a139..17fce6a7 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -2,6 +2,7 @@ import { closeSync, existsSync, fstatSync, + lstatSync, mkdirSync, openSync, readSync, @@ -221,6 +222,12 @@ function refersToSameFile(leftPath, rightPath) { return left.dev === right.dev && left.ino === right.ino; } +function assertRegularOutputDestination(path) { + if (existsSync(path) && !lstatSync(path).isFile()) { + throw new Error('Benchmark summary output paths must be regular files.'); + } +} + function main() { const { inputPath, outputDirectory } = resolveArguments(process.argv.slice(2)); const summaryJsonPath = resolve(outputDirectory, 'summary.json'); @@ -234,6 +241,8 @@ function main() { ) { throw new Error('Benchmark output must not overwrite the sample input.'); } + assertRegularOutputDestination(summaryJsonPath); + assertRegularOutputDestination(summaryTextPath); const input = validateInput(readBoundedJson(inputPath)); const summary = summarize(input); writeFileSync( From 3efb68cd7271226a42ece1ca5755d05b0a6ccb1b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 02:09:52 -0700 Subject: [PATCH 028/209] test(perf): reject arbitrary benchmark metadata --- ...formanceMeasurementPrivacyContract.test.ts | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 src/performanceMeasurementPrivacyContract.test.ts diff --git a/src/performanceMeasurementPrivacyContract.test.ts b/src/performanceMeasurementPrivacyContract.test.ts new file mode 100644 index 00000000..83dfccfd --- /dev/null +++ b/src/performanceMeasurementPrivacyContract.test.ts @@ -0,0 +1,59 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +describe('benchmark evidence privacy contract', () => { + it('rejects unsupported metadata instead of accepting arbitrary evidence payloads', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-privacy-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + writeFileSync( + input, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [10, 20, 30], + prompt: 'must-not-enter-benchmark-evidence', + })}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input contains unsupported fields.', + ); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 6cb64186e37fbfa1e1765fd8df31996915e9e849 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 02:13:41 -0700 Subject: [PATCH 029/209] fix(perf): reject unsupported benchmark metadata --- benchmarks/summarize-samples.mjs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 17fce6a7..b457f71c 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -20,6 +20,17 @@ const SHA1_PATTERN = /^[0-9a-f]{40}$/u; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; const EVIDENCE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const BENCHMARK_INPUT_KEYS = new Set([ + 'contractVersion', + 'benchmarkId', + 'unit', + 'sourceCommitSha', + 'artifactSha256', + 'documentProfile', + 'runtimeId', + 'referenceHardwareId', + 'samples', +]); const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true }); function resolveArguments(argv) { @@ -96,6 +107,9 @@ function validateInput(value) { if (value === null || typeof value !== 'object' || Array.isArray(value)) { throw new Error('Benchmark sample input must be an object.'); } + if (Object.keys(value).some((key) => !BENCHMARK_INPUT_KEYS.has(key))) { + throw new Error('Benchmark sample input contains unsupported fields.'); + } if (value.contractVersion !== 1) { throw new Error('Benchmark sample contractVersion must be 1.'); } From ba35b28f21b63ca8fbd8d2d5b8f2d0862464fa19 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 06:36:36 -0700 Subject: [PATCH 030/209] test(perf): require explicit benchmark regression comparator --- ...rmanceRegressionComparatorContract.test.ts | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 src/performanceRegressionComparatorContract.test.ts diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts new file mode 100644 index 00000000..0c7e72f5 --- /dev/null +++ b/src/performanceRegressionComparatorContract.test.ts @@ -0,0 +1,155 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/compare-summaries.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +type SummaryOverrides = Partial<{ + benchmarkId: string; + unit: string; + sourceCommitSha: string; + artifactSha256: string; + documentProfile: string; + runtimeId: string; + referenceHardwareId: string; + sampleCount: number; + percentileMethod: string; + minimum: number; + p50: number; + p75: number; + p95: number; + maximum: number; +}>; + +function summary(overrides: SummaryOverrides = {}) { + return { + contractVersion: 1, + benchmarkId: 'editor-input-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + minimum: 70, + p50: 80, + p75: 90, + p95: 100, + maximum: 110, + ...overrides, + }; +} + +function runComparison( + root: string, + baseline: ReturnType, + current: ReturnType, + tolerancePercent: string, +) { + const baselinePath = join(root, 'baseline.json'); + const currentPath = join(root, 'current.json'); + writeFileSync(baselinePath, `${JSON.stringify(baseline)}\n`, 'utf8'); + writeFileSync(currentPath, `${JSON.stringify(current)}\n`, 'utf8'); + return spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + tolerancePercent, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); +} + +describe('benchmark regression comparator contract', () => { + it('passes only when a current exact-context metric stays within an explicit tolerance', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-pass-')); + try { + const result = runComparison(root, summary(), summary({ p95: 104 }), '5'); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toEqual({ + contractVersion: 1, + benchmarkId: 'editor-input-large', + metric: 'p95', + baselineValue: 100, + currentValue: 104, + maxRegressionPercent: 5, + regressionPercent: 4, + passed: true, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails a material unapproved regression without hiding the measured receipt', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-fail-')); + try { + const result = runComparison(root, summary(), summary({ p95: 106 }), '5'); + + expect(result.status).toBe(1); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toEqual({ + contractVersion: 1, + benchmarkId: 'editor-input-large', + metric: 'p95', + baselineValue: 100, + currentValue: 106, + maxRegressionPercent: 5, + regressionPercent: 6, + passed: false, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects incomparable runtime or hardware evidence instead of laundering it through a tolerance', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-context-')); + try { + const result = runComparison( + root, + summary(), + summary({ referenceHardwareId: 'different-runner' }), + '5', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summaries are not comparable: referenceHardwareId differs.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('requires an explicit finite non-negative regression tolerance', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-tolerance-')); + try { + const result = runComparison(root, summary(), summary(), '-1'); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark max regression percent must be a finite non-negative number.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 9504eadc784a68c32aad9e69b9806a97a25b5543 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 06:37:22 -0700 Subject: [PATCH 031/209] feat(perf): add explicit benchmark regression comparator --- benchmarks/compare-summaries.mjs | 286 +++++++++++++++++++++++++++++++ 1 file changed, 286 insertions(+) create mode 100644 benchmarks/compare-summaries.mjs diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs new file mode 100644 index 00000000..7c48fd77 --- /dev/null +++ b/benchmarks/compare-summaries.mjs @@ -0,0 +1,286 @@ +import { closeSync, fstatSync, openSync, readSync } from 'node:fs'; +import { resolve } from 'node:path'; + +const MAX_INPUT_BYTES = 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000_000; +const BENCHMARK_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; +const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const EVIDENCE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const METRICS = new Set(['p50', 'p75', 'p95', 'maximum']); +const SUMMARY_KEYS = new Set([ + 'contractVersion', + 'benchmarkId', + 'unit', + 'sourceCommitSha', + 'artifactSha256', + 'documentProfile', + 'runtimeId', + 'referenceHardwareId', + 'sampleCount', + 'percentileMethod', + 'minimum', + 'p50', + 'p75', + 'p95', + 'maximum', +]); +const COMPARABLE_FIELDS = [ + 'benchmarkId', + 'unit', + 'documentProfile', + 'runtimeId', + 'referenceHardwareId', + 'sampleCount', + 'percentileMethod', +]; +const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true }); + +function resolveArguments(argv) { + if ( + argv.length !== 8 || + argv[0] !== '--baseline' || + argv[1].length === 0 || + argv[2] !== '--current' || + argv[3].length === 0 || + argv[4] !== '--metric' || + !METRICS.has(argv[5]) || + argv[6] !== '--max-regression-percent' || + argv[7].trim().length === 0 + ) { + throw new Error( + 'Usage: node benchmarks/compare-summaries.mjs --baseline --current --metric --max-regression-percent ', + ); + } + + const maxRegressionPercent = Number(argv[7]); + if (!Number.isFinite(maxRegressionPercent) || maxRegressionPercent < 0) { + throw new Error( + 'Benchmark max regression percent must be a finite non-negative number.', + ); + } + + return Object.freeze({ + baselinePath: resolve(argv[1]), + currentPath: resolve(argv[3]), + metric: argv[5], + maxRegressionPercent, + }); +} + +function readBoundedJson(path) { + const descriptor = openSync(path, 'r'); + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error('Benchmark summary input must be a regular file.'); + } + if (metadata.size > MAX_INPUT_BYTES) { + throw new Error('Benchmark summary input exceeds the supported size.'); + } + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= MAX_INPUT_BYTES) { + const remainingBudget = MAX_INPUT_BYTES + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > MAX_INPUT_BYTES) { + throw new Error('Benchmark summary input exceeds the supported size.'); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + + let text; + try { + text = UTF8_DECODER.decode(Buffer.concat(chunks, totalBytes)); + } catch { + throw new Error('Benchmark summary input must be valid UTF-8 JSON.'); + } + + try { + return JSON.parse(text); + } catch { + throw new Error('Benchmark summary input must be valid JSON.'); + } + } finally { + closeSync(descriptor); + } +} + +function validateSummary(value) { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Benchmark summary input must be an object.'); + } + const keys = Object.keys(value); + if ( + keys.length !== SUMMARY_KEYS.size || + keys.some((key) => !SUMMARY_KEYS.has(key)) + ) { + throw new Error('Benchmark summary input has an unsupported shape.'); + } + if (value.contractVersion !== 1) { + throw new Error('Benchmark summary contractVersion must be 1.'); + } + if ( + typeof value.benchmarkId !== 'string' || + !BENCHMARK_ID_PATTERN.test(value.benchmarkId) + ) { + throw new Error('Benchmark summary benchmarkId is invalid.'); + } + if (typeof value.unit !== 'string' || !UNIT_PATTERN.test(value.unit)) { + throw new Error('Benchmark summary unit is invalid.'); + } + if ( + typeof value.sourceCommitSha !== 'string' || + !SHA1_PATTERN.test(value.sourceCommitSha) + ) { + throw new Error('Benchmark summary sourceCommitSha is invalid.'); + } + if ( + typeof value.artifactSha256 !== 'string' || + !SHA256_PATTERN.test(value.artifactSha256) + ) { + throw new Error('Benchmark summary artifactSha256 is invalid.'); + } + if ( + typeof value.documentProfile !== 'string' || + !DOCUMENT_PROFILES.has(value.documentProfile) + ) { + throw new Error('Benchmark summary documentProfile is invalid.'); + } + if ( + typeof value.runtimeId !== 'string' || + !EVIDENCE_ID_PATTERN.test(value.runtimeId) + ) { + throw new Error('Benchmark summary runtimeId is invalid.'); + } + if ( + typeof value.referenceHardwareId !== 'string' || + !EVIDENCE_ID_PATTERN.test(value.referenceHardwareId) + ) { + throw new Error('Benchmark summary referenceHardwareId is invalid.'); + } + if ( + !Number.isSafeInteger(value.sampleCount) || + value.sampleCount <= 0 || + value.sampleCount > MAX_SAMPLES + ) { + throw new Error('Benchmark summary sampleCount is invalid.'); + } + if (value.percentileMethod !== 'nearest-rank') { + throw new Error('Benchmark summary percentileMethod is invalid.'); + } + + const measurements = [ + value.minimum, + value.p50, + value.p75, + value.p95, + value.maximum, + ]; + if ( + measurements.some( + (measurement) => + typeof measurement !== 'number' || + !Number.isFinite(measurement) || + measurement < 0, + ) + ) { + throw new Error( + 'Benchmark summary measurements must be finite non-negative numbers.', + ); + } + for (let index = 1; index < measurements.length; index += 1) { + if (measurements[index] < measurements[index - 1]) { + throw new Error('Benchmark summary percentile ordering is invalid.'); + } + } + + return Object.freeze({ + benchmarkId: value.benchmarkId, + unit: value.unit, + sourceCommitSha: value.sourceCommitSha, + artifactSha256: value.artifactSha256, + documentProfile: value.documentProfile, + runtimeId: value.runtimeId, + referenceHardwareId: value.referenceHardwareId, + sampleCount: value.sampleCount, + percentileMethod: value.percentileMethod, + minimum: value.minimum, + p50: value.p50, + p75: value.p75, + p95: value.p95, + maximum: value.maximum, + }); +} + +function assertComparable(baseline, current) { + for (const field of COMPARABLE_FIELDS) { + if (baseline[field] !== current[field]) { + throw new Error(`Benchmark summaries are not comparable: ${field} differs.`); + } + } +} + +function normalizePercent(value) { + const rounded = Number(value.toFixed(12)); + return Object.is(rounded, -0) ? 0 : rounded; +} + +function compare(baseline, current, metric, maxRegressionPercent) { + assertComparable(baseline, current); + const baselineValue = baseline[metric]; + const currentValue = current[metric]; + if (baselineValue === 0 && currentValue !== 0) { + throw new Error( + 'Benchmark regression percent is undefined for a zero non-matching baseline.', + ); + } + const regressionPercent = + baselineValue === 0 + ? 0 + : normalizePercent(((currentValue - baselineValue) / baselineValue) * 100); + return Object.freeze({ + contractVersion: 1, + benchmarkId: baseline.benchmarkId, + metric, + baselineValue, + currentValue, + maxRegressionPercent, + regressionPercent, + passed: regressionPercent <= maxRegressionPercent, + }); +} + +function main() { + const { baselinePath, currentPath, metric, maxRegressionPercent } = + resolveArguments(process.argv.slice(2)); + const baseline = validateSummary(readBoundedJson(baselinePath)); + const current = validateSummary(readBoundedJson(currentPath)); + const result = compare(baseline, current, metric, maxRegressionPercent); + process.stdout.write(`${JSON.stringify(result)}\n`); + if (!result.passed) process.exitCode = 1; +} + +try { + main(); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Benchmark comparison failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} From f5439a1f0708182526449e98d6fcbab7feaedb7f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 09:40:00 -0700 Subject: [PATCH 032/209] test(perf): prove comparator rejects blocking FIFO inputs --- ...rmanceRegressionComparatorContract.test.ts | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts index 0c7e72f5..7392cd03 100644 --- a/src/performanceRegressionComparatorContract.test.ts +++ b/src/performanceRegressionComparatorContract.test.ts @@ -152,4 +152,42 @@ describe('benchmark regression comparator contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('fails closed on a named-pipe summary instead of blocking before regular-file validation', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-fifo-')); + const baselinePath = join(root, 'baseline.pipe'); + const currentPath = join(root, 'current.json'); + try { + const mkfifo = spawnSync('mkfifo', [baselinePath], { encoding: 'utf8' }); + expect(mkfifo.status).toBe(0); + writeFileSync(currentPath, `${JSON.stringify(summary())}\n`, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8', timeout: 1000 }, + ); + + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary input must be a regular file.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From 13c09c58dcb10849b6bdac756bfd6a53da8ae2d4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 09:40:19 -0700 Subject: [PATCH 033/209] test(perf): prove summarizer rejects blocking FIFO inputs --- ...rformanceMeasurementOutputContract.test.ts | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/performanceMeasurementOutputContract.test.ts b/src/performanceMeasurementOutputContract.test.ts index aca7928e..6ae5fc90 100644 --- a/src/performanceMeasurementOutputContract.test.ts +++ b/src/performanceMeasurementOutputContract.test.ts @@ -58,4 +58,37 @@ describe('benchmark summary output contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('fails closed on a named-pipe sample input instead of blocking before regular-file validation', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-input-fifo-')); + const input = join(root, 'samples.pipe'); + const output = join(root, 'output'); + try { + const mkfifo = spawnSync('mkfifo', [input], { encoding: 'utf8' }); + expect(mkfifo.status).toBe(0); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + timeout: 1000, + }, + ); + + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input must be a regular file.', + ); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From dbf53455d9cc2d76d41c23d64223da8942a19f01 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 09:41:04 -0700 Subject: [PATCH 034/209] fix(perf): validate comparator inputs without blocking --- benchmarks/compare-summaries.mjs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index 7c48fd77..15f60255 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -1,9 +1,17 @@ -import { closeSync, fstatSync, openSync, readSync } from 'node:fs'; +import { + closeSync, + constants, + fstatSync, + openSync, + readSync, +} from 'node:fs'; import { resolve } from 'node:path'; const MAX_INPUT_BYTES = 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000_000; +const READ_ONLY_NONBLOCKING = + constants.O_RDONLY | (constants.O_NONBLOCK ?? 0); const BENCHMARK_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; const SHA1_PATTERN = /^[0-9a-f]{40}$/u; @@ -72,7 +80,7 @@ function resolveArguments(argv) { } function readBoundedJson(path) { - const descriptor = openSync(path, 'r'); + const descriptor = openSync(path, READ_ONLY_NONBLOCKING); try { const metadata = fstatSync(descriptor); if (!metadata.isFile()) { From e98faeeaad97aa64098efbe75178acfbd2066634 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 09:41:36 -0700 Subject: [PATCH 035/209] fix(perf): validate sample inputs without blocking --- benchmarks/summarize-samples.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index b457f71c..ee7de3e3 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -1,5 +1,6 @@ import { closeSync, + constants, existsSync, fstatSync, lstatSync, @@ -14,6 +15,8 @@ import { resolve } from 'node:path'; const MAX_INPUT_BYTES = 16 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000_000; +const READ_ONLY_NONBLOCKING = + constants.O_RDONLY | (constants.O_NONBLOCK ?? 0); const BENCHMARK_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; const SHA1_PATTERN = /^[0-9a-f]{40}$/u; @@ -52,7 +55,7 @@ function resolveArguments(argv) { } function readBoundedJson(path) { - const descriptor = openSync(path, 'r'); + const descriptor = openSync(path, READ_ONLY_NONBLOCKING); try { const metadata = fstatSync(descriptor); if (!metadata.isFile()) { From d82015018dea0ca03fc1777a32e41af1161faf96 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 09:57:39 -0700 Subject: [PATCH 036/209] test(perf): reject dangling benchmark output symlinks --- ...rformanceMeasurementOutputContract.test.ts | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/performanceMeasurementOutputContract.test.ts b/src/performanceMeasurementOutputContract.test.ts index 6ae5fc90..ade9f61d 100644 --- a/src/performanceMeasurementOutputContract.test.ts +++ b/src/performanceMeasurementOutputContract.test.ts @@ -4,6 +4,7 @@ import { mkdirSync, mkdtempSync, rmSync, + symlinkSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -59,6 +60,53 @@ describe('benchmark summary output contract', () => { } }); + it('rejects a dangling summary symlink before it can create the symlink target', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-output-symlink-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const summaryJson = join(output, 'summary.json'); + const escapedTarget = join(root, 'escaped-summary.json'); + try { + writeFileSync( + input, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [1, 2, 3], + })}\n`, + 'utf8', + ); + mkdirSync(output, { recursive: true }); + symlinkSync(escapedTarget, summaryJson); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output paths must be regular files.', + ); + expect(existsSync(escapedTarget)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('fails closed on a named-pipe sample input instead of blocking before regular-file validation', () => { if (process.platform === 'win32') return; From 42c951670b3482a8ec36370deb660a9a83a544ea Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 09:59:19 -0700 Subject: [PATCH 037/209] fix(perf): reject dangling benchmark output symlinks --- benchmarks/summarize-samples.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index ee7de3e3..d857865f 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -240,7 +240,8 @@ function refersToSameFile(leftPath, rightPath) { } function assertRegularOutputDestination(path) { - if (existsSync(path) && !lstatSync(path).isFile()) { + const metadata = lstatSync(path, { throwIfNoEntry: false }); + if (metadata !== undefined && !metadata.isFile()) { throw new Error('Benchmark summary output paths must be regular files.'); } } From 2b2f537fda7729d374b4f33ff74caf68979367d7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 12:08:55 -0700 Subject: [PATCH 038/209] test(perf): fail closed on regression percentage overflow --- ...formanceRegressionOverflowContract.test.ts | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 src/performanceRegressionOverflowContract.test.ts diff --git a/src/performanceRegressionOverflowContract.test.ts b/src/performanceRegressionOverflowContract.test.ts new file mode 100644 index 00000000..2676c458 --- /dev/null +++ b/src/performanceRegressionOverflowContract.test.ts @@ -0,0 +1,71 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/compare-summaries.mjs'); + +function summary(measurement: number, digestCharacter: string) { + return { + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: digestCharacter.repeat(40), + artifactSha256: digestCharacter.repeat(64), + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 1, + percentileMethod: 'nearest-rank', + minimum: measurement, + p50: measurement, + p75: measurement, + p95: measurement, + maximum: measurement, + }; +} + +describe('benchmark regression comparator overflow contract', () => { + it('fails closed instead of serializing an overflowing regression percentage as JSON null', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-overflow-')); + const baseline = join(root, 'baseline.json'); + const current = join(root, 'current.json'); + try { + writeFileSync( + baseline, + `${JSON.stringify(summary(1e-308, 'a'))}\n`, + 'utf8', + ); + writeFileSync( + current, + `${JSON.stringify(summary(1e308, 'b'))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baseline, + '--current', + current, + '--metric', + 'p95', + '--max-regression-percent', + '10', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark regression percent is not finite for the supplied measurements.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 712cd9db034d0c6cf2bc5e1e631c62d14d256a7a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 12:09:31 -0700 Subject: [PATCH 039/209] fix(perf): fail closed on regression percentage overflow --- benchmarks/compare-summaries.mjs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index 15f60255..a2c3ea62 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -262,6 +262,11 @@ function compare(baseline, current, metric, maxRegressionPercent) { baselineValue === 0 ? 0 : normalizePercent(((currentValue - baselineValue) / baselineValue) * 100); + if (!Number.isFinite(regressionPercent)) { + throw new Error( + 'Benchmark regression percent is not finite for the supplied measurements.', + ); + } return Object.freeze({ contractVersion: 1, benchmarkId: baseline.benchmarkId, From 1604f20475b80df255320e71fe0d19e551f67a07 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 14:06:34 -0700 Subject: [PATCH 040/209] test(perf): require provenance-bound regression receipts --- ...rmanceRegressionProvenanceContract.test.ts | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 src/performanceRegressionProvenanceContract.test.ts diff --git a/src/performanceRegressionProvenanceContract.test.ts b/src/performanceRegressionProvenanceContract.test.ts new file mode 100644 index 00000000..48d55840 --- /dev/null +++ b/src/performanceRegressionProvenanceContract.test.ts @@ -0,0 +1,92 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/compare-summaries.mjs'); + +function summary(sourceCommitSha: string, artifactSha256: string) { + return { + contractVersion: 1, + benchmarkId: 'editor-input-large', + unit: 'ms', + sourceCommitSha, + artifactSha256, + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + minimum: 70, + p50: 80, + p75: 90, + p95: 100, + maximum: 110, + }; +} + +describe('benchmark regression provenance contract', () => { + it('binds each comparison receipt to both exact measured artifacts and the shared context', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-provenance-')); + try { + const baselinePath = join(root, 'baseline.json'); + const currentPath = join(root, 'current.json'); + const baselineSourceCommitSha = 'a'.repeat(40); + const currentSourceCommitSha = 'c'.repeat(40); + const baselineArtifactSha256 = 'b'.repeat(64); + const currentArtifactSha256 = 'd'.repeat(64); + writeFileSync( + baselinePath, + `${JSON.stringify(summary(baselineSourceCommitSha, baselineArtifactSha256))}\n`, + 'utf8', + ); + writeFileSync( + currentPath, + `${JSON.stringify(summary(currentSourceCommitSha, currentArtifactSha256))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toEqual({ + contractVersion: 1, + benchmarkId: 'editor-input-large', + unit: 'ms', + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + metric: 'p95', + baselineSourceCommitSha, + baselineArtifactSha256, + currentSourceCommitSha, + currentArtifactSha256, + baselineValue: 100, + currentValue: 100, + maxRegressionPercent: 5, + regressionPercent: 0, + passed: true, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From ceac649067a35dfdf3567256828c23de7d950aa5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 14:07:35 -0700 Subject: [PATCH 041/209] fix(perf): bind regression receipts to exact evidence --- benchmarks/compare-summaries.mjs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index a2c3ea62..b41471c1 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -270,7 +270,17 @@ function compare(baseline, current, metric, maxRegressionPercent) { return Object.freeze({ contractVersion: 1, benchmarkId: baseline.benchmarkId, + unit: baseline.unit, + documentProfile: baseline.documentProfile, + runtimeId: baseline.runtimeId, + referenceHardwareId: baseline.referenceHardwareId, + sampleCount: baseline.sampleCount, + percentileMethod: baseline.percentileMethod, metric, + baselineSourceCommitSha: baseline.sourceCommitSha, + baselineArtifactSha256: baseline.artifactSha256, + currentSourceCommitSha: current.sourceCommitSha, + currentArtifactSha256: current.artifactSha256, baselineValue, currentValue, maxRegressionPercent, From 4dc7984ab131188d59fbba7844789168e63e0393 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 14:09:42 -0700 Subject: [PATCH 042/209] test(perf): align comparator receipt contract --- ...rmanceRegressionComparatorContract.test.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts index 7392cd03..57fcab69 100644 --- a/src/performanceRegressionComparatorContract.test.ts +++ b/src/performanceRegressionComparatorContract.test.ts @@ -84,7 +84,17 @@ describe('benchmark regression comparator contract', () => { expect(JSON.parse(result.stdout)).toEqual({ contractVersion: 1, benchmarkId: 'editor-input-large', + unit: 'ms', + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', metric: 'p95', + baselineSourceCommitSha: SOURCE_COMMIT_SHA, + baselineArtifactSha256: ARTIFACT_SHA256, + currentSourceCommitSha: SOURCE_COMMIT_SHA, + currentArtifactSha256: ARTIFACT_SHA256, baselineValue: 100, currentValue: 104, maxRegressionPercent: 5, @@ -106,7 +116,17 @@ describe('benchmark regression comparator contract', () => { expect(JSON.parse(result.stdout)).toEqual({ contractVersion: 1, benchmarkId: 'editor-input-large', + unit: 'ms', + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', metric: 'p95', + baselineSourceCommitSha: SOURCE_COMMIT_SHA, + baselineArtifactSha256: ARTIFACT_SHA256, + currentSourceCommitSha: SOURCE_COMMIT_SHA, + currentArtifactSha256: ARTIFACT_SHA256, baselineValue: 100, currentValue: 106, maxRegressionPercent: 5, From 36af472d144dd961a126375ba427350ffc30f167 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 15:34:24 -0700 Subject: [PATCH 043/209] test(perf): reject aliased summary destinations --- ...rformanceMeasurementOutputContract.test.ts | 85 ++++++++++++------- 1 file changed, 55 insertions(+), 30 deletions(-) diff --git a/src/performanceMeasurementOutputContract.test.ts b/src/performanceMeasurementOutputContract.test.ts index ade9f61d..45376b1f 100644 --- a/src/performanceMeasurementOutputContract.test.ts +++ b/src/performanceMeasurementOutputContract.test.ts @@ -1,8 +1,10 @@ import { spawnSync } from 'node:child_process'; import { existsSync, + linkSync, mkdirSync, mkdtempSync, + readFileSync, rmSync, symlinkSync, writeFileSync, @@ -15,6 +17,24 @@ const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); const SOURCE_COMMIT_SHA = 'a'.repeat(40); const ARTIFACT_SHA256 = 'b'.repeat(64); +function writeValidInput(path: string): void { + writeFileSync( + path, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [1, 2, 3], + })}\n`, + 'utf8', + ); +} + describe('benchmark summary output contract', () => { it('rejects an invalid second destination before publishing the first summary', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-output-')); @@ -23,21 +43,7 @@ describe('benchmark summary output contract', () => { const summaryJson = join(output, 'summary.json'); const summaryText = join(output, 'summary.txt'); try { - writeFileSync( - input, - `${JSON.stringify({ - contractVersion: 1, - benchmarkId: 'markdown-serialization-large', - unit: 'ms', - sourceCommitSha: SOURCE_COMMIT_SHA, - artifactSha256: ARTIFACT_SHA256, - documentProfile: 'large', - runtimeId: 'node-22.18.0', - referenceHardwareId: 'github-actions-ubuntu-24.04-x64', - samples: [1, 2, 3], - })}\n`, - 'utf8', - ); + writeValidInput(input); mkdirSync(summaryText, { recursive: true }); const result = spawnSync( @@ -69,21 +75,7 @@ describe('benchmark summary output contract', () => { const summaryJson = join(output, 'summary.json'); const escapedTarget = join(root, 'escaped-summary.json'); try { - writeFileSync( - input, - `${JSON.stringify({ - contractVersion: 1, - benchmarkId: 'markdown-serialization-large', - unit: 'ms', - sourceCommitSha: SOURCE_COMMIT_SHA, - artifactSha256: ARTIFACT_SHA256, - documentProfile: 'large', - runtimeId: 'node-22.18.0', - referenceHardwareId: 'github-actions-ubuntu-24.04-x64', - samples: [1, 2, 3], - })}\n`, - 'utf8', - ); + writeValidInput(input); mkdirSync(output, { recursive: true }); symlinkSync(escapedTarget, summaryJson); @@ -107,6 +99,39 @@ describe('benchmark summary output contract', () => { } }); + it('rejects two output paths that alias the same regular file before publication', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-output-alias-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const summaryJson = join(output, 'summary.json'); + const summaryText = join(output, 'summary.txt'); + try { + writeValidInput(input); + mkdirSync(output, { recursive: true }); + writeFileSync(summaryJson, 'sentinel', 'utf8'); + linkSync(summaryJson, summaryText); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary outputs must be distinct files.', + ); + expect(readFileSync(summaryJson, 'utf8')).toBe('sentinel'); + expect(readFileSync(summaryText, 'utf8')).toBe('sentinel'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('fails closed on a named-pipe sample input instead of blocking before regular-file validation', () => { if (process.platform === 'win32') return; From a86955632a201f7592ae00c81f1f3e3b8148b8ff Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 15:35:10 -0700 Subject: [PATCH 044/209] fix(perf): reject aliased summary outputs --- benchmarks/summarize-samples.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index d857865f..12f8cc7b 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -261,6 +261,9 @@ function main() { } assertRegularOutputDestination(summaryJsonPath); assertRegularOutputDestination(summaryTextPath); + if (refersToSameFile(summaryJsonPath, summaryTextPath)) { + throw new Error('Benchmark summary outputs must be distinct files.'); + } const input = validateInput(readBoundedJson(inputPath)); const summary = summarize(input); writeFileSync( From 1902b63f92ecaa2ef03250c2a758edde2099dc2e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 16:57:43 -0700 Subject: [PATCH 045/209] test(perf): reject vacuous same-artifact comparisons --- ...rmanceRegressionProvenanceContract.test.ts | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/performanceRegressionProvenanceContract.test.ts b/src/performanceRegressionProvenanceContract.test.ts index 48d55840..55e90cda 100644 --- a/src/performanceRegressionProvenanceContract.test.ts +++ b/src/performanceRegressionProvenanceContract.test.ts @@ -89,4 +89,47 @@ describe('benchmark regression provenance contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('rejects a vacuous comparison when both summaries identify the same exact artifact', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-provenance-')); + try { + const baselinePath = join(root, 'baseline.json'); + const currentPath = join(root, 'current.json'); + const artifactSha256 = 'b'.repeat(64); + writeFileSync( + baselinePath, + `${JSON.stringify(summary('a'.repeat(40), artifactSha256))}\n`, + 'utf8', + ); + writeFileSync( + currentPath, + `${JSON.stringify(summary('c'.repeat(40), artifactSha256))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark summaries must identify distinct measured artifacts.\n', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From a7467876ae37ad70965536b8aa15cc23cbd52a8d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 17:00:12 -0700 Subject: [PATCH 046/209] fix(perf): reject same-artifact regression evidence --- benchmarks/compare-summaries.mjs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index b41471c1..af21bb45 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -242,6 +242,11 @@ function assertComparable(baseline, current) { throw new Error(`Benchmark summaries are not comparable: ${field} differs.`); } } + if (baseline.artifactSha256 === current.artifactSha256) { + throw new Error( + 'Benchmark summaries must identify distinct measured artifacts.', + ); + } } function normalizePercent(value) { From 8e321140ef4beeeaf247f46958b38f02eebfff45 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 17:02:19 -0700 Subject: [PATCH 047/209] test(perf): use distinct artifacts in comparator receipts --- ...rmanceRegressionComparatorContract.test.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts index 57fcab69..4b0a556d 100644 --- a/src/performanceRegressionComparatorContract.test.ts +++ b/src/performanceRegressionComparatorContract.test.ts @@ -7,6 +7,7 @@ import { describe, expect, it } from 'vitest'; const script = resolve(process.cwd(), 'benchmarks/compare-summaries.mjs'); const SOURCE_COMMIT_SHA = 'a'.repeat(40); const ARTIFACT_SHA256 = 'b'.repeat(64); +const CURRENT_ARTIFACT_SHA256 = 'c'.repeat(64); type SummaryOverrides = Partial<{ benchmarkId: string; @@ -77,7 +78,12 @@ describe('benchmark regression comparator contract', () => { it('passes only when a current exact-context metric stays within an explicit tolerance', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-pass-')); try { - const result = runComparison(root, summary(), summary({ p95: 104 }), '5'); + const result = runComparison( + root, + summary(), + summary({ artifactSha256: CURRENT_ARTIFACT_SHA256, p95: 104 }), + '5', + ); expect(result.status).toBe(0); expect(result.stderr).toBe(''); @@ -94,7 +100,7 @@ describe('benchmark regression comparator contract', () => { baselineSourceCommitSha: SOURCE_COMMIT_SHA, baselineArtifactSha256: ARTIFACT_SHA256, currentSourceCommitSha: SOURCE_COMMIT_SHA, - currentArtifactSha256: ARTIFACT_SHA256, + currentArtifactSha256: CURRENT_ARTIFACT_SHA256, baselineValue: 100, currentValue: 104, maxRegressionPercent: 5, @@ -109,7 +115,12 @@ describe('benchmark regression comparator contract', () => { it('fails a material unapproved regression without hiding the measured receipt', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-fail-')); try { - const result = runComparison(root, summary(), summary({ p95: 106 }), '5'); + const result = runComparison( + root, + summary(), + summary({ artifactSha256: CURRENT_ARTIFACT_SHA256, p95: 106 }), + '5', + ); expect(result.status).toBe(1); expect(result.stderr).toBe(''); @@ -126,7 +137,7 @@ describe('benchmark regression comparator contract', () => { baselineSourceCommitSha: SOURCE_COMMIT_SHA, baselineArtifactSha256: ARTIFACT_SHA256, currentSourceCommitSha: SOURCE_COMMIT_SHA, - currentArtifactSha256: ARTIFACT_SHA256, + currentArtifactSha256: CURRENT_ARTIFACT_SHA256, baselineValue: 100, currentValue: 106, maxRegressionPercent: 5, From f6b87db8c3ee82cf37a3f3e0bfdfa4472a7bac9f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 00:30:44 -0700 Subject: [PATCH 048/209] test(perf): reject private-looking benchmark evidence identifiers --- ...formanceMeasurementPrivacyContract.test.ts | 94 +++++++++++++------ 1 file changed, 65 insertions(+), 29 deletions(-) diff --git a/src/performanceMeasurementPrivacyContract.test.ts b/src/performanceMeasurementPrivacyContract.test.ts index 83dfccfd..1d7a3c5b 100644 --- a/src/performanceMeasurementPrivacyContract.test.ts +++ b/src/performanceMeasurementPrivacyContract.test.ts @@ -13,38 +13,43 @@ const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); const SOURCE_COMMIT_SHA = 'a'.repeat(40); const ARTIFACT_SHA256 = 'b'.repeat(64); +const validInput = { + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [10, 20, 30], +} as const; + +function runSummary(inputValue: object) { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-privacy-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + writeFileSync(input, `${JSON.stringify(inputValue)}\n`, 'utf8'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + return { root, output, result }; +} + describe('benchmark evidence privacy contract', () => { it('rejects unsupported metadata instead of accepting arbitrary evidence payloads', () => { - const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-privacy-')); - const input = join(root, 'samples.json'); - const output = join(root, 'output'); + const { root, output, result } = runSummary({ + ...validInput, + prompt: 'must-not-enter-benchmark-evidence', + }); try { - writeFileSync( - input, - `${JSON.stringify({ - contractVersion: 1, - benchmarkId: 'markdown-serialization-large', - unit: 'ms', - sourceCommitSha: SOURCE_COMMIT_SHA, - artifactSha256: ARTIFACT_SHA256, - documentProfile: 'large', - runtimeId: 'node-22.18.0', - referenceHardwareId: 'github-actions-ubuntu-24.04-x64', - samples: [10, 20, 30], - prompt: 'must-not-enter-benchmark-evidence', - })}\n`, - 'utf8', - ); - - const result = spawnSync( - process.execPath, - [script, '--input', input, '--output', output], - { - cwd: process.cwd(), - encoding: 'utf8', - }, - ); - expect(result.status).toBe(1); expect(result.stdout).toBe(''); expect(result.stderr.trim()).toBe( @@ -56,4 +61,35 @@ describe('benchmark evidence privacy contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it.each([ + [ + 'benchmarkId', + 'tenant-acme-large', + 'Benchmark benchmarkId is invalid.', + ], + ['runtimeId', 'tenant-acme', 'Benchmark runtimeId is invalid.'], + [ + 'referenceHardwareId', + 'tenant-acme', + 'Benchmark referenceHardwareId is invalid.', + ], + ])( + 'rejects caller-controlled %s values that could launder private identifiers into evidence', + (field, value, expectedError) => { + const { root, output, result } = runSummary({ + ...validInput, + [field]: value, + }); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe(expectedError); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); }); From b6fef32bcbc57d27ec3478d7b995bf13c3e7b073 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 00:36:01 -0700 Subject: [PATCH 049/209] test(perf): keep hardware mismatch fixture privacy-safe --- src/performanceRegressionComparatorContract.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts index 4b0a556d..fd9806ee 100644 --- a/src/performanceRegressionComparatorContract.test.ts +++ b/src/performanceRegressionComparatorContract.test.ts @@ -155,7 +155,7 @@ describe('benchmark regression comparator contract', () => { const result = runComparison( root, summary(), - summary({ referenceHardwareId: 'different-runner' }), + summary({ referenceHardwareId: 'github-actions-ubuntu-22.04-x64' }), '5', ); From 4a7786cc780406bdeac5be1206f09b87436d809a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 00:36:42 -0700 Subject: [PATCH 050/209] fix(perf): constrain benchmark evidence identifiers --- benchmarks/summarize-samples.mjs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 12f8cc7b..6daa81f5 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -17,11 +17,15 @@ const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000_000; const READ_ONLY_NONBLOCKING = constants.O_RDONLY | (constants.O_NONBLOCK ?? 0); -const BENCHMARK_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; +const BENCHMARK_ID_PATTERN = + /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; const SHA1_PATTERN = /^[0-9a-f]{40}$/u; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; -const EVIDENCE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); const BENCHMARK_INPUT_KEYS = new Set([ 'contractVersion', @@ -149,13 +153,13 @@ function validateInput(value) { } if ( typeof value.runtimeId !== 'string' || - !EVIDENCE_ID_PATTERN.test(value.runtimeId) + !RUNTIME_ID_PATTERN.test(value.runtimeId) ) { throw new Error('Benchmark runtimeId is invalid.'); } if ( typeof value.referenceHardwareId !== 'string' || - !EVIDENCE_ID_PATTERN.test(value.referenceHardwareId) + !REFERENCE_HARDWARE_ID_PATTERN.test(value.referenceHardwareId) ) { throw new Error('Benchmark referenceHardwareId is invalid.'); } From 37ec73e7cdb34d58ffc86d7664b5ddba66a99ca0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 00:37:40 -0700 Subject: [PATCH 051/209] fix(perf): validate comparison evidence identifiers --- benchmarks/compare-summaries.mjs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index af21bb45..65f4ea37 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -12,11 +12,15 @@ const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000_000; const READ_ONLY_NONBLOCKING = constants.O_RDONLY | (constants.O_NONBLOCK ?? 0); -const BENCHMARK_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; +const BENCHMARK_ID_PATTERN = + /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; const SHA1_PATTERN = /^[0-9a-f]{40}$/u; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; -const EVIDENCE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); const METRICS = new Set(['p50', 'p75', 'p95', 'maximum']); const SUMMARY_KEYS = new Set([ @@ -172,13 +176,13 @@ function validateSummary(value) { } if ( typeof value.runtimeId !== 'string' || - !EVIDENCE_ID_PATTERN.test(value.runtimeId) + !RUNTIME_ID_PATTERN.test(value.runtimeId) ) { throw new Error('Benchmark summary runtimeId is invalid.'); } if ( typeof value.referenceHardwareId !== 'string' || - !EVIDENCE_ID_PATTERN.test(value.referenceHardwareId) + !REFERENCE_HARDWARE_ID_PATTERN.test(value.referenceHardwareId) ) { throw new Error('Benchmark summary referenceHardwareId is invalid.'); } From 142499dee4585dfdd5a307d287d38497e6604fbb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 00:42:43 -0700 Subject: [PATCH 052/209] test(perf): reject private-looking benchmark units --- src/performanceMeasurementPrivacyContract.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/performanceMeasurementPrivacyContract.test.ts b/src/performanceMeasurementPrivacyContract.test.ts index 1d7a3c5b..9f994c5e 100644 --- a/src/performanceMeasurementPrivacyContract.test.ts +++ b/src/performanceMeasurementPrivacyContract.test.ts @@ -68,6 +68,7 @@ describe('benchmark evidence privacy contract', () => { 'tenant-acme-large', 'Benchmark benchmarkId is invalid.', ], + ['unit', 'tenant-acme', 'Benchmark unit is invalid.'], ['runtimeId', 'tenant-acme', 'Benchmark runtimeId is invalid.'], [ 'referenceHardwareId', From 7248c18c57ce8da60be82beb4cc3086aa1dda358 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 00:45:16 -0700 Subject: [PATCH 053/209] test(perf): reject private units at comparator boundary --- ...ormanceRegressionComparatorContract.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts index fd9806ee..997a02ac 100644 --- a/src/performanceRegressionComparatorContract.test.ts +++ b/src/performanceRegressionComparatorContract.test.ts @@ -169,6 +169,24 @@ describe('benchmark regression comparator contract', () => { } }); + it('rejects private-looking units at the direct summary-comparison boundary', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-unit-')); + try { + const result = runComparison( + root, + summary({ unit: 'tenant-acme' }), + summary({ artifactSha256: CURRENT_ARTIFACT_SHA256, unit: 'tenant-acme' }), + '5', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe('Benchmark summary unit is invalid.'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('requires an explicit finite non-negative regression tolerance', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-tolerance-')); try { From 92eaeb13c2c3d71dd185751ad941f397fbbd73c5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 00:49:26 -0700 Subject: [PATCH 054/209] fix(perf): constrain benchmark measurement units --- benchmarks/summarize-samples.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 6daa81f5..354c2539 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -19,7 +19,7 @@ const READ_ONLY_NONBLOCKING = constants.O_RDONLY | (constants.O_NONBLOCK ?? 0); const BENCHMARK_ID_PATTERN = /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; -const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; +const UNITS = new Set(['ms', 'bytes']); const SHA1_PATTERN = /^[0-9a-f]{40}$/u; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; const RUNTIME_ID_PATTERN = @@ -126,7 +126,7 @@ function validateInput(value) { ) { throw new Error('Benchmark benchmarkId is invalid.'); } - if (typeof value.unit !== 'string' || !UNIT_PATTERN.test(value.unit)) { + if (typeof value.unit !== 'string' || !UNITS.has(value.unit)) { throw new Error('Benchmark unit is invalid.'); } if ( From 3fe89a3dfde92b592b1ddb4bbeb0e5012255e7ad Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 00:50:06 -0700 Subject: [PATCH 055/209] fix(perf): validate comparator measurement units --- benchmarks/compare-summaries.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index 65f4ea37..975e8c97 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -14,7 +14,7 @@ const READ_ONLY_NONBLOCKING = constants.O_RDONLY | (constants.O_NONBLOCK ?? 0); const BENCHMARK_ID_PATTERN = /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; -const UNIT_PATTERN = /^[A-Za-z][A-Za-z0-9._/%-]{0,31}$/u; +const UNITS = new Set(['ms', 'bytes']); const SHA1_PATTERN = /^[0-9a-f]{40}$/u; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; const RUNTIME_ID_PATTERN = @@ -153,7 +153,7 @@ function validateSummary(value) { ) { throw new Error('Benchmark summary benchmarkId is invalid.'); } - if (typeof value.unit !== 'string' || !UNIT_PATTERN.test(value.unit)) { + if (typeof value.unit !== 'string' || !UNITS.has(value.unit)) { throw new Error('Benchmark summary unit is invalid.'); } if ( From 3e1f34585885a7f7a52a8c110f306dd6207cc5f2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:34:49 -0700 Subject: [PATCH 056/209] test(perf): require markdown measurement harness --- ...ormanceMarkdownMeasurementContract.test.ts | 211 ++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 src/performanceMarkdownMeasurementContract.test.ts diff --git a/src/performanceMarkdownMeasurementContract.test.ts b/src/performanceMarkdownMeasurementContract.test.ts new file mode 100644 index 00000000..fb763477 --- /dev/null +++ b/src/performanceMarkdownMeasurementContract.test.ts @@ -0,0 +1,211 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +interface BenchmarkSamples { + readonly contractVersion: 1; + readonly benchmarkId: string; + readonly unit: 'ms'; + readonly sourceCommitSha: string; + readonly artifactSha256: string; + readonly documentProfile: 'small' | 'medium' | 'large' | 'stress'; + readonly runtimeId: string; + readonly referenceHardwareId: string; + readonly samples: number[]; +} + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-markdown.mjs', +); +const summaryScript = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); +const RUNTIME_ID = 'node-22.18.0'; +const HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function measurementArguments( + input: string, + modulePath: string, + output: string, +): string[] { + return [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '3', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + ARTIFACT_SHA256, + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + HARDWARE_ID, + '--output', + output, + ]; +} + +describe('Markdown runtime measurement contract', () => { + it('writes bounded privacy-safe samples consumable by the canonical summarizer', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-')); + const input = join(root, 'large.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const samplesPath = join(root, 'samples.json'); + const summaryDirectory = join(root, 'summary'); + try { + writeFileSync(input, '# Buyer benchmark fixture\n\nSynthetic content only.\n', 'utf8'); + writeFileSync( + modulePath, + "export function markdownToHtml(source) { return `

${source.length}

`; }\n", + 'utf8', + ); + + execFileSync( + process.execPath, + measurementArguments(input, modulePath, samplesPath), + { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'pipe'] }, + ); + + const samples = JSON.parse( + readFileSync(samplesPath, 'utf8'), + ) as BenchmarkSamples; + expect(samples).toMatchObject({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: RUNTIME_ID, + referenceHardwareId: HARDWARE_ID, + }); + expect(samples.samples).toHaveLength(3); + expect( + samples.samples.every( + (sample) => Number.isFinite(sample) && sample >= 0, + ), + ).toBe(true); + expect(readFileSync(samplesPath, 'utf8')).not.toContain('Buyer benchmark fixture'); + expect(readFileSync(samplesPath, 'utf8')).not.toContain('Synthetic content only'); + + execFileSync( + process.execPath, + [summaryScript, '--input', samplesPath, '--output', summaryDirectory], + { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'pipe'] }, + ); + const summary = JSON.parse( + readFileSync(join(summaryDirectory, 'summary.json'), 'utf8'), + ) as { sampleCount: number; benchmarkId: string; unit: string }; + expect(summary).toMatchObject({ + sampleCount: 3, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before output when the measured module lacks the public serializer', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-export-')); + const input = join(root, 'small.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic\n', 'utf8'); + writeFileSync(modulePath, 'export const other = true;\n', 'utf8'); + + const result = spawnSync( + process.execPath, + measurementArguments(input, modulePath, samplesPath), + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module must export markdownToHtml().', + ); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects symlinked document inputs before reading benchmark content', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-symlink-')); + const realInput = join(root, 'real.md'); + const input = join(root, 'linked.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeFileSync(realInput, '# Synthetic\n', 'utf8'); + symlinkSync(realInput, input); + writeFileSync( + modulePath, + 'export function markdownToHtml(source) { return source; }\n', + 'utf8', + ); + + const result = spawnSync( + process.execPath, + measurementArguments(input, modulePath, samplesPath), + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark input must be a regular non-symlink file.', + ); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('requires a file-backed module URL rather than network or package-name resolution', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-module-')); + const input = join(root, 'small.md'); + const samplesPath = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic\n', 'utf8'); + const result = spawnSync( + process.execPath, + measurementArguments( + input, + 'https://example.invalid/markdown.mjs', + samplesPath, + ), + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module must be a local regular file.', + ); + expect(existsSync(samplesPath)).toBe(false); + expect(() => pathToFileURL(input)).not.toThrow(); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 124e9446ad3799ee4eba479d486010fa562570ef Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:35:59 -0700 Subject: [PATCH 057/209] feat(perf): add bounded markdown runtime measurement --- benchmarks/measure-markdown.mjs | 250 ++++++++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 benchmarks/measure-markdown.mjs diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs new file mode 100644 index 00000000..e1728b1b --- /dev/null +++ b/benchmarks/measure-markdown.mjs @@ -0,0 +1,250 @@ +import { performance } from 'node:perf_hooks'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readSync, + realpathSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { mkdirSync } from 'node:fs'; + +const MAX_INPUT_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000; +const READ_ONLY_NOFOLLOW = + constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; + +function resolveArguments(argv) { + const expectedFlags = [ + '--input', + '--module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', + ]; + if ( + argv.length !== expectedFlags.length * 2 || + expectedFlags.some((flag, index) => argv[index * 2] !== flag) || + expectedFlags.some((_, index) => argv[index * 2 + 1]?.length === 0) + ) { + throw new Error( + 'Usage: node benchmarks/measure-markdown.mjs --input --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', + ); + } + + const values = Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); + const profile = values['--profile']; + if (!DOCUMENT_PROFILES.has(profile)) { + throw new Error('Markdown benchmark profile is invalid.'); + } + const sampleCount = Number(values['--samples']); + if ( + !Number.isSafeInteger(sampleCount) || + sampleCount < 1 || + sampleCount > MAX_SAMPLES + ) { + throw new Error('Markdown benchmark sample count must be an integer from 1 to 1000.'); + } + const sourceCommitSha = values['--source-commit-sha']; + if (!SHA1_PATTERN.test(sourceCommitSha)) { + throw new Error( + 'Markdown benchmark source commit must be a lowercase 40-character SHA.', + ); + } + const artifactSha256 = values['--artifact-sha256']; + if (!SHA256_PATTERN.test(artifactSha256)) { + throw new Error( + 'Markdown benchmark artifact digest must be a lowercase 64-character SHA-256.', + ); + } + const runtimeId = values['--runtime-id']; + if (!RUNTIME_ID_PATTERN.test(runtimeId)) { + throw new Error('Markdown benchmark runtime ID is invalid.'); + } + const referenceHardwareId = values['--reference-hardware-id']; + if (!REFERENCE_HARDWARE_ID_PATTERN.test(referenceHardwareId)) { + throw new Error('Markdown benchmark reference hardware ID is invalid.'); + } + + return Object.freeze({ + inputPath: resolve(values['--input']), + modulePath: values['--module'], + profile, + sampleCount, + sourceCommitSha, + artifactSha256, + runtimeId, + referenceHardwareId, + outputPath: resolve(values['--output']), + }); +} + +function readBoundedMarkdown(path) { + const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + if (pathMetadata === undefined || pathMetadata.isSymbolicLink() || !pathMetadata.isFile()) { + throw new Error('Markdown benchmark input must be a regular non-symlink file.'); + } + + const descriptor = openSync(path, READ_ONLY_NOFOLLOW); + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error('Markdown benchmark input must be a regular non-symlink file.'); + } + if (metadata.size > MAX_INPUT_BYTES) { + throw new Error('Markdown benchmark input exceeds the supported size.'); + } + const chunks = []; + let totalBytes = 0; + while (totalBytes <= MAX_INPUT_BYTES) { + const remainingBudget = MAX_INPUT_BYTES + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > MAX_INPUT_BYTES) { + throw new Error('Markdown benchmark input exceeds the supported size.'); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + + try { + return new TextDecoder('utf-8', { fatal: true }).decode( + Buffer.concat(chunks, totalBytes), + ); + } catch { + throw new Error('Markdown benchmark input must be valid UTF-8.'); + } + } finally { + closeSync(descriptor); + } +} + +function resolveLocalModule(pathOrUrl) { + if ( + pathOrUrl.startsWith('http:') || + pathOrUrl.startsWith('https:') || + pathOrUrl.startsWith('data:') || + pathOrUrl.startsWith('node:') + ) { + throw new Error('Measured Markdown module must be a local regular file.'); + } + let modulePath; + try { + modulePath = pathOrUrl.startsWith('file:') + ? new URL(pathOrUrl) + : pathToFileURL(resolve(pathOrUrl)); + } catch { + throw new Error('Measured Markdown module must be a local regular file.'); + } + if (modulePath.protocol !== 'file:') { + throw new Error('Measured Markdown module must be a local regular file.'); + } + const resolvedPath = resolve(decodeURIComponent(modulePath.pathname)); + const metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); + if (metadata === undefined || metadata.isSymbolicLink() || !metadata.isFile()) { + throw new Error('Measured Markdown module must be a local regular file.'); + } + return realpathSync(resolvedPath); +} + +function refersToSameFile(leftPath, rightPath) { + const rightMetadata = lstatSync(rightPath, { throwIfNoEntry: false }); + if (rightMetadata === undefined) return false; + if (!rightMetadata.isFile()) { + throw new Error('Markdown benchmark output must be a regular file.'); + } + const left = statSync(leftPath); + const right = statSync(rightPath); + return left.dev === right.dev && left.ino === right.ino; +} + +async function main() { + const args = resolveArguments(process.argv.slice(2)); + if (args.inputPath === args.outputPath || refersToSameFile(args.inputPath, args.outputPath)) { + throw new Error('Markdown benchmark output must not overwrite its input.'); + } + const source = readBoundedMarkdown(args.inputPath); + const modulePath = resolveLocalModule(args.modulePath); + const measuredModule = await import(pathToFileURL(modulePath).href); + if (typeof measuredModule.markdownToHtml !== 'function') { + throw new Error('Measured Markdown module must export markdownToHtml().'); + } + + const warmup = measuredModule.markdownToHtml(source); + if (typeof warmup !== 'string') { + throw new Error('Measured markdownToHtml() must return a string.'); + } + + const samples = []; + for (let index = 0; index < args.sampleCount; index += 1) { + const start = performance.now(); + const output = measuredModule.markdownToHtml(source); + const elapsed = performance.now() - start; + if (typeof output !== 'string' || !Number.isFinite(elapsed) || elapsed < 0) { + throw new Error('Markdown measurement produced invalid runtime evidence.'); + } + samples.push(elapsed); + } + + const outputMetadata = lstatSync(args.outputPath, { throwIfNoEntry: false }); + if (outputMetadata !== undefined && !outputMetadata.isFile()) { + throw new Error('Markdown benchmark output must be a regular file.'); + } + mkdirSync(dirname(args.outputPath), { recursive: true }); + writeFileSync( + args.outputPath, + `${JSON.stringify( + { + contractVersion: 1, + benchmarkId: `markdown-serialization-${args.profile}`, + unit: 'ms', + sourceCommitSha: args.sourceCommitSha, + artifactSha256: args.artifactSha256, + documentProfile: args.profile, + runtimeId: args.runtimeId, + referenceHardwareId: args.referenceHardwareId, + samples, + }, + null, + 2, + )}\n`, + 'utf8', + ); +} + +try { + await main(); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Markdown benchmark measurement failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} From a08eb096b907ab9c4c0a2b44b39496fc547bf80d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:37:52 -0700 Subject: [PATCH 058/209] test(perf): bind markdown measurements to measured artifact --- ...kdownMeasurementProvenanceContract.test.ts | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 src/performanceMarkdownMeasurementProvenanceContract.test.ts diff --git a/src/performanceMarkdownMeasurementProvenanceContract.test.ts b/src/performanceMarkdownMeasurementProvenanceContract.test.ts new file mode 100644 index 00000000..dea6bb44 --- /dev/null +++ b/src/performanceMarkdownMeasurementProvenanceContract.test.ts @@ -0,0 +1,62 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-markdown.mjs', +); + +describe('Markdown measurement artifact provenance', () => { + it('rejects caller metadata that does not match the exact measured module bytes', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-provenance-')); + const input = join(root, 'large.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const output = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic benchmark fixture\n', 'utf8'); + writeFileSync( + modulePath, + 'export function markdownToHtml(source) { return `

${source}

`; }\n', + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'f'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark artifact digest does not match the measured module.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From e29e5698e7133837592e7ec65dc2452723350bb5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:39:11 -0700 Subject: [PATCH 059/209] test(perf): use exact measured module digests --- ...rformanceMarkdownMeasurementContract.test.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/performanceMarkdownMeasurementContract.test.ts b/src/performanceMarkdownMeasurementContract.test.ts index fb763477..54d4fd8a 100644 --- a/src/performanceMarkdownMeasurementContract.test.ts +++ b/src/performanceMarkdownMeasurementContract.test.ts @@ -1,4 +1,5 @@ import { execFileSync, spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; import { existsSync, mkdtempSync, @@ -30,14 +31,21 @@ const measurementScript = resolve( ); const summaryScript = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); const SOURCE_COMMIT_SHA = 'a'.repeat(40); -const ARTIFACT_SHA256 = 'b'.repeat(64); +const FALLBACK_ARTIFACT_SHA256 = 'b'.repeat(64); const RUNTIME_ID = 'node-22.18.0'; const HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; +function fileSha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + function measurementArguments( input: string, modulePath: string, output: string, + artifactSha256 = existsSync(modulePath) + ? fileSha256(modulePath) + : FALLBACK_ARTIFACT_SHA256, ): string[] { return [ measurementScript, @@ -52,7 +60,7 @@ function measurementArguments( '--source-commit-sha', SOURCE_COMMIT_SHA, '--artifact-sha256', - ARTIFACT_SHA256, + artifactSha256, '--runtime-id', RUNTIME_ID, '--reference-hardware-id', @@ -76,10 +84,11 @@ describe('Markdown runtime measurement contract', () => { "export function markdownToHtml(source) { return `

${source.length}

`; }\n", 'utf8', ); + const artifactSha256 = fileSha256(modulePath); execFileSync( process.execPath, - measurementArguments(input, modulePath, samplesPath), + measurementArguments(input, modulePath, samplesPath, artifactSha256), { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'pipe'] }, ); @@ -91,7 +100,7 @@ describe('Markdown runtime measurement contract', () => { benchmarkId: 'markdown-serialization-large', unit: 'ms', sourceCommitSha: SOURCE_COMMIT_SHA, - artifactSha256: ARTIFACT_SHA256, + artifactSha256, documentProfile: 'large', runtimeId: RUNTIME_ID, referenceHardwareId: HARDWARE_ID, From 866ca00cc73dd68eecddec18903f5accd0b9d8af Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:39:42 -0700 Subject: [PATCH 060/209] fix(perf): verify measured markdown artifact provenance --- benchmarks/measure-markdown.mjs | 101 +++++++++++++++++++++++--------- 1 file changed, 74 insertions(+), 27 deletions(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index e1728b1b..d8713a49 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -1,9 +1,11 @@ +import { createHash } from 'node:crypto'; import { performance } from 'node:perf_hooks'; import { closeSync, constants, fstatSync, lstatSync, + mkdirSync, openSync, readSync, realpathSync, @@ -12,9 +14,9 @@ import { } from 'node:fs'; import { dirname, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; -import { mkdirSync } from 'node:fs'; const MAX_INPUT_BYTES = 16 * 1024 * 1024; +const MAX_MODULE_BYTES = 16 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000; const READ_ONLY_NOFOLLOW = @@ -98,25 +100,29 @@ function resolveArguments(argv) { }); } -function readBoundedMarkdown(path) { +function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversizedMessage) { const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); - if (pathMetadata === undefined || pathMetadata.isSymbolicLink() || !pathMetadata.isFile()) { - throw new Error('Markdown benchmark input must be a regular non-symlink file.'); + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error(invalidFileMessage); } const descriptor = openSync(path, READ_ONLY_NOFOLLOW); try { const metadata = fstatSync(descriptor); if (!metadata.isFile()) { - throw new Error('Markdown benchmark input must be a regular non-symlink file.'); + throw new Error(invalidFileMessage); } - if (metadata.size > MAX_INPUT_BYTES) { - throw new Error('Markdown benchmark input exceeds the supported size.'); + if (metadata.size > maximumBytes) { + throw new Error(oversizedMessage); } const chunks = []; let totalBytes = 0; - while (totalBytes <= MAX_INPUT_BYTES) { - const remainingBudget = MAX_INPUT_BYTES + 1 - totalBytes; + while (totalBytes <= maximumBytes) { + const remainingBudget = maximumBytes + 1 - totalBytes; const chunk = Buffer.allocUnsafe( Math.min(READ_CHUNK_BYTES, remainingBudget), ); @@ -129,24 +135,31 @@ function readBoundedMarkdown(path) { ); if (bytesRead === 0) break; totalBytes += bytesRead; - if (totalBytes > MAX_INPUT_BYTES) { - throw new Error('Markdown benchmark input exceeds the supported size.'); + if (totalBytes > maximumBytes) { + throw new Error(oversizedMessage); } chunks.push(chunk.subarray(0, bytesRead)); } - - try { - return new TextDecoder('utf-8', { fatal: true }).decode( - Buffer.concat(chunks, totalBytes), - ); - } catch { - throw new Error('Markdown benchmark input must be valid UTF-8.'); - } + return Buffer.concat(chunks, totalBytes); } finally { closeSync(descriptor); } } +function readBoundedMarkdown(path) { + const bytes = readBoundedRegularFile( + path, + MAX_INPUT_BYTES, + 'Markdown benchmark input must be a regular non-symlink file.', + 'Markdown benchmark input exceeds the supported size.', + ); + try { + return new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch { + throw new Error('Markdown benchmark input must be valid UTF-8.'); + } +} + function resolveLocalModule(pathOrUrl) { if ( pathOrUrl.startsWith('http:') || @@ -156,25 +169,47 @@ function resolveLocalModule(pathOrUrl) { ) { throw new Error('Measured Markdown module must be a local regular file.'); } - let modulePath; + let moduleUrl; try { - modulePath = pathOrUrl.startsWith('file:') + moduleUrl = pathOrUrl.startsWith('file:') ? new URL(pathOrUrl) : pathToFileURL(resolve(pathOrUrl)); } catch { throw new Error('Measured Markdown module must be a local regular file.'); } - if (modulePath.protocol !== 'file:') { + if (moduleUrl.protocol !== 'file:') { throw new Error('Measured Markdown module must be a local regular file.'); } - const resolvedPath = resolve(decodeURIComponent(modulePath.pathname)); + const resolvedPath = resolve(decodeURIComponent(moduleUrl.pathname)); const metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); - if (metadata === undefined || metadata.isSymbolicLink() || !metadata.isFile()) { + if ( + metadata === undefined || + metadata.isSymbolicLink() || + !metadata.isFile() + ) { throw new Error('Measured Markdown module must be a local regular file.'); } return realpathSync(resolvedPath); } +function measuredModuleSha256(modulePath) { + const bytes = readBoundedRegularFile( + modulePath, + MAX_MODULE_BYTES, + 'Measured Markdown module must be a local regular file.', + 'Measured Markdown module exceeds the supported size.', + ); + return createHash('sha256').update(bytes).digest('hex'); +} + +function verifyMeasuredModuleDigest(modulePath, expectedSha256) { + if (measuredModuleSha256(modulePath) !== expectedSha256) { + throw new Error( + 'Markdown benchmark artifact digest does not match the measured module.', + ); + } +} + function refersToSameFile(leftPath, rightPath) { const rightMetadata = lstatSync(rightPath, { throwIfNoEntry: false }); if (rightMetadata === undefined) return false; @@ -188,11 +223,16 @@ function refersToSameFile(leftPath, rightPath) { async function main() { const args = resolveArguments(process.argv.slice(2)); - if (args.inputPath === args.outputPath || refersToSameFile(args.inputPath, args.outputPath)) { + if ( + args.inputPath === args.outputPath || + refersToSameFile(args.inputPath, args.outputPath) + ) { throw new Error('Markdown benchmark output must not overwrite its input.'); } const source = readBoundedMarkdown(args.inputPath); const modulePath = resolveLocalModule(args.modulePath); + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + const measuredModule = await import(pathToFileURL(modulePath).href); if (typeof measuredModule.markdownToHtml !== 'function') { throw new Error('Measured Markdown module must export markdownToHtml().'); @@ -208,12 +248,17 @@ async function main() { const start = performance.now(); const output = measuredModule.markdownToHtml(source); const elapsed = performance.now() - start; - if (typeof output !== 'string' || !Number.isFinite(elapsed) || elapsed < 0) { + if ( + typeof output !== 'string' || + !Number.isFinite(elapsed) || + elapsed < 0 + ) { throw new Error('Markdown measurement produced invalid runtime evidence.'); } samples.push(elapsed); } + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); const outputMetadata = lstatSync(args.outputPath, { throwIfNoEntry: false }); if (outputMetadata !== undefined && !outputMetadata.isFile()) { throw new Error('Markdown benchmark output must be a regular file.'); @@ -244,7 +289,9 @@ try { await main(); } catch (error) { const message = - error instanceof Error ? error.message : 'Markdown benchmark measurement failed.'; + error instanceof Error + ? error.message + : 'Markdown benchmark measurement failed.'; process.stderr.write(`${message}\n`); process.exitCode = 1; } From 1ecdba32613fe706e7f5fa27154972b5bafef2fa Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:42:13 -0700 Subject: [PATCH 061/209] test(perf): reject measurement output aliasing module --- ...ormanceMarkdownMeasurementContract.test.ts | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/performanceMarkdownMeasurementContract.test.ts b/src/performanceMarkdownMeasurementContract.test.ts index 54d4fd8a..87edd0e3 100644 --- a/src/performanceMarkdownMeasurementContract.test.ts +++ b/src/performanceMarkdownMeasurementContract.test.ts @@ -190,6 +190,32 @@ describe('Markdown runtime measurement contract', () => { } }); + it('rejects an output path that aliases the measured module', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-output-')); + const input = join(root, 'small.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const moduleSource = + 'export function markdownToHtml(source) { return `

${source}

`; }\n'; + try { + writeFileSync(input, '# Synthetic\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + const result = spawnSync( + process.execPath, + measurementArguments(input, modulePath, modulePath), + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark output must not overwrite the measured module.', + ); + expect(readFileSync(modulePath, 'utf8')).toBe(moduleSource); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('requires a file-backed module URL rather than network or package-name resolution', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-module-')); const input = join(root, 'small.md'); From b5373c9d51d326e7b5e73446ba7445a17704c185 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:42:47 -0700 Subject: [PATCH 062/209] fix(perf): preserve measured module from output writes --- benchmarks/measure-markdown.mjs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index d8713a49..2154089a 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -231,6 +231,14 @@ async function main() { } const source = readBoundedMarkdown(args.inputPath); const modulePath = resolveLocalModule(args.modulePath); + if ( + modulePath === args.outputPath || + refersToSameFile(modulePath, args.outputPath) + ) { + throw new Error( + 'Markdown benchmark output must not overwrite the measured module.', + ); + } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); const measuredModule = await import(pathToFileURL(modulePath).href); From 4fb4f3faed55430bb777535596625054c4bbcecb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:48:30 -0700 Subject: [PATCH 063/209] test(perf): reject non-local file URL authorities --- ...rformanceMarkdownModuleUrlContract.test.ts | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 src/performanceMarkdownModuleUrlContract.test.ts diff --git a/src/performanceMarkdownModuleUrlContract.test.ts b/src/performanceMarkdownModuleUrlContract.test.ts new file mode 100644 index 00000000..af022d47 --- /dev/null +++ b/src/performanceMarkdownModuleUrlContract.test.ts @@ -0,0 +1,69 @@ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-markdown.mjs', +); + +describe('Markdown measurement module URL authority', () => { + it('rejects file URLs with a non-local host before loading the measured module', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-module-url-')); + const input = join(root, 'large.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const output = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic benchmark fixture\n', 'utf8'); + writeFileSync( + modulePath, + 'export function markdownToHtml(source) { return `

${source}

`; }\n', + 'utf8', + ); + const artifactSha256 = createHash('sha256') + .update(readFileSync(modulePath)) + .digest('hex'); + const nonLocalFileUrl = pathToFileURL(modulePath); + nonLocalFileUrl.hostname = 'example.invalid'; + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + nonLocalFileUrl.href, + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + artifactSha256, + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module must be a local regular file.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 2bcd1de9569f51bef83be2d0ae4f13f2cf085f60 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:49:22 -0700 Subject: [PATCH 064/209] fix(perf): require local file URL authority --- benchmarks/measure-markdown.mjs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 2154089a..04357926 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -13,7 +13,7 @@ import { writeFileSync, } from 'node:fs'; import { dirname, resolve } from 'node:path'; -import { pathToFileURL } from 'node:url'; +import { fileURLToPath, pathToFileURL } from 'node:url'; const MAX_INPUT_BYTES = 16 * 1024 * 1024; const MAX_MODULE_BYTES = 16 * 1024 * 1024; @@ -180,7 +180,12 @@ function resolveLocalModule(pathOrUrl) { if (moduleUrl.protocol !== 'file:') { throw new Error('Measured Markdown module must be a local regular file.'); } - const resolvedPath = resolve(decodeURIComponent(moduleUrl.pathname)); + let resolvedPath; + try { + resolvedPath = resolve(fileURLToPath(moduleUrl)); + } catch { + throw new Error('Measured Markdown module must be a local regular file.'); + } const metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); if ( metadata === undefined || From 398c16b5dc6aaafda5296470908ed8960cbac8bc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 04:36:26 -0700 Subject: [PATCH 065/209] test(perf): redact measured module failures --- ...ownMeasurementErrorPrivacyContract.test.ts | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts diff --git a/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts new file mode 100644 index 00000000..4f6a08c3 --- /dev/null +++ b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts @@ -0,0 +1,96 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const RUNTIME_ID = 'node-22.18.0'; +const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function sha256(value: string) { + return createHash('sha256').update(value, 'utf8').digest('hex'); +} + +function runMeasurement(moduleSource: string) { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-error-privacy-')); + const input = join(root, 'document.md'); + const modulePath = join(root, 'measured.mjs'); + const output = join(root, 'samples.json'); + writeFileSync(input, '# Public benchmark fixture\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + REFERENCE_HARDWARE_ID, + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + return { root, output, result }; +} + +describe('Markdown measurement error privacy contract', () => { + it('redacts exceptions raised while loading the measured module', () => { + const privateSentinel = 'private-import-sentinel-must-not-leak'; + const moduleSource = `throw new Error('${privateSentinel}');\nexport function markdownToHtml(value) { return value; }\n`; + const { root, output, result } = runMeasurement(moduleSource); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module could not be loaded.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts exceptions raised by the measured serializer', () => { + const privateSentinel = 'private-serializer-sentinel-must-not-leak'; + const moduleSource = `export function markdownToHtml() { throw new Error('${privateSentinel}'); }\n`; + const { root, output, result } = runMeasurement(moduleSource); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured markdownToHtml() execution failed.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 8369d42a1ae18fd2e6096e574e456f9d08209b78 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 04:39:12 -0700 Subject: [PATCH 066/209] fix(perf): redact measured module failures --- benchmarks/measure-markdown.mjs | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 04357926..c50733c7 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -226,6 +226,22 @@ function refersToSameFile(leftPath, rightPath) { return left.dev === right.dev && left.ino === right.ino; } +async function loadMeasuredModule(modulePath) { + try { + return await import(pathToFileURL(modulePath).href); + } catch { + throw new Error('Measured Markdown module could not be loaded.'); + } +} + +function runMeasuredMarkdownToHtml(markdownToHtml, source) { + try { + return markdownToHtml(source); + } catch { + throw new Error('Measured markdownToHtml() execution failed.'); + } +} + async function main() { const args = resolveArguments(process.argv.slice(2)); if ( @@ -246,12 +262,15 @@ async function main() { } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); - const measuredModule = await import(pathToFileURL(modulePath).href); + const measuredModule = await loadMeasuredModule(modulePath); if (typeof measuredModule.markdownToHtml !== 'function') { throw new Error('Measured Markdown module must export markdownToHtml().'); } - const warmup = measuredModule.markdownToHtml(source); + const warmup = runMeasuredMarkdownToHtml( + measuredModule.markdownToHtml, + source, + ); if (typeof warmup !== 'string') { throw new Error('Measured markdownToHtml() must return a string.'); } @@ -259,7 +278,10 @@ async function main() { const samples = []; for (let index = 0; index < args.sampleCount; index += 1) { const start = performance.now(); - const output = measuredModule.markdownToHtml(source); + const output = runMeasuredMarkdownToHtml( + measuredModule.markdownToHtml, + source, + ); const elapsed = performance.now() - start; if ( typeof output !== 'string' || From df31a769e8f2ce780aa30fd06e82c0c8480ec8bc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 05:04:27 -0700 Subject: [PATCH 067/209] test(perf): require deterministic demo chunking --- src/demoBundleChunking.test.ts | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 src/demoBundleChunking.test.ts diff --git a/src/demoBundleChunking.test.ts b/src/demoBundleChunking.test.ts new file mode 100644 index 00000000..d6097e45 --- /dev/null +++ b/src/demoBundleChunking.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest'; +import { demoVendorChunk } from '../vite.demo.chunking'; + +describe('demo bundle chunking contract', () => { + it('keeps major editor dependency families in deterministic vendor chunks', () => { + const pnpmPrefix = '/repo/node_modules/.pnpm/example/node_modules/'; + + expect(demoVendorChunk(`${pnpmPrefix}react-dom/client.js`)).toBe( + 'react-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}@tiptap/pm/state/index.js`)).toBe( + 'prosemirror-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}prosemirror-state/dist/index.js`)).toBe( + 'prosemirror-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}@tiptap/core/dist/index.js`)).toBe( + 'tiptap-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}marked/lib/marked.esm.js`)).toBe( + 'serialization-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}turndown/lib/turndown.es.js`)).toBe( + 'serialization-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}yjs/dist/yjs.mjs`)).toBe( + 'collaboration-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}lodash-es/lodash.js`)).toBe('vendor'); + }); + + it('leaves application modules to Rollup and normalizes Windows paths', () => { + expect(demoVendorChunk('/repo/demo/App.tsx')).toBeUndefined(); + expect(demoVendorChunk(String.raw`C:\repo\node_modules\react\index.js`)).toBe( + 'react-vendor', + ); + }); +}); From 0ec9731ddae2fd8729a00e6dd155885780654234 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 05:05:06 -0700 Subject: [PATCH 068/209] feat(perf): classify demo vendor chunks --- vite.demo.chunking.ts | 54 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 vite.demo.chunking.ts diff --git a/vite.demo.chunking.ts b/vite.demo.chunking.ts new file mode 100644 index 00000000..87cc2834 --- /dev/null +++ b/vite.demo.chunking.ts @@ -0,0 +1,54 @@ +const NODE_MODULES = '/node_modules/'; + +function hasPackage(moduleId: string, packageName: string): boolean { + return moduleId.includes(`${NODE_MODULES}${packageName}/`); +} + +/** + * Assign large demo-only dependency families to stable Rollup vendor chunks. + * Product/library entry points are unchanged; this only keeps the standalone + * buyer demo from regressing into one oversized JavaScript payload. + */ +export function demoVendorChunk(id: string): string | undefined { + const moduleId = id.replace(/\\/g, '/'); + + if (!moduleId.includes(NODE_MODULES)) { + return undefined; + } + + if ( + hasPackage(moduleId, 'react') || + hasPackage(moduleId, 'react-dom') || + hasPackage(moduleId, 'scheduler') + ) { + return 'react-vendor'; + } + + if ( + hasPackage(moduleId, '@tiptap/pm') || + moduleId.includes(`${NODE_MODULES}prosemirror-`) + ) { + return 'prosemirror-vendor'; + } + + if (moduleId.includes(`${NODE_MODULES}@tiptap/`)) { + return 'tiptap-vendor'; + } + + if ( + hasPackage(moduleId, 'marked') || + hasPackage(moduleId, 'turndown') || + hasPackage(moduleId, 'turndown-plugin-gfm') + ) { + return 'serialization-vendor'; + } + + if ( + hasPackage(moduleId, 'yjs') || + hasPackage(moduleId, 'y-prosemirror') + ) { + return 'collaboration-vendor'; + } + + return 'vendor'; +} From 490795bfa0389841ff3c7a17ad261e86df7b0eef Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 05:05:24 -0700 Subject: [PATCH 069/209] fix(perf): split oversized demo bundle --- vite.demo.config.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vite.demo.config.ts b/vite.demo.config.ts index dd5e2d18..e5d2a0b6 100644 --- a/vite.demo.config.ts +++ b/vite.demo.config.ts @@ -1,6 +1,7 @@ import { resolve } from 'node:path'; import { defineConfig } from 'vite'; import react from '@vitejs/plugin-react'; +import { demoVendorChunk } from './vite.demo.chunking'; // Standalone demo build (Vite). `pnpm build:demo` emits a static site to // dist-demo/ that can be served by any static host or the provided Dockerfile. @@ -11,5 +12,10 @@ export default defineConfig({ build: { outDir: resolve(__dirname, 'dist-demo'), emptyOutDir: true, + rollupOptions: { + output: { + manualChunks: demoVendorChunk, + }, + }, }, }); From 84aab25f818874eaddb458a5b457aa51d3ea84f7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 06:07:27 -0700 Subject: [PATCH 070/209] test(perf): require revision-evidence measurement harness --- ...ormanceRevisionMeasurementContract.test.ts | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 src/performanceRevisionMeasurementContract.test.ts diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts new file mode 100644 index 00000000..43c2cb21 --- /dev/null +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -0,0 +1,172 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +interface BenchmarkSamples { + readonly contractVersion: 1; + readonly benchmarkId: string; + readonly unit: 'ms'; + readonly sourceCommitSha: string; + readonly artifactSha256: string; + readonly documentProfile: 'small' | 'medium' | 'large' | 'stress'; + readonly runtimeId: string; + readonly referenceHardwareId: string; + readonly samples: number[]; +} + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-revision-evidence.mjs', +); +const summaryScript = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const RUNTIME_ID = 'node-22.18.0'; +const HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function fileSha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +function argumentsFor( + input: string, + modulePath: string, + output: string, +): string[] { + return [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '3', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + fileSha256(modulePath), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + HARDWARE_ID, + '--output', + output, + ]; +} + +function writeSyntheticEnvelope(path: string): void { + writeFileSync( + path, + JSON.stringify({ + schemaId: 'https://inkspan.io/schemas/document-envelope/v1', + schemaVersion: 1, + documentJson: { + type: 'doc', + content: [ + { + type: 'paragraph', + content: [{ type: 'text', text: 'Synthetic benchmark content' }], + }, + ], + }, + }), + 'utf8', + ); +} + +describe('revision-evidence runtime measurement contract', () => { + it('writes privacy-safe revision samples consumable by the canonical summarizer', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const samplesPath = join(root, 'samples.json'); + const summaryDirectory = join(root, 'summary'); + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) { return { revision: { digestHex: String(source.byteLength).padStart(64, "0") } }; }\n', + 'utf8', + ); + + execFileSync(process.execPath, argumentsFor(input, modulePath, samplesPath), { + cwd: process.cwd(), + stdio: ['ignore', 'pipe', 'pipe'], + }); + + const samples = JSON.parse( + readFileSync(samplesPath, 'utf8'), + ) as BenchmarkSamples; + expect(samples).toMatchObject({ + contractVersion: 1, + benchmarkId: 'revision-evidence-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: fileSha256(modulePath), + documentProfile: 'large', + runtimeId: RUNTIME_ID, + referenceHardwareId: HARDWARE_ID, + }); + expect(samples.samples).toHaveLength(3); + expect( + samples.samples.every( + (sample) => Number.isFinite(sample) && sample >= 0, + ), + ).toBe(true); + expect(readFileSync(samplesPath, 'utf8')).not.toContain( + 'Synthetic benchmark content', + ); + + execFileSync( + process.execPath, + [summaryScript, '--input', samplesPath, '--output', summaryDirectory], + { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'pipe'] }, + ); + expect( + JSON.parse(readFileSync(join(summaryDirectory, 'summary.json'), 'utf8')), + ).toMatchObject({ + sampleCount: 3, + benchmarkId: 'revision-evidence-large', + unit: 'ms', + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before output when the measured module lacks the revision API', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-export-')); + const input = join(root, 'small.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync(modulePath, 'export const other = true;\n', 'utf8'); + + const result = spawnSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath), + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured revision module must export createDocumentEnvelopeRevisionEvidenceBytes().', + ); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 1f64915b563a9e5db66de4b2b6299eb5506faf61 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 06:08:09 -0700 Subject: [PATCH 071/209] feat(perf): measure packed revision-evidence latency --- benchmarks/measure-revision-evidence.mjs | 333 +++++++++++++++++++++++ 1 file changed, 333 insertions(+) create mode 100644 benchmarks/measure-revision-evidence.mjs diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs new file mode 100644 index 00000000..10040810 --- /dev/null +++ b/benchmarks/measure-revision-evidence.mjs @@ -0,0 +1,333 @@ +import { createHash } from 'node:crypto'; +import { performance } from 'node:perf_hooks'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdirSync, + openSync, + readSync, + realpathSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const MAX_INPUT_BYTES = 16 * 1024 * 1024; +const MAX_MODULE_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000; +const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; + +function resolveArguments(argv) { + const expectedFlags = [ + '--input', + '--module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', + ]; + if ( + argv.length !== expectedFlags.length * 2 || + expectedFlags.some((flag, index) => argv[index * 2] !== flag) || + expectedFlags.some((_, index) => argv[index * 2 + 1]?.length === 0) + ) { + throw new Error( + 'Usage: node benchmarks/measure-revision-evidence.mjs --input --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', + ); + } + + const values = Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); + const profile = values['--profile']; + if (!DOCUMENT_PROFILES.has(profile)) { + throw new Error('Revision benchmark profile is invalid.'); + } + const sampleCount = Number(values['--samples']); + if ( + !Number.isSafeInteger(sampleCount) || + sampleCount < 1 || + sampleCount > MAX_SAMPLES + ) { + throw new Error('Revision benchmark sample count must be an integer from 1 to 1000.'); + } + const sourceCommitSha = values['--source-commit-sha']; + if (!SHA1_PATTERN.test(sourceCommitSha)) { + throw new Error( + 'Revision benchmark source commit must be a lowercase 40-character SHA.', + ); + } + const artifactSha256 = values['--artifact-sha256']; + if (!SHA256_PATTERN.test(artifactSha256)) { + throw new Error( + 'Revision benchmark artifact digest must be a lowercase 64-character SHA-256.', + ); + } + const runtimeId = values['--runtime-id']; + if (!RUNTIME_ID_PATTERN.test(runtimeId)) { + throw new Error('Revision benchmark runtime ID is invalid.'); + } + const referenceHardwareId = values['--reference-hardware-id']; + if (!REFERENCE_HARDWARE_ID_PATTERN.test(referenceHardwareId)) { + throw new Error('Revision benchmark reference hardware ID is invalid.'); + } + + return Object.freeze({ + inputPath: resolve(values['--input']), + modulePath: values['--module'], + profile, + sampleCount, + sourceCommitSha, + artifactSha256, + runtimeId, + referenceHardwareId, + outputPath: resolve(values['--output']), + }); +} + +function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversizedMessage) { + const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error(invalidFileMessage); + } + + const descriptor = openSync(path, READ_ONLY_NOFOLLOW); + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error(invalidFileMessage); + } + if (metadata.size > maximumBytes) { + throw new Error(oversizedMessage); + } + const chunks = []; + let totalBytes = 0; + while (totalBytes <= maximumBytes) { + const remainingBudget = maximumBytes + 1 - totalBytes; + const chunk = Buffer.allocUnsafe(Math.min(READ_CHUNK_BYTES, remainingBudget)); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > maximumBytes) { + throw new Error(oversizedMessage); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + return Buffer.concat(chunks, totalBytes); + } finally { + closeSync(descriptor); + } +} + +function readBoundedEnvelopeBytes(path) { + return readBoundedRegularFile( + path, + MAX_INPUT_BYTES, + 'Revision benchmark input must be a regular non-symlink file.', + 'Revision benchmark input exceeds the supported size.', + ); +} + +function resolveLocalModule(pathOrUrl) { + if ( + pathOrUrl.startsWith('http:') || + pathOrUrl.startsWith('https:') || + pathOrUrl.startsWith('data:') || + pathOrUrl.startsWith('node:') + ) { + throw new Error('Measured revision module must be a local regular file.'); + } + let moduleUrl; + try { + moduleUrl = pathOrUrl.startsWith('file:') + ? new URL(pathOrUrl) + : pathToFileURL(resolve(pathOrUrl)); + } catch { + throw new Error('Measured revision module must be a local regular file.'); + } + if (moduleUrl.protocol !== 'file:') { + throw new Error('Measured revision module must be a local regular file.'); + } + let resolvedPath; + try { + resolvedPath = resolve(fileURLToPath(moduleUrl)); + } catch { + throw new Error('Measured revision module must be a local regular file.'); + } + const metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); + if ( + metadata === undefined || + metadata.isSymbolicLink() || + !metadata.isFile() + ) { + throw new Error('Measured revision module must be a local regular file.'); + } + return realpathSync(resolvedPath); +} + +function measuredModuleSha256(modulePath) { + const bytes = readBoundedRegularFile( + modulePath, + MAX_MODULE_BYTES, + 'Measured revision module must be a local regular file.', + 'Measured revision module exceeds the supported size.', + ); + return createHash('sha256').update(bytes).digest('hex'); +} + +function verifyMeasuredModuleDigest(modulePath, expectedSha256) { + if (measuredModuleSha256(modulePath) !== expectedSha256) { + throw new Error( + 'Revision benchmark artifact digest does not match the measured module.', + ); + } +} + +function refersToSameFile(leftPath, rightPath) { + const rightMetadata = lstatSync(rightPath, { throwIfNoEntry: false }); + if (rightMetadata === undefined) return false; + if (!rightMetadata.isFile()) { + throw new Error('Revision benchmark output must be a regular file.'); + } + const left = statSync(leftPath); + const right = statSync(rightPath); + return left.dev === right.dev && left.ino === right.ino; +} + +async function loadMeasuredModule(modulePath) { + try { + return await import(pathToFileURL(modulePath).href); + } catch { + throw new Error('Measured revision module could not be loaded.'); + } +} + +async function runMeasuredRevision(createRevisionEvidence, source) { + let evidence; + try { + evidence = await createRevisionEvidence(source); + } catch { + throw new Error('Measured revision-evidence execution failed.'); + } + if ( + typeof evidence !== 'object' || + evidence === null || + typeof evidence.revision !== 'object' || + evidence.revision === null || + typeof evidence.revision.digestHex !== 'string' || + !SHA256_PATTERN.test(evidence.revision.digestHex) + ) { + throw new Error('Measured revision-evidence result is invalid.'); + } +} + +async function main() { + const args = resolveArguments(process.argv.slice(2)); + if ( + args.inputPath === args.outputPath || + refersToSameFile(args.inputPath, args.outputPath) + ) { + throw new Error('Revision benchmark output must not overwrite its input.'); + } + const source = readBoundedEnvelopeBytes(args.inputPath); + const modulePath = resolveLocalModule(args.modulePath); + if ( + modulePath === args.outputPath || + refersToSameFile(modulePath, args.outputPath) + ) { + throw new Error( + 'Revision benchmark output must not overwrite the measured module.', + ); + } + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + + const measuredModule = await loadMeasuredModule(modulePath); + if ( + typeof measuredModule.createDocumentEnvelopeRevisionEvidenceBytes !== + 'function' + ) { + throw new Error( + 'Measured revision module must export createDocumentEnvelopeRevisionEvidenceBytes().', + ); + } + + await runMeasuredRevision( + measuredModule.createDocumentEnvelopeRevisionEvidenceBytes, + source, + ); + + const samples = []; + for (let index = 0; index < args.sampleCount; index += 1) { + const start = performance.now(); + await runMeasuredRevision( + measuredModule.createDocumentEnvelopeRevisionEvidenceBytes, + source, + ); + const elapsed = performance.now() - start; + if (!Number.isFinite(elapsed) || elapsed < 0) { + throw new Error('Revision measurement produced invalid runtime evidence.'); + } + samples.push(elapsed); + } + + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + const outputMetadata = lstatSync(args.outputPath, { throwIfNoEntry: false }); + if (outputMetadata !== undefined && !outputMetadata.isFile()) { + throw new Error('Revision benchmark output must be a regular file.'); + } + mkdirSync(dirname(args.outputPath), { recursive: true }); + writeFileSync( + args.outputPath, + `${JSON.stringify( + { + contractVersion: 1, + benchmarkId: `revision-evidence-${args.profile}`, + unit: 'ms', + sourceCommitSha: args.sourceCommitSha, + artifactSha256: args.artifactSha256, + documentProfile: args.profile, + runtimeId: args.runtimeId, + referenceHardwareId: args.referenceHardwareId, + samples, + }, + null, + 2, + )}\n`, + 'utf8', + ); +} + +try { + await main(); +} catch (error) { + const message = + error instanceof Error + ? error.message + : 'Revision benchmark measurement failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} From 407b8a15a2db64f686b6971bf9e77daa9f8090d1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 06:41:06 -0700 Subject: [PATCH 072/209] test(perf): reject symlinked summary inputs --- ...rmanceRegressionComparatorContract.test.ts | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts index 997a02ac..dfeab91e 100644 --- a/src/performanceRegressionComparatorContract.test.ts +++ b/src/performanceRegressionComparatorContract.test.ts @@ -1,5 +1,5 @@ import { spawnSync } from 'node:child_process'; -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; @@ -202,6 +202,48 @@ describe('benchmark regression comparator contract', () => { } }); + it('rejects symlinked summary inputs instead of comparing mutable aliases', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-symlink-')); + const baselineTargetPath = join(root, 'baseline-target.json'); + const baselinePath = join(root, 'baseline-link.json'); + const currentPath = join(root, 'current.json'); + try { + writeFileSync(baselineTargetPath, `${JSON.stringify(summary())}\n`, 'utf8'); + symlinkSync(baselineTargetPath, baselinePath); + writeFileSync( + currentPath, + `${JSON.stringify(summary({ artifactSha256: CURRENT_ARTIFACT_SHA256 }))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary input must be a regular non-symlink file.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('fails closed on a named-pipe summary instead of blocking before regular-file validation', () => { if (process.platform === 'win32') return; From 5882974bb8d63ade28567a1823d21f918d520c90 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 06:44:12 -0700 Subject: [PATCH 073/209] fix(perf): reject symlinked summary evidence --- benchmarks/compare-summaries.mjs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index 975e8c97..6b2ba4be 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -2,6 +2,7 @@ import { closeSync, constants, fstatSync, + lstatSync, openSync, readSync, } from 'node:fs'; @@ -11,7 +12,9 @@ const MAX_INPUT_BYTES = 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000_000; const READ_ONLY_NONBLOCKING = - constants.O_RDONLY | (constants.O_NONBLOCK ?? 0); + constants.O_RDONLY | + (constants.O_NONBLOCK ?? 0) | + (constants.O_NOFOLLOW ?? 0); const BENCHMARK_ID_PATTERN = /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; const UNITS = new Set(['ms', 'bytes']); @@ -84,6 +87,16 @@ function resolveArguments(argv) { } function readBoundedJson(path) { + const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + if (pathMetadata === undefined || pathMetadata.isSymbolicLink()) { + throw new Error( + 'Benchmark summary input must be a regular non-symlink file.', + ); + } + if (!pathMetadata.isFile()) { + throw new Error('Benchmark summary input must be a regular file.'); + } + const descriptor = openSync(path, READ_ONLY_NONBLOCKING); try { const metadata = fstatSync(descriptor); From 24d05fd21359813df77024f44f7a4c5df89d8820 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 11:30:30 -0700 Subject: [PATCH 074/209] test(perf): redact hostile revision result accessors --- ...ormanceRevisionMeasurementContract.test.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts index 43c2cb21..1db36f76 100644 --- a/src/performanceRevisionMeasurementContract.test.ts +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -169,4 +169,36 @@ describe('revision-evidence runtime measurement contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('redacts hostile revision-result accessors before publishing output', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-result-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const samplesPath = join(root, 'samples.json'); + const privateSentinel = 'tenant-private-revision-result-sentinel'; + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return new Proxy({}, { get(_target, property) { if (property === 'revision') throw new Error('${privateSentinel}'); return undefined; } }); }\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath), + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured revision-evidence result is invalid.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From 10b071d106b31e9445df4c9bea2ba81d68f521fe Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 11:34:15 -0700 Subject: [PATCH 075/209] fix(perf): contain hostile revision result accessors --- benchmarks/measure-revision-evidence.mjs | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index 10040810..24b293e4 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -233,14 +233,22 @@ async function runMeasuredRevision(createRevisionEvidence, source) { } catch { throw new Error('Measured revision-evidence execution failed.'); } - if ( - typeof evidence !== 'object' || - evidence === null || - typeof evidence.revision !== 'object' || - evidence.revision === null || - typeof evidence.revision.digestHex !== 'string' || - !SHA256_PATTERN.test(evidence.revision.digestHex) - ) { + + let digestHex; + try { + if (typeof evidence !== 'object' || evidence === null) { + throw new Error('invalid revision evidence'); + } + const revision = evidence.revision; + if (typeof revision !== 'object' || revision === null) { + throw new Error('invalid revision evidence'); + } + digestHex = revision.digestHex; + } catch { + throw new Error('Measured revision-evidence result is invalid.'); + } + + if (typeof digestHex !== 'string' || !SHA256_PATTERN.test(digestHex)) { throw new Error('Measured revision-evidence result is invalid.'); } } From 286bc0762e068797d2ecf6362b733df7dd3b13a6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 14:01:53 -0700 Subject: [PATCH 076/209] test(perf): define retained-memory settling RED contract --- src/performanceMemorySettlingContract.test.ts | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 src/performanceMemorySettlingContract.test.ts diff --git a/src/performanceMemorySettlingContract.test.ts b/src/performanceMemorySettlingContract.test.ts new file mode 100644 index 00000000..c8602354 --- /dev/null +++ b/src/performanceMemorySettlingContract.test.ts @@ -0,0 +1,133 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/analyze-memory-settling.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +type MemoryEvidenceOverrides = Partial<{ + sourceCommitSha: string; + artifactSha256: string; + documentProfile: string; + runtimeId: string; + referenceHardwareId: string; + warmupSamples: number; + samples: number[]; +}>; + +function evidence(overrides: MemoryEvidenceOverrides = {}) { + return { + contractVersion: 1, + benchmarkId: 'editor-lifecycle-retained-memory-large', + unit: 'bytes', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-24.0.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + warmupSamples: 2, + samples: [900, 920, 1000, 1005, 995, 1010, 1015, 1008, 1012, 1010], + ...overrides, + }; +} + +function runAnalysis( + root: string, + input: ReturnType, + maxGrowthBytes: string, +) { + const inputPath = join(root, 'memory-evidence.json'); + writeFileSync(inputPath, `${JSON.stringify(input)}\n`, 'utf8'); + return spawnSync( + process.execPath, + [ + script, + '--input', + inputPath, + '--window-size', + '3', + '--max-growth-bytes', + maxGrowthBytes, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); +} + +describe('retained-memory settling evidence contract', () => { + it('passes bounded settled growth using explicit warmup and comparison windows', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-pass-')); + try { + const result = runAnalysis(root, evidence(), '16'); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toEqual({ + contractVersion: 1, + benchmarkId: 'editor-lifecycle-retained-memory-large', + unit: 'bytes', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-24.0.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 10, + warmupSamples: 2, + windowSize: 3, + firstWindowMedianBytes: 1000, + lastWindowMedianBytes: 1010, + retainedGrowthBytes: 10, + maxGrowthBytes: 16, + passed: true, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails a retained-memory growth breach while preserving the public receipt', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-fail-')); + try { + const result = runAnalysis( + root, + evidence({ + samples: [900, 920, 1000, 1005, 995, 1100, 1120, 1110, 1130, 1140], + }), + '50', + ); + + expect(result.status).toBe(1); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toMatchObject({ + firstWindowMedianBytes: 1000, + lastWindowMedianBytes: 1130, + retainedGrowthBytes: 130, + maxGrowthBytes: 50, + passed: false, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects evidence that cannot supply two disjoint settled windows', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-short-')); + try { + const result = runAnalysis( + root, + evidence({ warmupSamples: 2, samples: [900, 920, 1000, 1005, 1010] }), + '50', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Memory settling evidence requires warmup plus two disjoint comparison windows.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From ef33eba152daf3144acd9101322e71df98c68c70 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 14:05:21 -0700 Subject: [PATCH 077/209] feat(perf): analyze retained-memory settling --- benchmarks/analyze-memory-settling.mjs | 283 +++++++++++++++++++++++++ 1 file changed, 283 insertions(+) create mode 100644 benchmarks/analyze-memory-settling.mjs diff --git a/benchmarks/analyze-memory-settling.mjs b/benchmarks/analyze-memory-settling.mjs new file mode 100644 index 00000000..9f5009d3 --- /dev/null +++ b/benchmarks/analyze-memory-settling.mjs @@ -0,0 +1,283 @@ +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readSync, +} from 'node:fs'; +import { resolve } from 'node:path'; + +const MAX_INPUT_BYTES = 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000_000; +const READ_ONLY_NONBLOCKING = + constants.O_RDONLY | + (constants.O_NONBLOCK ?? 0) | + (constants.O_NOFOLLOW ?? 0); +const BENCHMARK_ID_PATTERN = + /^editor-lifecycle-retained-memory-(?:small|medium|large|stress)$/u; +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const EVIDENCE_KEYS = new Set([ + 'contractVersion', + 'benchmarkId', + 'unit', + 'sourceCommitSha', + 'artifactSha256', + 'documentProfile', + 'runtimeId', + 'referenceHardwareId', + 'warmupSamples', + 'samples', +]); +const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true }); + +function resolveArguments(argv) { + if ( + argv.length !== 6 || + argv[0] !== '--input' || + argv[1].length === 0 || + argv[2] !== '--window-size' || + argv[3].trim().length === 0 || + argv[4] !== '--max-growth-bytes' || + argv[5].trim().length === 0 + ) { + throw new Error( + 'Usage: node benchmarks/analyze-memory-settling.mjs --input --window-size --max-growth-bytes ', + ); + } + + const windowSize = Number(argv[3]); + if (!Number.isSafeInteger(windowSize) || windowSize <= 0) { + throw new Error('Memory settling window size must be a positive safe integer.'); + } + + const maxGrowthBytes = Number(argv[5]); + if (!Number.isFinite(maxGrowthBytes) || maxGrowthBytes < 0) { + throw new Error( + 'Memory settling max growth bytes must be a finite non-negative number.', + ); + } + + return Object.freeze({ + inputPath: resolve(argv[1]), + windowSize, + maxGrowthBytes, + }); +} + +function readBoundedJson(path) { + const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + if (pathMetadata === undefined || pathMetadata.isSymbolicLink()) { + throw new Error( + 'Memory settling evidence input must be a regular non-symlink file.', + ); + } + if (!pathMetadata.isFile()) { + throw new Error('Memory settling evidence input must be a regular file.'); + } + + const descriptor = openSync(path, READ_ONLY_NONBLOCKING); + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error('Memory settling evidence input must be a regular file.'); + } + if (metadata.size > MAX_INPUT_BYTES) { + throw new Error('Memory settling evidence input exceeds the supported size.'); + } + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= MAX_INPUT_BYTES) { + const remainingBudget = MAX_INPUT_BYTES + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > MAX_INPUT_BYTES) { + throw new Error( + 'Memory settling evidence input exceeds the supported size.', + ); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + + let text; + try { + text = UTF8_DECODER.decode(Buffer.concat(chunks, totalBytes)); + } catch { + throw new Error('Memory settling evidence input must be valid UTF-8 JSON.'); + } + + try { + return JSON.parse(text); + } catch { + throw new Error('Memory settling evidence input must be valid JSON.'); + } + } finally { + closeSync(descriptor); + } +} + +function validateEvidence(value) { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Memory settling evidence input must be an object.'); + } + const keys = Object.keys(value); + if ( + keys.length !== EVIDENCE_KEYS.size || + keys.some((key) => !EVIDENCE_KEYS.has(key)) + ) { + throw new Error('Memory settling evidence input has an unsupported shape.'); + } + if (value.contractVersion !== 1) { + throw new Error('Memory settling evidence contractVersion must be 1.'); + } + if ( + typeof value.benchmarkId !== 'string' || + !BENCHMARK_ID_PATTERN.test(value.benchmarkId) + ) { + throw new Error('Memory settling evidence benchmarkId is invalid.'); + } + if (value.unit !== 'bytes') { + throw new Error('Memory settling evidence unit must be bytes.'); + } + if ( + typeof value.sourceCommitSha !== 'string' || + !SHA1_PATTERN.test(value.sourceCommitSha) + ) { + throw new Error('Memory settling evidence sourceCommitSha is invalid.'); + } + if ( + typeof value.artifactSha256 !== 'string' || + !SHA256_PATTERN.test(value.artifactSha256) + ) { + throw new Error('Memory settling evidence artifactSha256 is invalid.'); + } + if ( + typeof value.documentProfile !== 'string' || + !DOCUMENT_PROFILES.has(value.documentProfile) + ) { + throw new Error('Memory settling evidence documentProfile is invalid.'); + } + if ( + typeof value.runtimeId !== 'string' || + !RUNTIME_ID_PATTERN.test(value.runtimeId) + ) { + throw new Error('Memory settling evidence runtimeId is invalid.'); + } + if ( + typeof value.referenceHardwareId !== 'string' || + !REFERENCE_HARDWARE_ID_PATTERN.test(value.referenceHardwareId) + ) { + throw new Error('Memory settling evidence referenceHardwareId is invalid.'); + } + if ( + !Number.isSafeInteger(value.warmupSamples) || + value.warmupSamples < 0 || + value.warmupSamples > MAX_SAMPLES + ) { + throw new Error('Memory settling evidence warmupSamples is invalid.'); + } + if ( + !Array.isArray(value.samples) || + value.samples.length === 0 || + value.samples.length > MAX_SAMPLES || + value.samples.some( + (sample) => + !Number.isSafeInteger(sample) || sample < 0, + ) + ) { + throw new Error( + 'Memory settling evidence samples must be bounded non-negative safe integers.', + ); + } + if (value.warmupSamples >= value.samples.length) { + throw new Error('Memory settling evidence warmupSamples is invalid.'); + } + + return Object.freeze({ + benchmarkId: value.benchmarkId, + unit: value.unit, + sourceCommitSha: value.sourceCommitSha, + artifactSha256: value.artifactSha256, + documentProfile: value.documentProfile, + runtimeId: value.runtimeId, + referenceHardwareId: value.referenceHardwareId, + warmupSamples: value.warmupSamples, + samples: Object.freeze([...value.samples]), + }); +} + +function median(values) { + const sorted = [...values].sort((left, right) => left - right); + const midpoint = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 1) return sorted[midpoint]; + return (sorted[midpoint - 1] + sorted[midpoint]) / 2; +} + +function analyze(evidence, windowSize, maxGrowthBytes) { + const settledSamples = evidence.samples.slice(evidence.warmupSamples); + if (settledSamples.length < windowSize * 2) { + throw new Error( + 'Memory settling evidence requires warmup plus two disjoint comparison windows.', + ); + } + + const firstWindowMedianBytes = median(settledSamples.slice(0, windowSize)); + const lastWindowMedianBytes = median(settledSamples.slice(-windowSize)); + const retainedGrowthBytes = lastWindowMedianBytes - firstWindowMedianBytes; + + return Object.freeze({ + contractVersion: 1, + benchmarkId: evidence.benchmarkId, + unit: evidence.unit, + sourceCommitSha: evidence.sourceCommitSha, + artifactSha256: evidence.artifactSha256, + documentProfile: evidence.documentProfile, + runtimeId: evidence.runtimeId, + referenceHardwareId: evidence.referenceHardwareId, + sampleCount: evidence.samples.length, + warmupSamples: evidence.warmupSamples, + windowSize, + firstWindowMedianBytes, + lastWindowMedianBytes, + retainedGrowthBytes, + maxGrowthBytes, + passed: retainedGrowthBytes <= maxGrowthBytes, + }); +} + +function main() { + const { inputPath, windowSize, maxGrowthBytes } = resolveArguments( + process.argv.slice(2), + ); + const evidence = validateEvidence(readBoundedJson(inputPath)); + const result = analyze(evidence, windowSize, maxGrowthBytes); + process.stdout.write(`${JSON.stringify(result)}\n`); + if (!result.passed) process.exitCode = 1; +} + +try { + main(); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Memory settling analysis failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} From e4cab92ef0693fe37c5e2d0208750c6dd96cbcc8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 14:19:34 -0700 Subject: [PATCH 078/209] test(perf): reject mismatched memory evidence profiles --- src/performanceMemorySettlingContract.test.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/performanceMemorySettlingContract.test.ts b/src/performanceMemorySettlingContract.test.ts index c8602354..ef44d607 100644 --- a/src/performanceMemorySettlingContract.test.ts +++ b/src/performanceMemorySettlingContract.test.ts @@ -9,6 +9,7 @@ const SOURCE_COMMIT_SHA = 'a'.repeat(40); const ARTIFACT_SHA256 = 'b'.repeat(64); type MemoryEvidenceOverrides = Partial<{ + benchmarkId: string; sourceCommitSha: string; artifactSha256: string; documentProfile: string; @@ -130,4 +131,26 @@ describe('retained-memory settling evidence contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('rejects evidence whose benchmark profile disagrees with documentProfile', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-profile-')); + try { + const result = runAnalysis( + root, + evidence({ + benchmarkId: 'editor-lifecycle-retained-memory-small', + documentProfile: 'large', + }), + '50', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Memory settling evidence benchmark profile must match documentProfile.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From e671a5a9c40742e2a699016bf15277a590aad5ac Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 15:36:47 -0700 Subject: [PATCH 079/209] fix(perf): bind memory benchmark id to profile --- benchmarks/analyze-memory-settling.mjs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/benchmarks/analyze-memory-settling.mjs b/benchmarks/analyze-memory-settling.mjs index 9f5009d3..50a9f095 100644 --- a/benchmarks/analyze-memory-settling.mjs +++ b/benchmarks/analyze-memory-settling.mjs @@ -15,6 +15,7 @@ const READ_ONLY_NONBLOCKING = constants.O_RDONLY | (constants.O_NONBLOCK ?? 0) | (constants.O_NOFOLLOW ?? 0); +const BENCHMARK_ID_PREFIX = 'editor-lifecycle-retained-memory-'; const BENCHMARK_ID_PATTERN = /^editor-lifecycle-retained-memory-(?:small|medium|large|stress)$/u; const SHA1_PATTERN = /^[0-9a-f]{40}$/u; @@ -175,6 +176,13 @@ function validateEvidence(value) { ) { throw new Error('Memory settling evidence documentProfile is invalid.'); } + if ( + value.benchmarkId.slice(BENCHMARK_ID_PREFIX.length) !== value.documentProfile + ) { + throw new Error( + 'Memory settling evidence benchmark profile must match documentProfile.', + ); + } if ( typeof value.runtimeId !== 'string' || !RUNTIME_ID_PATTERN.test(value.runtimeId) From 5a6944c096665b96c307852524e8bbe58db16369 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 17:03:01 -0700 Subject: [PATCH 080/209] test(perf): reject inexact memory medians --- src/performanceMemorySettlingContract.test.ts | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/performanceMemorySettlingContract.test.ts b/src/performanceMemorySettlingContract.test.ts index ef44d607..93819962 100644 --- a/src/performanceMemorySettlingContract.test.ts +++ b/src/performanceMemorySettlingContract.test.ts @@ -39,6 +39,7 @@ function runAnalysis( root: string, input: ReturnType, maxGrowthBytes: string, + windowSize = '3', ) { const inputPath = join(root, 'memory-evidence.json'); writeFileSync(inputPath, `${JSON.stringify(input)}\n`, 'utf8'); @@ -49,7 +50,7 @@ function runAnalysis( '--input', inputPath, '--window-size', - '3', + windowSize, '--max-growth-bytes', maxGrowthBytes, ], @@ -153,4 +154,28 @@ describe('retained-memory settling evidence contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('fails closed before a precision-loss false green from an inexact even-window median', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-precision-')); + try { + const maxSafe = Number.MAX_SAFE_INTEGER; + const result = runAnalysis( + root, + evidence({ + warmupSamples: 0, + samples: [maxSafe - 1, maxSafe - 1, maxSafe - 1, maxSafe], + }), + '0.25', + '2', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Memory settling window median must be exactly representable.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From a1b5201b1d1708a8c61989987f0eca0b27e9db10 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 17:07:21 -0700 Subject: [PATCH 081/209] fix(perf): fail closed on inexact memory medians --- benchmarks/analyze-memory-settling.mjs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/benchmarks/analyze-memory-settling.mjs b/benchmarks/analyze-memory-settling.mjs index 50a9f095..9e60c947 100644 --- a/benchmarks/analyze-memory-settling.mjs +++ b/benchmarks/analyze-memory-settling.mjs @@ -236,7 +236,17 @@ function median(values) { const sorted = [...values].sort((left, right) => left - right); const midpoint = Math.floor(sorted.length / 2); if (sorted.length % 2 === 1) return sorted[midpoint]; - return (sorted[midpoint - 1] + sorted[midpoint]) / 2; + + const left = sorted[midpoint - 1]; + const right = sorted[midpoint]; + const distance = right - left; + const wholeMidpoint = left + Math.floor(distance / 2); + if (distance % 2 === 1 && wholeMidpoint >= 2 ** 52) { + throw new Error( + 'Memory settling window median must be exactly representable.', + ); + } + return wholeMidpoint + (distance % 2) / 2; } function analyze(evidence, windowSize, maxGrowthBytes) { From 80c8bbb95a0072d87b30a783bd6b9dc83d5b2767 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 18:29:25 -0700 Subject: [PATCH 082/209] test(perf): reject mismatched benchmark profiles --- ...gressionProfileConsistencyContract.test.ts | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 src/performanceRegressionProfileConsistencyContract.test.ts diff --git a/src/performanceRegressionProfileConsistencyContract.test.ts b/src/performanceRegressionProfileConsistencyContract.test.ts new file mode 100644 index 00000000..bf787527 --- /dev/null +++ b/src/performanceRegressionProfileConsistencyContract.test.ts @@ -0,0 +1,64 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/compare-summaries.mjs'); + +function summary(artifactSha256: string) { + return { + contractVersion: 1, + benchmarkId: 'editor-input-small', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256, + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + minimum: 70, + p50: 80, + p75: 90, + p95: 100, + maximum: 110, + }; +} + +describe('benchmark regression profile consistency contract', () => { + it('rejects summaries whose benchmarkId profile disagrees with documentProfile', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-profile-')); + const baselinePath = join(root, 'baseline.json'); + const currentPath = join(root, 'current.json'); + + try { + writeFileSync(baselinePath, `${JSON.stringify(summary('b'.repeat(64)))}\n`, 'utf8'); + writeFileSync(currentPath, `${JSON.stringify(summary('c'.repeat(64)))}\n`, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary profile must match documentProfile.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From dcf8b65b3a97979c8b9cf176e18b699247648c00 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 18:33:14 -0700 Subject: [PATCH 083/209] fix(perf): bind benchmark ids to document profiles --- benchmarks/compare-summaries.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index 6b2ba4be..4b891f14 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -187,6 +187,9 @@ function validateSummary(value) { ) { throw new Error('Benchmark summary documentProfile is invalid.'); } + if (!value.benchmarkId.endsWith(`-${value.documentProfile}`)) { + throw new Error('Benchmark summary profile must match documentProfile.'); + } if ( typeof value.runtimeId !== 'string' || !RUNTIME_ID_PATTERN.test(value.runtimeId) From b240eace91de6f2c6a91fdc752ec0cd96c1bf1b8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 18:35:29 -0700 Subject: [PATCH 084/209] test(perf): reject mismatched sample profiles --- ...eSummaryProfileConsistencyContract.test.ts | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 src/performanceSummaryProfileConsistencyContract.test.ts diff --git a/src/performanceSummaryProfileConsistencyContract.test.ts b/src/performanceSummaryProfileConsistencyContract.test.ts new file mode 100644 index 00000000..118346de --- /dev/null +++ b/src/performanceSummaryProfileConsistencyContract.test.ts @@ -0,0 +1,47 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +describe('benchmark summary profile consistency contract', () => { + it('rejects sample evidence whose benchmarkId profile disagrees with documentProfile', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-summary-profile-')); + const inputPath = join(root, 'samples.json'); + const outputDirectory = join(root, 'summary'); + + try { + writeFileSync( + inputPath, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'editor-input-small', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256: 'b'.repeat(64), + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [70, 80, 90, 100], + })}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [script, '--input', inputPath, '--output', outputDirectory], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample profile must match documentProfile.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 00b062e680ba6d7395a1a3d55cd923f9b097b2ea Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 18:38:21 -0700 Subject: [PATCH 085/209] fix(perf): bind sample ids to document profiles --- benchmarks/summarize-samples.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 354c2539..58d00536 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -151,6 +151,9 @@ function validateInput(value) { ) { throw new Error('Benchmark documentProfile is invalid.'); } + if (!value.benchmarkId.endsWith(`-${value.documentProfile}`)) { + throw new Error('Benchmark sample profile must match documentProfile.'); + } if ( typeof value.runtimeId !== 'string' || !RUNTIME_ID_PATTERN.test(value.runtimeId) From 1a76294f71eb4f8f17505b3bc773879c54373bf5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 20:05:51 -0700 Subject: [PATCH 086/209] test(perf): reject corpus output symlink escape --- src/performanceCorpusContract.test.ts | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/performanceCorpusContract.test.ts b/src/performanceCorpusContract.test.ts index e92c7f2c..02bc5c02 100644 --- a/src/performanceCorpusContract.test.ts +++ b/src/performanceCorpusContract.test.ts @@ -1,5 +1,12 @@ import { execFileSync } from 'node:child_process'; -import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; @@ -85,4 +92,20 @@ describe('deterministic synthetic performance corpus', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('fails closed instead of following a corpus output symlink', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-corpus-symlink-')); + const outputDirectory = join(root, 'output'); + const victimPath = join(root, 'victim.md'); + try { + mkdirSync(outputDirectory, { recursive: true }); + writeFileSync(victimPath, 'buyer-owned evidence\n', 'utf8'); + symlinkSync(victimPath, join(outputDirectory, 'small.md')); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readFileSync(victimPath, 'utf8')).toBe('buyer-owned evidence\n'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From abf7d8944b83fc2e538b19b939450a9e428d2ddd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 20:10:50 -0700 Subject: [PATCH 087/209] fix(perf): protect corpus output paths --- benchmarks/generate-corpus.mjs | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/benchmarks/generate-corpus.mjs b/benchmarks/generate-corpus.mjs index af73290e..af9d85e9 100644 --- a/benchmarks/generate-corpus.mjs +++ b/benchmarks/generate-corpus.mjs @@ -1,5 +1,11 @@ import { createHash } from 'node:crypto'; -import { mkdirSync, writeFileSync } from 'node:fs'; +import { + lstatSync, + mkdirSync, + renameSync, + rmSync, + writeFileSync, +} from 'node:fs'; import { resolve } from 'node:path'; const RASTER_FIXTURES = Object.freeze([ @@ -42,6 +48,24 @@ const SCRIPT_LABELS = Object.freeze([ 'mixed', ]); +let outputWriteCounter = 0; + +function writeRegularOutput(path, content) { + const existing = lstatSync(path, { throwIfNoEntry: false }); + if (existing !== undefined && !existing.isFile()) { + throw new Error('Benchmark corpus output must be a regular file.'); + } + + const temporaryPath = `${path}.tmp-${process.pid}-${outputWriteCounter}`; + outputWriteCounter += 1; + try { + writeFileSync(temporaryPath, content, { flag: 'wx' }); + renameSync(temporaryPath, path); + } finally { + rmSync(temporaryPath, { force: true }); + } +} + function buildSection(index) { const id = String(index).padStart(4, '0'); const tableRows = Array.from({ length: 4 }, (_, rowIndex) => { @@ -111,7 +135,7 @@ const profileManifest = {}; for (const [profile, sections] of Object.entries(PROFILE_SECTIONS)) { const body = buildProfile(profile, sections); const bytes = Buffer.from(body, 'utf8'); - writeFileSync(resolve(outputDirectory, `${profile}.md`), bytes); + writeRegularOutput(resolve(outputDirectory, `${profile}.md`), bytes); profileManifest[profile] = Object.freeze({ sections, bytes: bytes.byteLength, @@ -125,8 +149,7 @@ const manifest = Object.freeze({ scripts: SCRIPT_LABELS, profiles: profileManifest, }); -writeFileSync( +writeRegularOutput( resolve(outputDirectory, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`, - 'utf8', ); From 4002bc46202de7757f2274997d60bdb24c89e259 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 20:54:27 -0700 Subject: [PATCH 088/209] test(perf): redact benchmark output path failures --- ...ownMeasurementErrorPrivacyContract.test.ts | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts index 4f6a08c3..9b81bdec 100644 --- a/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts +++ b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts @@ -19,11 +19,14 @@ function sha256(value: string) { return createHash('sha256').update(value, 'utf8').digest('hex'); } -function runMeasurement(moduleSource: string) { +function runMeasurement( + moduleSource: string, + outputForRoot: (root: string) => string = (root) => join(root, 'samples.json'), +) { const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-error-privacy-')); const input = join(root, 'document.md'); const modulePath = join(root, 'measured.mjs'); - const output = join(root, 'samples.json'); + const output = outputForRoot(root); writeFileSync(input, '# Public benchmark fixture\n', 'utf8'); writeFileSync(modulePath, moduleSource, 'utf8'); @@ -93,4 +96,26 @@ describe('Markdown measurement error privacy contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('redacts filesystem details when output path traversal fails', () => { + const privateSentinel = 'private-output-sentinel-must-not-leak'; + const moduleSource = + 'export function markdownToHtml(value) { return value; }\n'; + const { root, output, result } = runMeasurement(moduleSource, (testRoot) => { + const blockedParent = join(testRoot, privateSentinel); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + return join(blockedParent, 'samples.json'); + }); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark output path could not be inspected.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From 1528f0eb3a53aa86453e8e77f74adcd6d743c8c3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 20:57:01 -0700 Subject: [PATCH 089/209] fix(perf): redact benchmark output path failures --- benchmarks/measure-markdown.mjs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index c50733c7..cbe42a96 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -215,8 +215,16 @@ function verifyMeasuredModuleDigest(modulePath, expectedSha256) { } } +function inspectOutputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Markdown benchmark output path could not be inspected.'); + } +} + function refersToSameFile(leftPath, rightPath) { - const rightMetadata = lstatSync(rightPath, { throwIfNoEntry: false }); + const rightMetadata = inspectOutputPath(rightPath); if (rightMetadata === undefined) return false; if (!rightMetadata.isFile()) { throw new Error('Markdown benchmark output must be a regular file.'); @@ -294,7 +302,7 @@ async function main() { } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); - const outputMetadata = lstatSync(args.outputPath, { throwIfNoEntry: false }); + const outputMetadata = inspectOutputPath(args.outputPath); if (outputMetadata !== undefined && !outputMetadata.isFile()) { throw new Error('Markdown benchmark output must be a regular file.'); } From da69be66ec10b4345ad95209df3367f37570ac17 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 21:01:19 -0700 Subject: [PATCH 090/209] test(perf): redact revision output path failures --- ...ormanceRevisionMeasurementContract.test.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts index 1db36f76..8c604a8c 100644 --- a/src/performanceRevisionMeasurementContract.test.ts +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -201,4 +201,38 @@ describe('revision-evidence runtime measurement contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('redacts filesystem details when output path traversal fails', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-output-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const privateSentinel = 'private-revision-output-sentinel-must-not-leak'; + const blockedParent = join(root, privateSentinel); + const samplesPath = join(blockedParent, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) { return { revision: { digestHex: String(source.byteLength).padStart(64, "0") } }; }\n', + 'utf8', + ); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath), + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Revision benchmark output path could not be inspected.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From 523c98605e3a24ab94154ed8d56c238cd2a100a6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 21:04:46 -0700 Subject: [PATCH 091/209] fix(perf): redact revision output path failures --- benchmarks/measure-revision-evidence.mjs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index 24b293e4..d830cb5d 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -207,8 +207,16 @@ function verifyMeasuredModuleDigest(modulePath, expectedSha256) { } } +function inspectOutputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Revision benchmark output path could not be inspected.'); + } +} + function refersToSameFile(leftPath, rightPath) { - const rightMetadata = lstatSync(rightPath, { throwIfNoEntry: false }); + const rightMetadata = inspectOutputPath(rightPath); if (rightMetadata === undefined) return false; if (!rightMetadata.isFile()) { throw new Error('Revision benchmark output must be a regular file.'); @@ -303,7 +311,7 @@ async function main() { } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); - const outputMetadata = lstatSync(args.outputPath, { throwIfNoEntry: false }); + const outputMetadata = inspectOutputPath(args.outputPath); if (outputMetadata !== undefined && !outputMetadata.isFile()) { throw new Error('Revision benchmark output must be a regular file.'); } From 28a45ac43b48c92d4124b6f71d699ca8875bcfc8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 21:06:45 -0700 Subject: [PATCH 092/209] test(perf): redact summary output preparation failures --- ...rformanceMeasurementOutputContract.test.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/performanceMeasurementOutputContract.test.ts b/src/performanceMeasurementOutputContract.test.ts index 45376b1f..895732ad 100644 --- a/src/performanceMeasurementOutputContract.test.ts +++ b/src/performanceMeasurementOutputContract.test.ts @@ -164,4 +164,36 @@ describe('benchmark summary output contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('redacts filesystem details when the output directory cannot be prepared', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-output-privacy-')); + const input = join(root, 'samples.json'); + const privateSentinel = 'private-summary-output-sentinel-must-not-leak'; + const blockedParent = join(root, privateSentinel); + const output = join(blockedParent, 'output'); + try { + writeValidInput(input); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output directory could not be prepared.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From d44925ea6ca13a6f5f5233bc70e56018a1bc6273 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 22:03:11 -0700 Subject: [PATCH 093/209] fix(perf): redact output directory preparation failures --- benchmarks/summarize-samples.mjs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 58d00536..65592796 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -253,11 +253,19 @@ function assertRegularOutputDestination(path) { } } +function prepareOutputDirectory(path) { + try { + mkdirSync(path, { recursive: true }); + } catch { + throw new Error('Benchmark summary output directory could not be prepared.'); + } +} + function main() { const { inputPath, outputDirectory } = resolveArguments(process.argv.slice(2)); const summaryJsonPath = resolve(outputDirectory, 'summary.json'); const summaryTextPath = resolve(outputDirectory, 'summary.txt'); - mkdirSync(outputDirectory, { recursive: true }); + prepareOutputDirectory(outputDirectory); if ( inputPath === summaryJsonPath || inputPath === summaryTextPath || From d4541760f8036ce1c637a87529eb9451fd673a9d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 01:01:44 -0700 Subject: [PATCH 094/209] test(perf): redact Markdown output publication failures --- ...ownMeasurementErrorPrivacyContract.test.ts | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts index 9b81bdec..c069b76f 100644 --- a/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts +++ b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts @@ -118,4 +118,26 @@ describe('Markdown measurement error privacy contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('redacts filesystem details when output publication cannot create its directory', () => { + const privateSentinel = `private-markdown-publication-${process.pid}`; + const moduleSource = + 'export function markdownToHtml(value) { return value; }\n'; + const blockedOutput = join('/sys', privateSentinel, 'samples.json'); + const { root, output, result } = runMeasurement( + moduleSource, + () => blockedOutput, + ); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark output could not be written.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From e066321ef7fd40f256dbbd7d75db04968459d62e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 01:05:39 -0700 Subject: [PATCH 095/209] fix(perf): redact Markdown output publication failures --- benchmarks/measure-markdown.mjs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index cbe42a96..e240af3c 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -250,6 +250,15 @@ function runMeasuredMarkdownToHtml(markdownToHtml, source) { } } +function writeMeasurementOutput(path, content) { + try { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, content, 'utf8'); + } catch { + throw new Error('Markdown benchmark output could not be written.'); + } +} + async function main() { const args = resolveArguments(process.argv.slice(2)); if ( @@ -306,8 +315,7 @@ async function main() { if (outputMetadata !== undefined && !outputMetadata.isFile()) { throw new Error('Markdown benchmark output must be a regular file.'); } - mkdirSync(dirname(args.outputPath), { recursive: true }); - writeFileSync( + writeMeasurementOutput( args.outputPath, `${JSON.stringify( { @@ -324,7 +332,6 @@ async function main() { null, 2, )}\n`, - 'utf8', ); } From 0b724e0263403adb18affe6ca16cb1914991a929 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 01:14:23 -0700 Subject: [PATCH 096/209] test(perf): redact revision output publication failures --- ...ormanceRevisionMeasurementContract.test.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts index 8c604a8c..d7258f6a 100644 --- a/src/performanceRevisionMeasurementContract.test.ts +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -235,4 +235,36 @@ describe('revision-evidence runtime measurement contract', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('redacts filesystem details when output publication cannot create its directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-publication-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const privateSentinel = `private-revision-publication-${process.pid}`; + const samplesPath = join('/sys', privateSentinel, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) { return { revision: { digestHex: String(source.byteLength).padStart(64, "0") } }; }\n', + 'utf8', + ); + + const result = spawnSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath), + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Revision benchmark output could not be written.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From 12a48b402a680eb5b4d5722e5185db6270f42ab6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 01:33:45 -0700 Subject: [PATCH 097/209] fix(perf): redact revision output publication failures --- benchmarks/measure-revision-evidence.mjs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index d830cb5d..2bec7367 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -234,6 +234,15 @@ async function loadMeasuredModule(modulePath) { } } +function writeMeasurementOutput(path, content) { + try { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, content, 'utf8'); + } catch { + throw new Error('Revision benchmark output could not be written.'); + } +} + async function runMeasuredRevision(createRevisionEvidence, source) { let evidence; try { @@ -315,8 +324,7 @@ async function main() { if (outputMetadata !== undefined && !outputMetadata.isFile()) { throw new Error('Revision benchmark output must be a regular file.'); } - mkdirSync(dirname(args.outputPath), { recursive: true }); - writeFileSync( + writeMeasurementOutput( args.outputPath, `${JSON.stringify( { @@ -333,7 +341,6 @@ async function main() { null, 2, )}\n`, - 'utf8', ); } From 86a771e0ce8994b195616fe67a78da83dbdbd2c3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 04:32:57 -0700 Subject: [PATCH 098/209] test(perf): require single-command benchmark suite --- ...formanceSingleCommandSuiteContract.test.ts | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 src/performanceSingleCommandSuiteContract.test.ts diff --git a/src/performanceSingleCommandSuiteContract.test.ts b/src/performanceSingleCommandSuiteContract.test.ts new file mode 100644 index 00000000..4747d273 --- /dev/null +++ b/src/performanceSingleCommandSuiteContract.test.ts @@ -0,0 +1,95 @@ +import { createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +describe('single-command benchmark suite contract', () => { + it('measures and summarizes one deterministic Markdown profile with one command', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); + temporaryDirectories.push(directory); + const inputPath = join(directory, 'input.md'); + const modulePath = join(directory, 'measured.mjs'); + const outputDirectory = join(directory, 'evidence'); + const moduleSource = + "export function markdownToHtml(source) { return `

${source}

`; }\n"; + writeFileSync(inputPath, '# Buyer benchmark\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + const artifactSha256 = createHash('sha256') + .update(moduleSource) + .digest('hex'); + + const output = execFileSync( + process.execPath, + [ + suitePath, + '--input', + inputPath, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + artifactSha256, + '--runtime-id', + 'node-22.0.0', + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + outputDirectory, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(JSON.parse(output.trim())).toEqual({ + contractVersion: 1, + documentProfile: 'small', + samples: 'samples.json', + status: 'completed', + summaryJson: 'summary/summary.json', + summaryText: 'summary/summary.txt', + }); + + const samples = JSON.parse( + readFileSync(join(outputDirectory, 'samples.json'), 'utf8'), + ) as { benchmarkId?: unknown; documentProfile?: unknown; samples?: unknown }; + expect(samples.benchmarkId).toBe('markdown-serialization-small'); + expect(samples.documentProfile).toBe('small'); + expect(samples.samples).toHaveLength(2); + + const summary = JSON.parse( + readFileSync(join(outputDirectory, 'summary', 'summary.json'), 'utf8'), + ) as { benchmarkId?: unknown; documentProfile?: unknown }; + expect(summary.benchmarkId).toBe('markdown-serialization-small'); + expect(summary.documentProfile).toBe('small'); + expect( + readFileSync(join(outputDirectory, 'summary', 'summary.txt'), 'utf8'), + ).toContain('markdown-serialization-small'); + }); +}); From 9a618bcf05f4f49d4f2fe62d833fbf7e0ad3a84b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 04:36:38 -0700 Subject: [PATCH 099/209] feat(perf): compose single-command benchmark suite --- benchmarks/run-current-suite.mjs | 94 ++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 benchmarks/run-current-suite.mjs diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs new file mode 100644 index 00000000..95354bdf --- /dev/null +++ b/benchmarks/run-current-suite.mjs @@ -0,0 +1,94 @@ +import { spawnSync } from 'node:child_process'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(benchmarkDirectory, '..'); +const expectedFlags = Object.freeze([ + '--input', + '--module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); + +function resolveArguments(argv) { + if ( + argv.length !== expectedFlags.length * 2 || + expectedFlags.some((flag, index) => argv[index * 2] !== flag) || + expectedFlags.some((_, index) => argv[index * 2 + 1]?.length === 0) + ) { + throw new Error( + 'Usage: node benchmarks/run-current-suite.mjs --input --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', + ); + } + + return Object.freeze({ + documentProfile: argv[5], + forwardedArguments: Object.freeze(argv.slice(0, -2)), + outputDirectory: resolve(argv[17]), + }); +} + +function runBoundedNodeScript(scriptName, args, failureMessage) { + const result = spawnSync( + process.execPath, + [resolve(benchmarkDirectory, scriptName), ...args], + { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 4 * 1024 * 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 120_000, + }, + ); + + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 + ) { + throw new Error(failureMessage); + } +} + +function main(argv) { + const args = resolveArguments(argv); + const samplesPath = resolve(args.outputDirectory, 'samples.json'); + const summaryDirectory = resolve(args.outputDirectory, 'summary'); + + runBoundedNodeScript( + 'measure-markdown.mjs', + [...args.forwardedArguments, '--output', samplesPath], + 'Benchmark suite measurement failed.', + ); + runBoundedNodeScript( + 'summarize-samples.mjs', + ['--input', samplesPath, '--output', summaryDirectory], + 'Benchmark suite summary failed.', + ); + + process.stdout.write( + `${JSON.stringify({ + contractVersion: 1, + documentProfile: args.documentProfile, + samples: 'samples.json', + status: 'completed', + summaryJson: 'summary/summary.json', + summaryText: 'summary/summary.txt', + })}\n`, + ); +} + +try { + main(process.argv.slice(2)); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Benchmark suite failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} From c41f1017bc64aa79f730ef35e8f33ea0dc6684a0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 07:07:11 -0700 Subject: [PATCH 100/209] test(perf): reject symlink benchmark output directories --- ...formanceSingleCommandSuiteContract.test.ts | 118 +++++++++++++----- 1 file changed, 87 insertions(+), 31 deletions(-) diff --git a/src/performanceSingleCommandSuiteContract.test.ts b/src/performanceSingleCommandSuiteContract.test.ts index 4747d273..611fb419 100644 --- a/src/performanceSingleCommandSuiteContract.test.ts +++ b/src/performanceSingleCommandSuiteContract.test.ts @@ -1,9 +1,12 @@ import { createHash } from 'node:crypto'; -import { execFileSync } from 'node:child_process'; +import { execFileSync, spawnSync } from 'node:child_process'; import { + existsSync, + mkdirSync, mkdtempSync, readFileSync, rmSync, + symlinkSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -21,44 +24,64 @@ afterEach(() => { } }); +function benchmarkArguments( + inputPath: string, + modulePath: string, + artifactSha256: string, + outputDirectory: string, +): string[] { + return [ + suitePath, + '--input', + inputPath, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + artifactSha256, + '--runtime-id', + 'node-22.0.0', + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + outputDirectory, + ]; +} + +function writeBenchmarkInputs(directory: string): { + artifactSha256: string; + inputPath: string; + modulePath: string; +} { + const inputPath = join(directory, 'input.md'); + const modulePath = join(directory, 'measured.mjs'); + const moduleSource = + "export function markdownToHtml(source) { return `

${source}

`; }\n"; + writeFileSync(inputPath, '# Buyer benchmark\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + return { + artifactSha256: createHash('sha256').update(moduleSource).digest('hex'), + inputPath, + modulePath, + }; +} + describe('single-command benchmark suite contract', () => { it('measures and summarizes one deterministic Markdown profile with one command', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); temporaryDirectories.push(directory); - const inputPath = join(directory, 'input.md'); - const modulePath = join(directory, 'measured.mjs'); const outputDirectory = join(directory, 'evidence'); - const moduleSource = - "export function markdownToHtml(source) { return `

${source}

`; }\n"; - writeFileSync(inputPath, '# Buyer benchmark\n', 'utf8'); - writeFileSync(modulePath, moduleSource, 'utf8'); - const artifactSha256 = createHash('sha256') - .update(moduleSource) - .digest('hex'); + const { artifactSha256, inputPath, modulePath } = + writeBenchmarkInputs(directory); const output = execFileSync( process.execPath, - [ - suitePath, - '--input', - inputPath, - '--module', - modulePath, - '--profile', - 'small', - '--samples', - '2', - '--source-commit-sha', - 'a'.repeat(40), - '--artifact-sha256', - artifactSha256, - '--runtime-id', - 'node-22.0.0', - '--reference-hardware-id', - `refhw-sha256-${'b'.repeat(64)}`, - '--output', - outputDirectory, - ], + benchmarkArguments(inputPath, modulePath, artifactSha256, outputDirectory), { cwd: repositoryRoot, encoding: 'utf8', @@ -92,4 +115,37 @@ describe('single-command benchmark suite contract', () => { readFileSync(join(outputDirectory, 'summary', 'summary.txt'), 'utf8'), ).toContain('markdown-serialization-small'); }); + + it('fails closed before writing evidence through a symlink output directory', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); + temporaryDirectories.push(directory); + const { artifactSha256, inputPath, modulePath } = + writeBenchmarkInputs(directory); + const actualOutputDirectory = join(directory, 'outside-target'); + const outputDirectory = join(directory, 'evidence-link'); + mkdirSync(actualOutputDirectory); + symlinkSync( + actualOutputDirectory, + outputDirectory, + process.platform === 'win32' ? 'junction' : 'dir', + ); + + const result = spawnSync( + process.execPath, + benchmarkArguments(inputPath, modulePath, artifactSha256, outputDirectory), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).not.toBe(0); + expect(result.stderr).toBe( + 'Benchmark suite output directory must be a non-symlink directory.\n', + ); + expect(existsSync(join(actualOutputDirectory, 'samples.json'))).toBe(false); + expect(existsSync(join(actualOutputDirectory, 'summary'))).toBe(false); + }); }); From 18fe7381a94a8c03b2897ae38198435617f8236e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 07:11:20 -0700 Subject: [PATCH 101/209] fix(perf): reject symlink suite output roots --- benchmarks/run-current-suite.mjs | 37 ++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 95354bdf..304cc2bc 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -1,4 +1,5 @@ import { spawnSync } from 'node:child_process'; +import { lstatSync, mkdirSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -15,6 +16,8 @@ const expectedFlags = Object.freeze([ '--reference-hardware-id', '--output', ]); +const OUTPUT_DIRECTORY_ERROR = + 'Benchmark suite output directory must be a non-symlink directory.'; function resolveArguments(argv) { if ( @@ -34,6 +37,39 @@ function resolveArguments(argv) { }); } +function inspectOutputDirectory(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark suite output directory could not be inspected.'); + } +} + +function prepareOutputDirectory(path) { + const existing = inspectOutputDirectory(path); + if (existing !== undefined) { + if (existing.isSymbolicLink() || !existing.isDirectory()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + return; + } + + try { + mkdirSync(path, { recursive: true }); + } catch { + throw new Error('Benchmark suite output directory could not be prepared.'); + } + + const created = inspectOutputDirectory(path); + if ( + created === undefined || + created.isSymbolicLink() || + !created.isDirectory() + ) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } +} + function runBoundedNodeScript(scriptName, args, failureMessage) { const result = spawnSync( process.execPath, @@ -58,6 +94,7 @@ function runBoundedNodeScript(scriptName, args, failureMessage) { function main(argv) { const args = resolveArguments(argv); + prepareOutputDirectory(args.outputDirectory); const samplesPath = resolve(args.outputDirectory, 'samples.json'); const summaryDirectory = resolve(args.outputDirectory, 'summary'); From 156b8e3463e1cef190e139cc39dda63c0e31997c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 07:22:14 -0700 Subject: [PATCH 102/209] test(perf): reject symlink summary output directory --- ...manceMeasurementStatisticsContract.test.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/performanceMeasurementStatisticsContract.test.ts b/src/performanceMeasurementStatisticsContract.test.ts index 3c477543..7f7627b3 100644 --- a/src/performanceMeasurementStatisticsContract.test.ts +++ b/src/performanceMeasurementStatisticsContract.test.ts @@ -6,6 +6,7 @@ import { mkdtempSync, readFileSync, rmSync, + symlinkSync, truncateSync, writeFileSync, } from 'node:fs'; @@ -290,4 +291,35 @@ describe('deterministic benchmark sample statistics', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('fails closed before writing summaries through a symlink output directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-symlink-')); + const input = join(root, 'samples.json'); + const target = join(root, 'outside-target'); + const output = join(root, 'output-link'); + try { + writeInput(input, [10, 20, 30]); + mkdirSync(target); + symlinkSync(target, output, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + expect(existsSync(join(target, 'summary.json'))).toBe(false); + expect(existsSync(join(target, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From 52ba7337829822b75569c6e8cc718c92764738a9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 07:56:38 -0700 Subject: [PATCH 103/209] fix(perf): reject symlink summary directories --- benchmarks/summarize-samples.mjs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 65592796..f93be873 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -254,11 +254,32 @@ function assertRegularOutputDestination(path) { } function prepareOutputDirectory(path) { + const current = lstatSync(path, { throwIfNoEntry: false }); + if (current !== undefined) { + if (current.isSymbolicLink() || !current.isDirectory()) { + throw new Error( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + } + return; + } + try { mkdirSync(path, { recursive: true }); } catch { throw new Error('Benchmark summary output directory could not be prepared.'); } + + const created = lstatSync(path, { throwIfNoEntry: false }); + if ( + created === undefined || + created.isSymbolicLink() || + !created.isDirectory() + ) { + throw new Error( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + } } function main() { From bd3eb04ff6c9cc7a1c2d0943a9cd643656aefcd4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 07:58:54 -0700 Subject: [PATCH 104/209] test(perf): reject symlinked summary ancestors --- ...asurementStatisticsAncestorSymlink.test.ts | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 src/performanceMeasurementStatisticsAncestorSymlink.test.ts diff --git a/src/performanceMeasurementStatisticsAncestorSymlink.test.ts b/src/performanceMeasurementStatisticsAncestorSymlink.test.ts new file mode 100644 index 00000000..aae380be --- /dev/null +++ b/src/performanceMeasurementStatisticsAncestorSymlink.test.ts @@ -0,0 +1,66 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +function writeInput(path: string): void { + writeFileSync( + path, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256: 'b'.repeat(64), + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [10, 20, 30], + })}\n`, + 'utf8', + ); +} + +describe('benchmark summary output path ancestry', () => { + it('fails closed before writing through a symlinked output ancestor', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-ancestor-')); + const input = join(root, 'samples.json'); + const outside = join(root, 'outside-target'); + const alias = join(root, 'aliased-parent'); + const output = join(alias, 'nested-output'); + try { + writeInput(input); + mkdirSync(outside); + symlinkSync(outside, alias, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + expect(existsSync(join(outside, 'nested-output', 'summary.json'))).toBe(false); + expect(existsSync(join(outside, 'nested-output', 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 3b722e44f248a406160fd09dbb3f41d3f8f094b7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 07:59:41 -0700 Subject: [PATCH 105/209] fix(perf): reject symlinked summary ancestors --- benchmarks/summarize-samples.mjs | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index f93be873..da952e53 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -10,7 +10,7 @@ import { statSync, writeFileSync, } from 'node:fs'; -import { resolve } from 'node:path'; +import { dirname, resolve } from 'node:path'; const MAX_INPUT_BYTES = 16 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; @@ -253,10 +253,26 @@ function assertRegularOutputDestination(path) { } } +function assertNoSymlinkDirectoryComponents(path) { + let current = path; + while (true) { + const metadata = lstatSync(current, { throwIfNoEntry: false }); + if (metadata?.isSymbolicLink()) { + throw new Error( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + function prepareOutputDirectory(path) { + assertNoSymlinkDirectoryComponents(path); const current = lstatSync(path, { throwIfNoEntry: false }); if (current !== undefined) { - if (current.isSymbolicLink() || !current.isDirectory()) { + if (!current.isDirectory()) { throw new Error( 'Benchmark summary output directory must be a non-symlink directory.', ); @@ -270,12 +286,9 @@ function prepareOutputDirectory(path) { throw new Error('Benchmark summary output directory could not be prepared.'); } + assertNoSymlinkDirectoryComponents(path); const created = lstatSync(path, { throwIfNoEntry: false }); - if ( - created === undefined || - created.isSymbolicLink() || - !created.isDirectory() - ) { + if (created === undefined || !created.isDirectory()) { throw new Error( 'Benchmark summary output directory must be a non-symlink directory.', ); From eb15b2f1a35fbd9a9eb61ad8332b19024e531980 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:00:43 -0700 Subject: [PATCH 106/209] test(perf): reject symlinked suite ancestors --- ...nceMeasurementSuiteAncestorSymlink.test.ts | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 src/performanceMeasurementSuiteAncestorSymlink.test.ts diff --git a/src/performanceMeasurementSuiteAncestorSymlink.test.ts b/src/performanceMeasurementSuiteAncestorSymlink.test.ts new file mode 100644 index 00000000..7b61c4b5 --- /dev/null +++ b/src/performanceMeasurementSuiteAncestorSymlink.test.ts @@ -0,0 +1,64 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + symlinkSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/run-current-suite.mjs'); + +describe('benchmark suite output path ancestry', () => { + it('fails closed before preparing an output beneath a symlinked ancestor', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-ancestor-')); + const outside = join(root, 'outside-target'); + const alias = join(root, 'aliased-parent'); + const output = join(alias, 'nested-output'); + try { + mkdirSync(outside); + symlinkSync(outside, alias, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + join(root, 'unused.md'), + '--module', + join(root, 'unused.mjs'), + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'b'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark suite output directory must be a non-symlink directory.', + ); + expect(existsSync(join(outside, 'nested-output'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 0cfd5a9e86ab429e4ef3b6a078af7d3fa09c5a13 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:01:11 -0700 Subject: [PATCH 107/209] fix(perf): reject symlinked suite ancestors --- benchmarks/run-current-suite.mjs | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 304cc2bc..5d9471b2 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -45,10 +45,24 @@ function inspectOutputDirectory(path) { } } +function assertNoSymlinkDirectoryComponents(path) { + let current = path; + while (true) { + const metadata = inspectOutputDirectory(current); + if (metadata?.isSymbolicLink()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + function prepareOutputDirectory(path) { + assertNoSymlinkDirectoryComponents(path); const existing = inspectOutputDirectory(path); if (existing !== undefined) { - if (existing.isSymbolicLink() || !existing.isDirectory()) { + if (!existing.isDirectory()) { throw new Error(OUTPUT_DIRECTORY_ERROR); } return; @@ -60,12 +74,9 @@ function prepareOutputDirectory(path) { throw new Error('Benchmark suite output directory could not be prepared.'); } + assertNoSymlinkDirectoryComponents(path); const created = inspectOutputDirectory(path); - if ( - created === undefined || - created.isSymbolicLink() || - !created.isDirectory() - ) { + if (created === undefined || !created.isDirectory()) { throw new Error(OUTPUT_DIRECTORY_ERROR); } } From f1adf747086fa4c305f82af349599257e6c2d174 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:04:04 -0700 Subject: [PATCH 108/209] test(perf): reject symlinked markdown output ancestors --- ...wnMeasurementOutputAncestorSymlink.test.ts | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 src/performanceMarkdownMeasurementOutputAncestorSymlink.test.ts diff --git a/src/performanceMarkdownMeasurementOutputAncestorSymlink.test.ts b/src/performanceMarkdownMeasurementOutputAncestorSymlink.test.ts new file mode 100644 index 00000000..cc60c3cb --- /dev/null +++ b/src/performanceMarkdownMeasurementOutputAncestorSymlink.test.ts @@ -0,0 +1,76 @@ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); + +function sha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +describe('Markdown benchmark output path ancestry', () => { + it('fails closed before writing beneath a symlinked output ancestor', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-output-ancestor-')); + const input = join(root, 'document.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const outside = join(root, 'outside-target'); + const alias = join(root, 'aliased-parent'); + const output = join(alias, 'nested-output', 'samples.json'); + try { + writeFileSync(input, '# Synthetic\n', 'utf8'); + writeFileSync( + modulePath, + 'export function markdownToHtml(source) { return `

${source.length}

`; }\n', + 'utf8', + ); + mkdirSync(outside); + symlinkSync(outside, alias, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + sha256(modulePath), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark output directory must be a non-symlink directory.', + ); + expect(existsSync(join(outside, 'nested-output', 'samples.json'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 0ac8712eea1366c6582b2f6b1da7e963295db935 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:05:01 -0700 Subject: [PATCH 109/209] fix(perf): reject symlinked markdown output ancestors --- benchmarks/measure-markdown.mjs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index e240af3c..eb1b19b7 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -28,6 +28,8 @@ const RUNTIME_ID_PATTERN = /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; const REFERENCE_HARDWARE_ID_PATTERN = /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; +const OUTPUT_DIRECTORY_ERROR = + 'Markdown benchmark output directory must be a non-symlink directory.'; function resolveArguments(argv) { const expectedFlags = [ @@ -223,6 +225,27 @@ function inspectOutputPath(path) { } } +function inspectOutputDirectoryComponent(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Markdown benchmark output directory could not be inspected.'); + } +} + +function assertNoSymlinkOutputAncestors(path) { + let current = dirname(path); + while (true) { + const metadata = inspectOutputDirectoryComponent(current); + if (metadata?.isSymbolicLink()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + function refersToSameFile(leftPath, rightPath) { const rightMetadata = inspectOutputPath(rightPath); if (rightMetadata === undefined) return false; @@ -251,8 +274,14 @@ function runMeasuredMarkdownToHtml(markdownToHtml, source) { } function writeMeasurementOutput(path, content) { + assertNoSymlinkOutputAncestors(path); try { mkdirSync(dirname(path), { recursive: true }); + } catch { + throw new Error('Markdown benchmark output could not be written.'); + } + assertNoSymlinkOutputAncestors(path); + try { writeFileSync(path, content, 'utf8'); } catch { throw new Error('Markdown benchmark output could not be written.'); @@ -261,6 +290,7 @@ function writeMeasurementOutput(path, content) { async function main() { const args = resolveArguments(process.argv.slice(2)); + assertNoSymlinkOutputAncestors(args.outputPath); if ( args.inputPath === args.outputPath || refersToSameFile(args.inputPath, args.outputPath) @@ -311,6 +341,7 @@ async function main() { } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertNoSymlinkOutputAncestors(args.outputPath); const outputMetadata = inspectOutputPath(args.outputPath); if (outputMetadata !== undefined && !outputMetadata.isFile()) { throw new Error('Markdown benchmark output must be a regular file.'); From 2bcf06c9872aa497f6d1692928c64a790749de27 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:05:20 -0700 Subject: [PATCH 110/209] test(perf): reject symlinked revision output ancestors --- ...onMeasurementOutputAncestorSymlink.test.ts | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 src/performanceRevisionMeasurementOutputAncestorSymlink.test.ts diff --git a/src/performanceRevisionMeasurementOutputAncestorSymlink.test.ts b/src/performanceRevisionMeasurementOutputAncestorSymlink.test.ts new file mode 100644 index 00000000..48348e12 --- /dev/null +++ b/src/performanceRevisionMeasurementOutputAncestorSymlink.test.ts @@ -0,0 +1,84 @@ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-revision-evidence.mjs'); + +function sha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +describe('revision benchmark output path ancestry', () => { + it('fails closed before writing beneath a symlinked output ancestor', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-output-ancestor-')); + const input = join(root, 'document-envelope.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const outside = join(root, 'outside-target'); + const alias = join(root, 'aliased-parent'); + const output = join(alias, 'nested-output', 'samples.json'); + try { + writeFileSync( + input, + JSON.stringify({ + schemaId: 'https://inkspan.io/schemas/document-envelope/v1', + schemaVersion: 1, + documentJson: { type: 'doc', content: [] }, + }), + 'utf8', + ); + writeFileSync( + modulePath, + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) { return { revision: { digestHex: String(source.byteLength).padStart(64, "0") } }; }\n', + 'utf8', + ); + mkdirSync(outside); + symlinkSync(outside, alias, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + sha256(modulePath), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Revision benchmark output directory must be a non-symlink directory.', + ); + expect(existsSync(join(outside, 'nested-output', 'samples.json'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 2e9332e8bb350c9cfb2596a131df6b4878e350eb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:05:56 -0700 Subject: [PATCH 111/209] fix(perf): reject symlinked revision output ancestors --- benchmarks/measure-revision-evidence.mjs | 31 ++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index 2bec7367..3c413590 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -27,6 +27,8 @@ const RUNTIME_ID_PATTERN = /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; const REFERENCE_HARDWARE_ID_PATTERN = /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; +const OUTPUT_DIRECTORY_ERROR = + 'Revision benchmark output directory must be a non-symlink directory.'; function resolveArguments(argv) { const expectedFlags = [ @@ -215,6 +217,27 @@ function inspectOutputPath(path) { } } +function inspectOutputDirectoryComponent(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Revision benchmark output directory could not be inspected.'); + } +} + +function assertNoSymlinkOutputAncestors(path) { + let current = dirname(path); + while (true) { + const metadata = inspectOutputDirectoryComponent(current); + if (metadata?.isSymbolicLink()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + function refersToSameFile(leftPath, rightPath) { const rightMetadata = inspectOutputPath(rightPath); if (rightMetadata === undefined) return false; @@ -235,8 +258,14 @@ async function loadMeasuredModule(modulePath) { } function writeMeasurementOutput(path, content) { + assertNoSymlinkOutputAncestors(path); try { mkdirSync(dirname(path), { recursive: true }); + } catch { + throw new Error('Revision benchmark output could not be written.'); + } + assertNoSymlinkOutputAncestors(path); + try { writeFileSync(path, content, 'utf8'); } catch { throw new Error('Revision benchmark output could not be written.'); @@ -272,6 +301,7 @@ async function runMeasuredRevision(createRevisionEvidence, source) { async function main() { const args = resolveArguments(process.argv.slice(2)); + assertNoSymlinkOutputAncestors(args.outputPath); if ( args.inputPath === args.outputPath || refersToSameFile(args.inputPath, args.outputPath) @@ -320,6 +350,7 @@ async function main() { } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertNoSymlinkOutputAncestors(args.outputPath); const outputMetadata = inspectOutputPath(args.outputPath); if (outputMetadata !== undefined && !outputMetadata.isFile()) { throw new Error('Revision benchmark output must be a regular file.'); From ea00fe920a82601ab01cabca43c8acf00354b2eb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:09:44 -0700 Subject: [PATCH 112/209] fix(perf): preserve path-redacted summary failures --- benchmarks/summarize-samples.mjs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index da952e53..369a5c45 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -253,10 +253,18 @@ function assertRegularOutputDestination(path) { } } +function inspectOutputDirectoryComponent(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark summary output directory could not be prepared.'); + } +} + function assertNoSymlinkDirectoryComponents(path) { let current = path; while (true) { - const metadata = lstatSync(current, { throwIfNoEntry: false }); + const metadata = inspectOutputDirectoryComponent(current); if (metadata?.isSymbolicLink()) { throw new Error( 'Benchmark summary output directory must be a non-symlink directory.', @@ -270,7 +278,7 @@ function assertNoSymlinkDirectoryComponents(path) { function prepareOutputDirectory(path) { assertNoSymlinkDirectoryComponents(path); - const current = lstatSync(path, { throwIfNoEntry: false }); + const current = inspectOutputDirectoryComponent(path); if (current !== undefined) { if (!current.isDirectory()) { throw new Error( @@ -287,7 +295,7 @@ function prepareOutputDirectory(path) { } assertNoSymlinkDirectoryComponents(path); - const created = lstatSync(path, { throwIfNoEntry: false }); + const created = inspectOutputDirectoryComponent(path); if (created === undefined || !created.isDirectory()) { throw new Error( 'Benchmark summary output directory must be a non-symlink directory.', From dba75050566ec2867afa15328352f0e68e057588 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:17:35 -0700 Subject: [PATCH 113/209] test(perf): reject symlinked sample input --- ...eMeasurementStatisticsInputSymlink.test.ts | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 src/performanceMeasurementStatisticsInputSymlink.test.ts diff --git a/src/performanceMeasurementStatisticsInputSymlink.test.ts b/src/performanceMeasurementStatisticsInputSymlink.test.ts new file mode 100644 index 00000000..0c481d53 --- /dev/null +++ b/src/performanceMeasurementStatisticsInputSymlink.test.ts @@ -0,0 +1,53 @@ +import { spawnSync } from 'node:child_process'; +import { + mkdtempSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +function validSamples(): string { + return `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256: 'b'.repeat(64), + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [10, 20, 30], + })}\n`; +} + +describe('benchmark summary sample input file authority', () => { + it('fails closed instead of following a symlinked sample input', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-summary-input-symlink-')); + const target = join(root, 'private-samples.json'); + const alias = join(root, 'samples.json'); + const output = join(root, 'summary'); + try { + writeFileSync(target, validSamples(), 'utf8'); + symlinkSync(target, alias, process.platform === 'win32' ? 'file' : undefined); + + const result = spawnSync( + process.execPath, + [script, '--input', alias, '--output', output], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input must be a regular non-symlink file.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 9afff917c557173af9f3defccde37b0f8c284b48 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:34:25 -0700 Subject: [PATCH 114/209] fix(perf): reject symlinked summary sample input --- benchmarks/summarize-samples.mjs | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 369a5c45..2a99eba8 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -15,8 +15,10 @@ import { dirname, resolve } from 'node:path'; const MAX_INPUT_BYTES = 16 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const MAX_SAMPLES = 1_000_000; -const READ_ONLY_NONBLOCKING = - constants.O_RDONLY | (constants.O_NONBLOCK ?? 0); +const READ_ONLY_NONBLOCKING_NOFOLLOW = + constants.O_RDONLY | + (constants.O_NONBLOCK ?? 0) | + (constants.O_NOFOLLOW ?? 0); const BENCHMARK_ID_PATTERN = /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; const UNITS = new Set(['ms', 'bytes']); @@ -59,11 +61,24 @@ function resolveArguments(argv) { } function readBoundedJson(path) { - const descriptor = openSync(path, READ_ONLY_NONBLOCKING); + const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error( + 'Benchmark sample input must be a regular non-symlink file.', + ); + } + + const descriptor = openSync(path, READ_ONLY_NONBLOCKING_NOFOLLOW); try { const metadata = fstatSync(descriptor); if (!metadata.isFile()) { - throw new Error('Benchmark sample input must be a regular file.'); + throw new Error( + 'Benchmark sample input must be a regular non-symlink file.', + ); } if (metadata.size > MAX_INPUT_BYTES) { throw new Error('Benchmark sample input exceeds the supported size.'); From 824561c84a398cd6c396f2c1cdb52f3b7823e5cf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 08:39:54 -0700 Subject: [PATCH 115/209] fix(perf): preserve nonregular input diagnostics --- benchmarks/summarize-samples.mjs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 2a99eba8..c2ff872e 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -62,23 +62,20 @@ function resolveArguments(argv) { function readBoundedJson(path) { const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); - if ( - pathMetadata === undefined || - pathMetadata.isSymbolicLink() || - !pathMetadata.isFile() - ) { + if (pathMetadata?.isSymbolicLink()) { throw new Error( 'Benchmark sample input must be a regular non-symlink file.', ); } + if (pathMetadata === undefined || !pathMetadata.isFile()) { + throw new Error('Benchmark sample input must be a regular file.'); + } const descriptor = openSync(path, READ_ONLY_NONBLOCKING_NOFOLLOW); try { const metadata = fstatSync(descriptor); if (!metadata.isFile()) { - throw new Error( - 'Benchmark sample input must be a regular non-symlink file.', - ); + throw new Error('Benchmark sample input must be a regular file.'); } if (metadata.size > MAX_INPUT_BYTES) { throw new Error('Benchmark sample input exceeds the supported size.'); From a7b6bc9dcea44142395832c8a1bb541f9b89aac2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 09:08:16 -0700 Subject: [PATCH 116/209] test(perf): cover markdown input path privacy --- ...ownMeasurementInputPrivacyContract.test.ts | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/performanceMarkdownMeasurementInputPrivacyContract.test.ts diff --git a/src/performanceMarkdownMeasurementInputPrivacyContract.test.ts b/src/performanceMarkdownMeasurementInputPrivacyContract.test.ts new file mode 100644 index 00000000..748e94a1 --- /dev/null +++ b/src/performanceMarkdownMeasurementInputPrivacyContract.test.ts @@ -0,0 +1,72 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const RUNTIME_ID = 'node-22.18.0'; +const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function sha256(value: string) { + return createHash('sha256').update(value, 'utf8').digest('hex'); +} + +describe('Markdown measurement input error privacy contract', () => { + it('redacts private filesystem details when input traversal fails', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-input-privacy-')); + const privateSentinel = 'private-input-sentinel-must-not-leak'; + const blockedParent = join(root, privateSentinel); + const input = join(blockedParent, 'document.md'); + const modulePath = join(root, 'measured.mjs'); + const output = join(root, 'samples.json'); + const moduleSource = + 'export function markdownToHtml(value) { return value; }\n'; + + writeFileSync(blockedParent, 'not a directory', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + REFERENCE_HARDWARE_ID, + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark input must be a regular non-symlink file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From dd35f070ba5d8852d5eb9de081d071d0ccb69bbf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 09:13:07 -0700 Subject: [PATCH 117/209] fix(perf): redact markdown input path errors --- benchmarks/measure-markdown.mjs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index eb1b19b7..815630f7 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -103,7 +103,12 @@ function resolveArguments(argv) { } function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversizedMessage) { - const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error(invalidFileMessage); + } if ( pathMetadata === undefined || pathMetadata.isSymbolicLink() || @@ -112,7 +117,12 @@ function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversize throw new Error(invalidFileMessage); } - const descriptor = openSync(path, READ_ONLY_NOFOLLOW); + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error(invalidFileMessage); + } try { const metadata = fstatSync(descriptor); if (!metadata.isFile()) { From 1781400353a49d20634458093ce1dc3e24735973 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 09:19:09 -0700 Subject: [PATCH 118/209] test(perf): cover markdown module path privacy --- ...asurementModulePathPrivacyContract.test.ts | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 src/performanceMarkdownMeasurementModulePathPrivacyContract.test.ts diff --git a/src/performanceMarkdownMeasurementModulePathPrivacyContract.test.ts b/src/performanceMarkdownMeasurementModulePathPrivacyContract.test.ts new file mode 100644 index 00000000..352e7f4d --- /dev/null +++ b/src/performanceMarkdownMeasurementModulePathPrivacyContract.test.ts @@ -0,0 +1,65 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const RUNTIME_ID = 'node-22.18.0'; +const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +describe('Markdown measurement module path error privacy contract', () => { + it('redacts private filesystem details when module path traversal fails', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-module-privacy-')); + const privateSentinel = 'private-module-sentinel-must-not-leak'; + const input = join(root, 'document.md'); + const blockedParent = join(root, privateSentinel); + const modulePath = join(blockedParent, 'measured.mjs'); + const output = join(root, 'samples.json'); + + writeFileSync(input, '# bounded\n', 'utf8'); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + '0'.repeat(64), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + REFERENCE_HARDWARE_ID, + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module must be a local regular file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 17bb601de3b1ff85b6159d3f88398f24fe4c1fd0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 09:25:07 -0700 Subject: [PATCH 119/209] fix(perf): redact markdown module path errors --- benchmarks/measure-markdown.mjs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 815630f7..e3458566 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -198,7 +198,12 @@ function resolveLocalModule(pathOrUrl) { } catch { throw new Error('Measured Markdown module must be a local regular file.'); } - const metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); + let metadata; + try { + metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); + } catch { + throw new Error('Measured Markdown module must be a local regular file.'); + } if ( metadata === undefined || metadata.isSymbolicLink() || @@ -206,7 +211,11 @@ function resolveLocalModule(pathOrUrl) { ) { throw new Error('Measured Markdown module must be a local regular file.'); } - return realpathSync(resolvedPath); + try { + return realpathSync(resolvedPath); + } catch { + throw new Error('Measured Markdown module must be a local regular file.'); + } } function measuredModuleSha256(modulePath) { From 2d22587fde08312dd2032e50252672f117ea25cb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 11:05:59 -0700 Subject: [PATCH 120/209] test(perf): reproduce revision module path leak --- ...asurementModulePathPrivacyContract.test.ts | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 src/performanceRevisionMeasurementModulePathPrivacyContract.test.ts diff --git a/src/performanceRevisionMeasurementModulePathPrivacyContract.test.ts b/src/performanceRevisionMeasurementModulePathPrivacyContract.test.ts new file mode 100644 index 00000000..814f35c6 --- /dev/null +++ b/src/performanceRevisionMeasurementModulePathPrivacyContract.test.ts @@ -0,0 +1,82 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-revision-evidence.mjs', +); + +function writeSyntheticEnvelope(path: string): void { + writeFileSync( + path, + JSON.stringify({ + schemaId: 'https://inkspan.io/schemas/document-envelope/v1', + schemaVersion: 1, + documentJson: { + type: 'doc', + content: [ + { + type: 'paragraph', + content: [{ type: 'text', text: 'Synthetic benchmark content' }], + }, + ], + }, + }), + 'utf8', + ); +} + +describe('revision benchmark module-path privacy contract', () => { + it('redacts filesystem details when module-path resolution crosses a non-directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-module-path-')); + const input = join(root, 'small.json'); + const privateSentinel = 'tenant-private-revision-module-parent'; + const blockedParent = join(root, privateSentinel); + const modulePath = join(blockedParent, 'packed-revision-evidence.mjs'); + const output = join(root, 'samples.json'); + + try { + writeSyntheticEnvelope(input); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'b'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured revision module must be a local regular file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 054781216ca33d227b2a681a527e513789fed5b6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 11:08:36 -0700 Subject: [PATCH 121/209] fix(perf): redact revision module path errors --- benchmarks/measure-revision-evidence.mjs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index 3c413590..e2ed197b 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -180,7 +180,12 @@ function resolveLocalModule(pathOrUrl) { } catch { throw new Error('Measured revision module must be a local regular file.'); } - const metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); + let metadata; + try { + metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); + } catch { + throw new Error('Measured revision module must be a local regular file.'); + } if ( metadata === undefined || metadata.isSymbolicLink() || @@ -188,7 +193,11 @@ function resolveLocalModule(pathOrUrl) { ) { throw new Error('Measured revision module must be a local regular file.'); } - return realpathSync(resolvedPath); + try { + return realpathSync(resolvedPath); + } catch { + throw new Error('Measured revision module must be a local regular file.'); + } } function measuredModuleSha256(modulePath) { From cecef7d4f042c8f5a6b32e8f5aa5bc1a6657a832 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 11:09:35 -0700 Subject: [PATCH 122/209] test(perf): reproduce revision input path leak --- ...easurementInputPathPrivacyContract.test.ts | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 src/performanceRevisionMeasurementInputPathPrivacyContract.test.ts diff --git a/src/performanceRevisionMeasurementInputPathPrivacyContract.test.ts b/src/performanceRevisionMeasurementInputPathPrivacyContract.test.ts new file mode 100644 index 00000000..fee61353 --- /dev/null +++ b/src/performanceRevisionMeasurementInputPathPrivacyContract.test.ts @@ -0,0 +1,60 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-revision-evidence.mjs', +); + +describe('revision benchmark input-path privacy contract', () => { + it('redacts filesystem details when input inspection crosses a non-directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-input-path-')); + const privateSentinel = 'tenant-private-revision-input-parent'; + const blockedParent = join(root, privateSentinel); + const input = join(blockedParent, 'document-envelope.json'); + const output = join(root, 'samples.json'); + + try { + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + join(root, 'unused-module.mjs'), + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'b'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Revision benchmark input must be a regular non-symlink file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From f412c50bc5d3ee95f236fc5870cb1543e2cd7bb4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 11:10:33 -0700 Subject: [PATCH 123/209] fix(perf): redact revision file-open path errors --- benchmarks/measure-revision-evidence.mjs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index e2ed197b..99286baa 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -102,7 +102,12 @@ function resolveArguments(argv) { } function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversizedMessage) { - const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error(invalidFileMessage); + } if ( pathMetadata === undefined || pathMetadata.isSymbolicLink() || @@ -111,7 +116,12 @@ function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversize throw new Error(invalidFileMessage); } - const descriptor = openSync(path, READ_ONLY_NOFOLLOW); + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error(invalidFileMessage); + } try { const metadata = fstatSync(descriptor); if (!metadata.isFile()) { From df1ac908bc04dfa6b5e290587b9f15c5ddce832a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 11:12:46 -0700 Subject: [PATCH 124/209] test(perf): reproduce existing-output alias path leaks --- ...kExistingOutputPathPrivacyContract.test.ts | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 src/performanceBenchmarkExistingOutputPathPrivacyContract.test.ts diff --git a/src/performanceBenchmarkExistingOutputPathPrivacyContract.test.ts b/src/performanceBenchmarkExistingOutputPathPrivacyContract.test.ts new file mode 100644 index 00000000..0555b3db --- /dev/null +++ b/src/performanceBenchmarkExistingOutputPathPrivacyContract.test.ts @@ -0,0 +1,69 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +const cases = [ + { + name: 'Markdown', + script: 'benchmarks/measure-markdown.mjs', + expectedError: 'Markdown benchmark input must be a regular non-symlink file.', + }, + { + name: 'revision', + script: 'benchmarks/measure-revision-evidence.mjs', + expectedError: 'Revision benchmark input must be a regular non-symlink file.', + }, +] as const; + +describe('benchmark existing-output path privacy contract', () => { + for (const testCase of cases) { + it(`redacts ${testCase.name} input paths before existing-output alias checks`, () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-alias-path-')); + const privateSentinel = `tenant-private-${testCase.name.toLowerCase()}-input-parent`; + const blockedParent = join(root, privateSentinel); + const input = join(blockedParent, 'document-input'); + const output = join(root, 'samples.json'); + + try { + writeFileSync(blockedParent, 'not a directory', 'utf8'); + writeFileSync(output, '{}\n', 'utf8'); + + const result = spawnSync( + process.execPath, + [ + resolve(process.cwd(), testCase.script), + '--input', + input, + '--module', + join(root, 'unused-module.mjs'), + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'b'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe(testCase.expectedError); + expect(result.stderr).not.toContain(privateSentinel); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + } +}); From 9488aad6098bf85d72485e44a0c553f7dbadcdba Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 11:14:11 -0700 Subject: [PATCH 125/209] fix(perf): validate Markdown input before alias stat --- benchmarks/measure-markdown.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index e3458566..8d0549b5 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -310,13 +310,13 @@ function writeMeasurementOutput(path, content) { async function main() { const args = resolveArguments(process.argv.slice(2)); assertNoSymlinkOutputAncestors(args.outputPath); + const source = readBoundedMarkdown(args.inputPath); if ( args.inputPath === args.outputPath || refersToSameFile(args.inputPath, args.outputPath) ) { throw new Error('Markdown benchmark output must not overwrite its input.'); } - const source = readBoundedMarkdown(args.inputPath); const modulePath = resolveLocalModule(args.modulePath); if ( modulePath === args.outputPath || From d2a7a1f5bda57c808a5514f975a5a3045065da7d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 11:17:01 -0700 Subject: [PATCH 126/209] fix(perf): validate revision input before alias stat --- benchmarks/measure-revision-evidence.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index 99286baa..b0064e60 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -321,13 +321,13 @@ async function runMeasuredRevision(createRevisionEvidence, source) { async function main() { const args = resolveArguments(process.argv.slice(2)); assertNoSymlinkOutputAncestors(args.outputPath); + const source = readBoundedEnvelopeBytes(args.inputPath); if ( args.inputPath === args.outputPath || refersToSameFile(args.inputPath, args.outputPath) ) { throw new Error('Revision benchmark output must not overwrite its input.'); } - const source = readBoundedEnvelopeBytes(args.inputPath); const modulePath = resolveLocalModule(args.modulePath); if ( modulePath === args.outputPath || From 82155d37b8226fe0052dde3c87c0dfc7c8940053 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 11:25:04 -0700 Subject: [PATCH 127/209] test(perf): reproduce summary input path leak --- ...nceSummaryInputPathPrivacyContract.test.ts | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/performanceSummaryInputPathPrivacyContract.test.ts diff --git a/src/performanceSummaryInputPathPrivacyContract.test.ts b/src/performanceSummaryInputPathPrivacyContract.test.ts new file mode 100644 index 00000000..64ca6050 --- /dev/null +++ b/src/performanceSummaryInputPathPrivacyContract.test.ts @@ -0,0 +1,37 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +const summarizer = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +describe('benchmark summary input-path privacy contract', () => { + it('redacts filesystem details when input inspection crosses a non-directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-summary-input-path-')); + const privateSentinel = 'tenant-private-summary-input-parent'; + const blockedParent = join(root, privateSentinel); + const input = join(blockedParent, 'samples.json'); + const output = join(root, 'summary'); + + try { + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [summarizer, '--input', input, '--output', output], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input must be a regular file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 463f100d3cf7f3d24b560e3632decc75b9526d27 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 12:09:29 -0700 Subject: [PATCH 128/209] fix(perf): redact summary input inspection failures --- benchmarks/summarize-samples.mjs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index c2ff872e..9193d97e 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -60,8 +60,16 @@ function resolveArguments(argv) { }); } +function inspectSampleInputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark sample input must be a regular file.'); + } +} + function readBoundedJson(path) { - const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + const pathMetadata = inspectSampleInputPath(path); if (pathMetadata?.isSymbolicLink()) { throw new Error( 'Benchmark sample input must be a regular non-symlink file.', From f4400a6023f35f46744ef6d43b416ad9910db1a4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 12:12:00 -0700 Subject: [PATCH 129/209] test(perf): expose retained-memory input path leakage --- src/performanceMemorySettlingContract.test.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/performanceMemorySettlingContract.test.ts b/src/performanceMemorySettlingContract.test.ts index 93819962..63f26dce 100644 --- a/src/performanceMemorySettlingContract.test.ts +++ b/src/performanceMemorySettlingContract.test.ts @@ -133,6 +133,38 @@ describe('retained-memory settling evidence contract', () => { } }); + it('redacts an input path when a parent component is not a directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-path-privacy-')); + try { + const privateMarker = 'tenant-private-memory-evidence-parent'; + const parentPath = join(root, privateMarker); + writeFileSync(parentPath, 'not-a-directory', 'utf8'); + const inputPath = join(parentPath, 'memory-evidence.json'); + const result = spawnSync( + process.execPath, + [ + script, + '--input', + inputPath, + '--window-size', + '3', + '--max-growth-bytes', + '50', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Memory settling evidence input must be a regular file.', + ); + expect(result.stderr).not.toContain(privateMarker); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('rejects evidence whose benchmark profile disagrees with documentProfile', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-profile-')); try { From 5d1a62e9b376d8bba870efe03d3c1281c006c458 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 12:14:13 -0700 Subject: [PATCH 130/209] fix(perf): redact retained-memory input inspection failures --- benchmarks/analyze-memory-settling.mjs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/benchmarks/analyze-memory-settling.mjs b/benchmarks/analyze-memory-settling.mjs index 9e60c947..f15d3d83 100644 --- a/benchmarks/analyze-memory-settling.mjs +++ b/benchmarks/analyze-memory-settling.mjs @@ -73,8 +73,16 @@ function resolveArguments(argv) { }); } +function inspectEvidenceInputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Memory settling evidence input must be a regular file.'); + } +} + function readBoundedJson(path) { - const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + const pathMetadata = inspectEvidenceInputPath(path); if (pathMetadata === undefined || pathMetadata.isSymbolicLink()) { throw new Error( 'Memory settling evidence input must be a regular non-symlink file.', From 6a6ee594e94db76295ca1888648b761f740374c5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 12:16:13 -0700 Subject: [PATCH 131/209] test(perf): expose comparator input path leakage --- ...rmanceRegressionComparatorContract.test.ts | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts index dfeab91e..d51ae9f1 100644 --- a/src/performanceRegressionComparatorContract.test.ts +++ b/src/performanceRegressionComparatorContract.test.ts @@ -244,6 +244,47 @@ describe('benchmark regression comparator contract', () => { } }); + it('redacts a summary input path when a parent component is not a directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-path-privacy-')); + try { + const privateMarker = 'tenant-private-performance-baseline-parent'; + const privateParentPath = join(root, privateMarker); + const baselinePath = join(privateParentPath, 'baseline.json'); + const currentPath = join(root, 'current.json'); + writeFileSync(privateParentPath, 'not-a-directory', 'utf8'); + writeFileSync( + currentPath, + `${JSON.stringify(summary({ artifactSha256: CURRENT_ARTIFACT_SHA256 }))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary input must be a regular file.', + ); + expect(result.stderr).not.toContain(privateMarker); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('fails closed on a named-pipe summary instead of blocking before regular-file validation', () => { if (process.platform === 'win32') return; From 00bdb5b6a378f3582daf26f41f9652027b1f3b95 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 12:18:00 -0700 Subject: [PATCH 132/209] fix(perf): redact comparator input inspection failures --- benchmarks/compare-summaries.mjs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs index 4b891f14..a5c252da 100644 --- a/benchmarks/compare-summaries.mjs +++ b/benchmarks/compare-summaries.mjs @@ -86,8 +86,16 @@ function resolveArguments(argv) { }); } +function inspectSummaryInputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark summary input must be a regular file.'); + } +} + function readBoundedJson(path) { - const pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + const pathMetadata = inspectSummaryInputPath(path); if (pathMetadata === undefined || pathMetadata.isSymbolicLink()) { throw new Error( 'Benchmark summary input must be a regular non-symlink file.', From 45c6f41c15ddb90b674f31170bf402c29bcfedcb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 12:21:18 -0700 Subject: [PATCH 133/209] test(perf): expose Office fixture output symlink overwrite --- src/performanceOfficeFixtureContract.test.ts | 27 +++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/performanceOfficeFixtureContract.test.ts b/src/performanceOfficeFixtureContract.test.ts index ebc1fa2d..1dfea952 100644 --- a/src/performanceOfficeFixtureContract.test.ts +++ b/src/performanceOfficeFixtureContract.test.ts @@ -1,5 +1,12 @@ import { execFileSync } from 'node:child_process'; -import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; @@ -144,4 +151,22 @@ describe('deterministic synthetic Office performance fixtures', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('fails closed instead of overwriting a file through an Office fixture output symlink', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-symlink-')); + const outputDirectory = join(root, 'output'); + const victimPath = join(root, 'victim.json'); + try { + mkdirSync(outputDirectory, { recursive: true }); + writeFileSync(victimPath, 'buyer-owned evidence\n', 'utf8'); + symlinkSync(victimPath, join(outputDirectory, 'docx-small.json')); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readFileSync(victimPath, 'utf8')).toBe('buyer-owned evidence\n'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From e4d3478f1d4eb03954ac37c9a5d07802190f6bca Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 13:05:56 -0700 Subject: [PATCH 134/209] fix(perf): fail closed on Office fixture output symlinks --- benchmarks/generate-office-fixtures.mjs | 58 ++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/benchmarks/generate-office-fixtures.mjs b/benchmarks/generate-office-fixtures.mjs index 40a512df..d09d4d36 100644 --- a/benchmarks/generate-office-fixtures.mjs +++ b/benchmarks/generate-office-fixtures.mjs @@ -1,5 +1,13 @@ import { createHash } from 'node:crypto'; -import { mkdirSync, writeFileSync } from 'node:fs'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdirSync, + openSync, + writeFileSync, +} from 'node:fs'; import { resolve } from 'node:path'; const DOCX_PROFILE_PAGES = Object.freeze({ @@ -15,6 +23,12 @@ const PPTX_PROFILE_SLIDES = Object.freeze({ const EXCEL_MAX_COLUMNS = 16_384; const MULTILINGUAL_PARAGRAPH = 'English: deterministic Office rendering fixture. 한국어: 합성 성능 문서입니다. 日本語: 合成性能文書です。 中文: 这是合成性能文档。 Tiếng Việt: Đây là tài liệu hiệu năng tổng hợp.'; +const WRITE_NOFOLLOW = + constants.O_WRONLY | + constants.O_CREAT | + constants.O_TRUNC | + (constants.O_NONBLOCK ?? 0) | + (constants.O_NOFOLLOW ?? 0); function buildDocxPage(pageNumber) { const page = String(pageNumber).padStart(3, '0'); @@ -154,10 +168,45 @@ function resolveOutputDirectory(argv) { return resolve(argv[1]); } +function writeOutputFile(outputPath, bytes) { + let pathMetadata; + try { + pathMetadata = lstatSync(outputPath, { throwIfNoEntry: false }); + } catch { + throw new Error('Office fixture output path could not be inspected.'); + } + if (pathMetadata?.isSymbolicLink()) { + throw new Error('Office fixture output must be a regular non-symlink file.'); + } + if (pathMetadata !== undefined && !pathMetadata.isFile()) { + throw new Error('Office fixture output must be a regular file.'); + } + + let descriptor; + try { + descriptor = openSync(outputPath, WRITE_NOFOLLOW, 0o600); + if (!fstatSync(descriptor).isFile()) { + throw new Error('Office fixture output must be a regular file.'); + } + writeFileSync(descriptor, bytes); + } catch (error) { + if ( + error instanceof Error && + (error.message === 'Office fixture output must be a regular file.' || + error.message === 'Office fixture output must be a regular non-symlink file.') + ) { + throw error; + } + throw new Error('Office fixture output could not be written safely.'); + } finally { + if (descriptor !== undefined) closeSync(descriptor); + } +} + function writeFixture(outputDirectory, fileName, request, units) { const body = `${JSON.stringify(request, null, 2)}\n`; const bytes = Buffer.from(body, 'utf8'); - writeFileSync(resolve(outputDirectory, fileName), bytes); + writeOutputFile(resolve(outputDirectory, fileName), bytes); return Object.freeze({ units, bytes: bytes.byteLength, @@ -212,8 +261,7 @@ const manifest = Object.freeze({ pptx: Object.freeze(pptx), }), }); -writeFileSync( +writeOutputFile( resolve(outputDirectory, 'manifest.json'), - `${JSON.stringify(manifest, null, 2)}\n`, - 'utf8', + Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`, 'utf8'), ); From 5fe6023c927e8944327ca4452e87caa0c464644d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 15:06:29 -0700 Subject: [PATCH 135/209] test(perf): expose Office fixture output directory symlinks --- src/performanceOfficeFixtureContract.test.ts | 38 ++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/performanceOfficeFixtureContract.test.ts b/src/performanceOfficeFixtureContract.test.ts index 1dfea952..39abb092 100644 --- a/src/performanceOfficeFixtureContract.test.ts +++ b/src/performanceOfficeFixtureContract.test.ts @@ -3,6 +3,7 @@ import { mkdirSync, mkdtempSync, readFileSync, + readdirSync, rmSync, symlinkSync, writeFileSync, @@ -169,4 +170,41 @@ describe('deterministic synthetic Office performance fixtures', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('fails closed instead of publishing through a symlinked Office fixture output directory', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-output-dir-')); + const victimDirectory = join(root, 'buyer-owned'); + const outputDirectory = join(root, 'output'); + try { + mkdirSync(victimDirectory, { recursive: true }); + writeFileSync(join(victimDirectory, 'sentinel.txt'), 'buyer-owned evidence\n', 'utf8'); + symlinkSync(victimDirectory, outputDirectory, 'dir'); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readdirSync(victimDirectory)).toEqual(['sentinel.txt']); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed instead of publishing through a symlinked Office fixture output ancestor', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-output-ancestor-')); + const victimDirectory = join(root, 'buyer-owned-parent'); + const linkedParent = join(root, 'linked-parent'); + const outputDirectory = join(linkedParent, 'nested-output'); + try { + mkdirSync(victimDirectory, { recursive: true }); + writeFileSync(join(victimDirectory, 'sentinel.txt'), 'buyer-owned evidence\n', 'utf8'); + symlinkSync(victimDirectory, linkedParent, 'dir'); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readdirSync(victimDirectory)).toEqual(['sentinel.txt']); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From 7983e66cea4264a04988117459fe202c2ce89663 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 15:10:29 -0700 Subject: [PATCH 136/209] fix(perf): reject Office fixture output directory symlinks --- benchmarks/generate-office-fixtures.mjs | 33 +++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/benchmarks/generate-office-fixtures.mjs b/benchmarks/generate-office-fixtures.mjs index d09d4d36..9a77b832 100644 --- a/benchmarks/generate-office-fixtures.mjs +++ b/benchmarks/generate-office-fixtures.mjs @@ -8,7 +8,7 @@ import { openSync, writeFileSync, } from 'node:fs'; -import { resolve } from 'node:path'; +import { dirname, resolve } from 'node:path'; const DOCX_PROFILE_PAGES = Object.freeze({ small: 2, @@ -29,6 +29,8 @@ const WRITE_NOFOLLOW = constants.O_TRUNC | (constants.O_NONBLOCK ?? 0) | (constants.O_NOFOLLOW ?? 0); +const OUTPUT_DIRECTORY_ERROR = + 'Office fixture output directory must be a non-symlink directory.'; function buildDocxPage(pageNumber) { const page = String(pageNumber).padStart(3, '0'); @@ -168,6 +170,27 @@ function resolveOutputDirectory(argv) { return resolve(argv[1]); } +function inspectOutputDirectoryComponent(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Office fixture output directory could not be inspected.'); + } +} + +function assertNoSymlinkOutputDirectoryAncestors(path) { + let current = path; + while (true) { + const metadata = inspectOutputDirectoryComponent(current); + if (metadata?.isSymbolicLink()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + function writeOutputFile(outputPath, bytes) { let pathMetadata; try { @@ -215,7 +238,13 @@ function writeFixture(outputDirectory, fileName, request, units) { } const outputDirectory = resolveOutputDirectory(process.argv.slice(2)); -mkdirSync(outputDirectory, { recursive: true }); +assertNoSymlinkOutputDirectoryAncestors(outputDirectory); +try { + mkdirSync(outputDirectory, { recursive: true }); +} catch { + throw new Error('Office fixture output directory could not be prepared.'); +} +assertNoSymlinkOutputDirectoryAncestors(outputDirectory); const docx = {}; for (const [profile, pages] of Object.entries(DOCX_PROFILE_PAGES)) { From 31dee74f8dcf7b1383e7ea95953754f08c8adb7e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 16:03:44 -0700 Subject: [PATCH 137/209] test(perf): reject hard-linked Office fixture outputs --- src/performanceOfficeFixtureContract.test.ts | 21 +++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/performanceOfficeFixtureContract.test.ts b/src/performanceOfficeFixtureContract.test.ts index 39abb092..c55b36dd 100644 --- a/src/performanceOfficeFixtureContract.test.ts +++ b/src/performanceOfficeFixtureContract.test.ts @@ -1,5 +1,6 @@ import { execFileSync } from 'node:child_process'; import { + linkSync, mkdirSync, mkdtempSync, readFileSync, @@ -171,6 +172,24 @@ describe('deterministic synthetic Office performance fixtures', () => { } }); + it('fails closed instead of overwriting a multiply linked Office fixture output', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-hardlink-')); + const outputDirectory = join(root, 'output'); + const victimPath = join(root, 'buyer-owned.json'); + try { + mkdirSync(outputDirectory, { recursive: true }); + writeFileSync(victimPath, 'buyer-owned evidence\n', 'utf8'); + linkSync(victimPath, join(outputDirectory, 'docx-small.json')); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readFileSync(victimPath, 'utf8')).toBe('buyer-owned evidence\n'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('fails closed instead of publishing through a symlinked Office fixture output directory', () => { if (process.platform === 'win32') return; @@ -207,4 +226,4 @@ describe('deterministic synthetic Office performance fixtures', () => { rmSync(root, { recursive: true, force: true }); } }); -}); +}); \ No newline at end of file From 233f81e26827b83f6961fccbd07f44bb1db80181 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 16:06:19 -0700 Subject: [PATCH 138/209] fix(perf): reject hard-linked Office fixture outputs --- benchmarks/generate-office-fixtures.mjs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/benchmarks/generate-office-fixtures.mjs b/benchmarks/generate-office-fixtures.mjs index 9a77b832..e3264cc4 100644 --- a/benchmarks/generate-office-fixtures.mjs +++ b/benchmarks/generate-office-fixtures.mjs @@ -3,6 +3,7 @@ import { closeSync, constants, fstatSync, + ftruncateSync, lstatSync, mkdirSync, openSync, @@ -26,7 +27,6 @@ const MULTILINGUAL_PARAGRAPH = const WRITE_NOFOLLOW = constants.O_WRONLY | constants.O_CREAT | - constants.O_TRUNC | (constants.O_NONBLOCK ?? 0) | (constants.O_NOFOLLOW ?? 0); const OUTPUT_DIRECTORY_ERROR = @@ -208,15 +208,21 @@ function writeOutputFile(outputPath, bytes) { let descriptor; try { descriptor = openSync(outputPath, WRITE_NOFOLLOW, 0o600); - if (!fstatSync(descriptor).isFile()) { + const descriptorMetadata = fstatSync(descriptor); + if (!descriptorMetadata.isFile()) { throw new Error('Office fixture output must be a regular file.'); } + if (descriptorMetadata.nlink !== 1) { + throw new Error('Office fixture output must not be multiply linked.'); + } + ftruncateSync(descriptor, 0); writeFileSync(descriptor, bytes); } catch (error) { if ( error instanceof Error && (error.message === 'Office fixture output must be a regular file.' || - error.message === 'Office fixture output must be a regular non-symlink file.') + error.message === 'Office fixture output must be a regular non-symlink file.' || + error.message === 'Office fixture output must not be multiply linked.') ) { throw error; } @@ -293,4 +299,4 @@ const manifest = Object.freeze({ writeOutputFile( resolve(outputDirectory, 'manifest.json'), Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`, 'utf8'), -); +); \ No newline at end of file From cdd9e3492d0e459b813108d48118def960ccfc93 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 17:10:54 -0700 Subject: [PATCH 139/209] test(perf): expose summary output hard-link overwrite --- ...easurementStatisticsOutputHardlink.test.ts | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 src/performanceMeasurementStatisticsOutputHardlink.test.ts diff --git a/src/performanceMeasurementStatisticsOutputHardlink.test.ts b/src/performanceMeasurementStatisticsOutputHardlink.test.ts new file mode 100644 index 00000000..75e18c00 --- /dev/null +++ b/src/performanceMeasurementStatisticsOutputHardlink.test.ts @@ -0,0 +1,67 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + linkSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +function writeInput(path: string): void { + writeFileSync( + path, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-small', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256: 'b'.repeat(64), + documentProfile: 'small', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [1, 2, 3], + })}\n`, + 'utf8', + ); +} + +describe('benchmark summary output hard-link safety', () => { + it('fails closed before truncating an unrelated hard-linked output target', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-summary-output-hardlink-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const sentinel = join(root, 'buyer-owned.txt'); + const summaryJson = join(output, 'summary.json'); + + try { + writeInput(input); + mkdirSync(output); + writeFileSync(sentinel, 'buyer-owned-content\n', 'utf8'); + linkSync(sentinel, summaryJson); + const originalSentinel = readFileSync(sentinel, 'utf8'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output paths must not be multiply linked.', + ); + expect(readFileSync(sentinel, 'utf8')).toBe(originalSentinel); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 36d88306e52e8037f2e87bd37c450b489e40d9e1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 17:14:12 -0700 Subject: [PATCH 140/209] fix(perf): reject hard-linked summary outputs --- benchmarks/summarize-samples.mjs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 9193d97e..a7330965 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -249,7 +249,7 @@ function formatSummary(summary) { `runtime_id=${summary.runtimeId}`, `reference_hardware_id=${summary.referenceHardwareId}`, `samples=${summary.sampleCount}`, - `percentile_method=${summary.percentileMethod}`, + `percentile_method=nearest-rank`, `minimum=${summary.minimum}`, `p50=${summary.p50}`, `p75=${summary.p75}`, @@ -271,6 +271,11 @@ function assertRegularOutputDestination(path) { if (metadata !== undefined && !metadata.isFile()) { throw new Error('Benchmark summary output paths must be regular files.'); } + if (metadata !== undefined && metadata.nlink !== 1) { + throw new Error( + 'Benchmark summary output paths must not be multiply linked.', + ); + } } function inspectOutputDirectoryComponent(path) { From 22223643b70ae681c4f2180d9fde8b14f032f8cf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 17:15:07 -0700 Subject: [PATCH 141/209] chore(perf): preserve summary metadata formatting --- benchmarks/summarize-samples.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index a7330965..3fa1c787 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -249,7 +249,7 @@ function formatSummary(summary) { `runtime_id=${summary.runtimeId}`, `reference_hardware_id=${summary.referenceHardwareId}`, `samples=${summary.sampleCount}`, - `percentile_method=nearest-rank`, + `percentile_method=${summary.percentileMethod}`, `minimum=${summary.minimum}`, `p50=${summary.p50}`, `p75=${summary.p75}`, From 3ddee5a64328829f5eb34766684434542616ef6a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 17:18:31 -0700 Subject: [PATCH 142/209] fix(perf): preserve summary alias diagnostics --- benchmarks/summarize-samples.mjs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs index 3fa1c787..b98d2b87 100644 --- a/benchmarks/summarize-samples.mjs +++ b/benchmarks/summarize-samples.mjs @@ -271,6 +271,10 @@ function assertRegularOutputDestination(path) { if (metadata !== undefined && !metadata.isFile()) { throw new Error('Benchmark summary output paths must be regular files.'); } +} + +function assertSingleLinkOutputDestination(path) { + const metadata = lstatSync(path, { throwIfNoEntry: false }); if (metadata !== undefined && metadata.nlink !== 1) { throw new Error( 'Benchmark summary output paths must not be multiply linked.', @@ -346,6 +350,8 @@ function main() { if (refersToSameFile(summaryJsonPath, summaryTextPath)) { throw new Error('Benchmark summary outputs must be distinct files.'); } + assertSingleLinkOutputDestination(summaryJsonPath); + assertSingleLinkOutputDestination(summaryTextPath); const input = validateInput(readBoundedJson(inputPath)); const summary = summarize(input); writeFileSync( From 399a67a17fdc271fa87dd6dd27b839d723014b61 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 17:21:50 -0700 Subject: [PATCH 143/209] test(perf): expose producer output hard-link overwrite --- ...eMeasurementProducerOutputHardlink.test.ts | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 src/performanceMeasurementProducerOutputHardlink.test.ts diff --git a/src/performanceMeasurementProducerOutputHardlink.test.ts b/src/performanceMeasurementProducerOutputHardlink.test.ts new file mode 100644 index 00000000..e3210db3 --- /dev/null +++ b/src/performanceMeasurementProducerOutputHardlink.test.ts @@ -0,0 +1,128 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + linkSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const markdownScript = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); +const revisionScript = resolve( + process.cwd(), + 'benchmarks/measure-revision-evidence.mjs', +); +const sourceCommitSha = 'a'.repeat(40); +const runtimeId = 'node-22.18.0'; +const referenceHardwareId = 'github-actions-ubuntu-24.04-x64'; + +function sha256(content: string): string { + return createHash('sha256').update(content).digest('hex'); +} + +function commonArguments( + input: string, + module: string, + moduleSha256: string, + output: string, +): string[] { + return [ + '--input', + input, + '--module', + module, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + sourceCommitSha, + '--artifact-sha256', + moduleSha256, + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ]; +} + +function expectHardlinkFailure( + script: string, + args: string[], + sentinel: string, + expectedMessage: string, +): void { + const originalSentinel = readFileSync(sentinel, 'utf8'); + const result = spawnSync(process.execPath, [script, ...args], { + cwd: process.cwd(), + encoding: 'utf8', + }); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe(expectedMessage); + expect(readFileSync(sentinel, 'utf8')).toBe(originalSentinel); +} + +describe('benchmark producer output hard-link safety', () => { + it('fails closed before Markdown measurement overwrites an unrelated hard link', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-hardlink-')); + const input = join(root, 'document.md'); + const module = join(root, 'markdown-module.mjs'); + const output = join(root, 'samples.json'); + const sentinel = join(root, 'buyer-owned.txt'); + const moduleSource = 'export const markdownToHtml = (source) => `

${source}

`;\n'; + + try { + writeFileSync(input, '# Hello\n', 'utf8'); + writeFileSync(module, moduleSource, 'utf8'); + writeFileSync(sentinel, 'buyer-owned-content\n', 'utf8'); + linkSync(sentinel, output); + + expectHardlinkFailure( + markdownScript, + commonArguments(input, module, sha256(moduleSource), output), + sentinel, + 'Markdown benchmark output must not be multiply linked.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before revision measurement overwrites an unrelated hard link', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-hardlink-')); + const input = join(root, 'document-envelope.json'); + const module = join(root, 'revision-module.mjs'); + const output = join(root, 'samples.json'); + const sentinel = join(root, 'buyer-owned.txt'); + const moduleSource = [ + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) {', + " return { revision: { digestHex: String(source.byteLength).padStart(64, '0') } };", + '}', + '', + ].join('\n'); + + try { + writeFileSync(input, '{"contractVersion":1}\n', 'utf8'); + writeFileSync(module, moduleSource, 'utf8'); + writeFileSync(sentinel, 'buyer-owned-content\n', 'utf8'); + linkSync(sentinel, output); + + expectHardlinkFailure( + revisionScript, + commonArguments(input, module, sha256(moduleSource), output), + sentinel, + 'Revision benchmark output must not be multiply linked.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 1b7da55909d83d9fdbf68f28e0e197577e437707 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 17:24:56 -0700 Subject: [PATCH 144/209] fix(perf): reject hard-linked Markdown output --- benchmarks/measure-markdown.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 8d0549b5..68f1cae1 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -365,6 +365,9 @@ async function main() { if (outputMetadata !== undefined && !outputMetadata.isFile()) { throw new Error('Markdown benchmark output must be a regular file.'); } + if (outputMetadata !== undefined && outputMetadata.nlink !== 1) { + throw new Error('Markdown benchmark output must not be multiply linked.'); + } writeMeasurementOutput( args.outputPath, `${JSON.stringify( From a48c1d40947451550a7392ee889ab2c23d51702b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 17:25:40 -0700 Subject: [PATCH 145/209] fix(perf): reject hard-linked revision output --- benchmarks/measure-revision-evidence.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index b0064e60..a209e3e0 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -374,6 +374,9 @@ async function main() { if (outputMetadata !== undefined && !outputMetadata.isFile()) { throw new Error('Revision benchmark output must be a regular file.'); } + if (outputMetadata !== undefined && outputMetadata.nlink !== 1) { + throw new Error('Revision benchmark output must not be multiply linked.'); + } writeMeasurementOutput( args.outputPath, `${JSON.stringify( From 2f94a6b7a551bc4b977ff36e5f20b3230522a2af Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 18:10:46 -0700 Subject: [PATCH 146/209] test(perf): require revision evidence in single-command suite --- ...formanceSingleCommandSuiteContract.test.ts | 153 +++++++++++++----- 1 file changed, 114 insertions(+), 39 deletions(-) diff --git a/src/performanceSingleCommandSuiteContract.test.ts b/src/performanceSingleCommandSuiteContract.test.ts index 611fb419..ac90dfbd 100644 --- a/src/performanceSingleCommandSuiteContract.test.ts +++ b/src/performanceSingleCommandSuiteContract.test.ts @@ -25,17 +25,24 @@ afterEach(() => { }); function benchmarkArguments( - inputPath: string, - modulePath: string, - artifactSha256: string, + markdownInputPath: string, + markdownModulePath: string, + markdownArtifactSha256: string, + revisionInputPath: string, + revisionModulePath: string, + revisionArtifactSha256: string, outputDirectory: string, ): string[] { return [ suitePath, '--input', - inputPath, + markdownInputPath, '--module', - modulePath, + markdownModulePath, + '--revision-input', + revisionInputPath, + '--revision-module', + revisionModulePath, '--profile', 'small', '--samples', @@ -43,7 +50,9 @@ function benchmarkArguments( '--source-commit-sha', 'a'.repeat(40), '--artifact-sha256', - artifactSha256, + markdownArtifactSha256, + '--revision-artifact-sha256', + revisionArtifactSha256, '--runtime-id', 'node-22.0.0', '--reference-hardware-id', @@ -54,34 +63,62 @@ function benchmarkArguments( } function writeBenchmarkInputs(directory: string): { - artifactSha256: string; - inputPath: string; - modulePath: string; + markdownArtifactSha256: string; + markdownInputPath: string; + markdownModulePath: string; + revisionArtifactSha256: string; + revisionInputPath: string; + revisionModulePath: string; } { - const inputPath = join(directory, 'input.md'); - const modulePath = join(directory, 'measured.mjs'); - const moduleSource = + const markdownInputPath = join(directory, 'input.md'); + const markdownModulePath = join(directory, 'markdown-measured.mjs'); + const markdownModuleSource = "export function markdownToHtml(source) { return `

${source}

`; }\n"; - writeFileSync(inputPath, '# Buyer benchmark\n', 'utf8'); - writeFileSync(modulePath, moduleSource, 'utf8'); + const revisionInputPath = join(directory, 'document-envelope.json'); + const revisionModulePath = join(directory, 'revision-measured.mjs'); + const revisionModuleSource = `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'c'.repeat(64)}' } }; }\n`; + + writeFileSync(markdownInputPath, '# Buyer benchmark\n', 'utf8'); + writeFileSync(markdownModulePath, markdownModuleSource, 'utf8'); + writeFileSync( + revisionInputPath, + '{"contractVersion":1,"mode":"markdown","document":"# Buyer benchmark"}\n', + 'utf8', + ); + writeFileSync(revisionModulePath, revisionModuleSource, 'utf8'); + return { - artifactSha256: createHash('sha256').update(moduleSource).digest('hex'), - inputPath, - modulePath, + markdownArtifactSha256: createHash('sha256') + .update(markdownModuleSource) + .digest('hex'), + markdownInputPath, + markdownModulePath, + revisionArtifactSha256: createHash('sha256') + .update(revisionModuleSource) + .digest('hex'), + revisionInputPath, + revisionModulePath, }; } describe('single-command benchmark suite contract', () => { - it('measures and summarizes one deterministic Markdown profile with one command', () => { + it('measures and summarizes Markdown serialization and revision evidence with one command', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); temporaryDirectories.push(directory); const outputDirectory = join(directory, 'evidence'); - const { artifactSha256, inputPath, modulePath } = - writeBenchmarkInputs(directory); + const inputs = writeBenchmarkInputs(directory); const output = execFileSync( process.execPath, - benchmarkArguments(inputPath, modulePath, artifactSha256, outputDirectory), + benchmarkArguments( + inputs.markdownInputPath, + inputs.markdownModulePath, + inputs.markdownArtifactSha256, + inputs.revisionInputPath, + inputs.revisionModulePath, + inputs.revisionArtifactSha256, + outputDirectory, + ), { cwd: repositoryRoot, encoding: 'utf8', @@ -93,34 +130,64 @@ describe('single-command benchmark suite contract', () => { expect(JSON.parse(output.trim())).toEqual({ contractVersion: 1, documentProfile: 'small', - samples: 'samples.json', + markdownSamples: 'markdown/samples.json', + markdownSummaryJson: 'markdown/summary/summary.json', + markdownSummaryText: 'markdown/summary/summary.txt', + revisionSamples: 'revision/samples.json', + revisionSummaryJson: 'revision/summary/summary.json', + revisionSummaryText: 'revision/summary/summary.txt', status: 'completed', - summaryJson: 'summary/summary.json', - summaryText: 'summary/summary.txt', }); - const samples = JSON.parse( - readFileSync(join(outputDirectory, 'samples.json'), 'utf8'), + const markdownSamples = JSON.parse( + readFileSync(join(outputDirectory, 'markdown', 'samples.json'), 'utf8'), ) as { benchmarkId?: unknown; documentProfile?: unknown; samples?: unknown }; - expect(samples.benchmarkId).toBe('markdown-serialization-small'); - expect(samples.documentProfile).toBe('small'); - expect(samples.samples).toHaveLength(2); + expect(markdownSamples.benchmarkId).toBe('markdown-serialization-small'); + expect(markdownSamples.documentProfile).toBe('small'); + expect(markdownSamples.samples).toHaveLength(2); - const summary = JSON.parse( - readFileSync(join(outputDirectory, 'summary', 'summary.json'), 'utf8'), + const markdownSummary = JSON.parse( + readFileSync( + join(outputDirectory, 'markdown', 'summary', 'summary.json'), + 'utf8', + ), ) as { benchmarkId?: unknown; documentProfile?: unknown }; - expect(summary.benchmarkId).toBe('markdown-serialization-small'); - expect(summary.documentProfile).toBe('small'); + expect(markdownSummary.benchmarkId).toBe('markdown-serialization-small'); + expect(markdownSummary.documentProfile).toBe('small'); expect( - readFileSync(join(outputDirectory, 'summary', 'summary.txt'), 'utf8'), + readFileSync( + join(outputDirectory, 'markdown', 'summary', 'summary.txt'), + 'utf8', + ), ).toContain('markdown-serialization-small'); + + const revisionSamples = JSON.parse( + readFileSync(join(outputDirectory, 'revision', 'samples.json'), 'utf8'), + ) as { benchmarkId?: unknown; documentProfile?: unknown; samples?: unknown }; + expect(revisionSamples.benchmarkId).toBe('revision-evidence-small'); + expect(revisionSamples.documentProfile).toBe('small'); + expect(revisionSamples.samples).toHaveLength(2); + + const revisionSummary = JSON.parse( + readFileSync( + join(outputDirectory, 'revision', 'summary', 'summary.json'), + 'utf8', + ), + ) as { benchmarkId?: unknown; documentProfile?: unknown }; + expect(revisionSummary.benchmarkId).toBe('revision-evidence-small'); + expect(revisionSummary.documentProfile).toBe('small'); + expect( + readFileSync( + join(outputDirectory, 'revision', 'summary', 'summary.txt'), + 'utf8', + ), + ).toContain('revision-evidence-small'); }); it('fails closed before writing evidence through a symlink output directory', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); temporaryDirectories.push(directory); - const { artifactSha256, inputPath, modulePath } = - writeBenchmarkInputs(directory); + const inputs = writeBenchmarkInputs(directory); const actualOutputDirectory = join(directory, 'outside-target'); const outputDirectory = join(directory, 'evidence-link'); mkdirSync(actualOutputDirectory); @@ -132,7 +199,15 @@ describe('single-command benchmark suite contract', () => { const result = spawnSync( process.execPath, - benchmarkArguments(inputPath, modulePath, artifactSha256, outputDirectory), + benchmarkArguments( + inputs.markdownInputPath, + inputs.markdownModulePath, + inputs.markdownArtifactSha256, + inputs.revisionInputPath, + inputs.revisionModulePath, + inputs.revisionArtifactSha256, + outputDirectory, + ), { cwd: repositoryRoot, encoding: 'utf8', @@ -145,7 +220,7 @@ describe('single-command benchmark suite contract', () => { expect(result.stderr).toBe( 'Benchmark suite output directory must be a non-symlink directory.\n', ); - expect(existsSync(join(actualOutputDirectory, 'samples.json'))).toBe(false); - expect(existsSync(join(actualOutputDirectory, 'summary'))).toBe(false); + expect(existsSync(join(actualOutputDirectory, 'markdown'))).toBe(false); + expect(existsSync(join(actualOutputDirectory, 'revision'))).toBe(false); }); }); From 49baff63b57411dd70239730999766673419a617 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 18:11:21 -0700 Subject: [PATCH 147/209] feat(perf): compose revision evidence in current benchmark suite --- benchmarks/run-current-suite.mjs | 118 ++++++++++++++++++++++++++----- 1 file changed, 101 insertions(+), 17 deletions(-) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 5d9471b2..58c07229 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -8,10 +8,13 @@ const repositoryRoot = resolve(benchmarkDirectory, '..'); const expectedFlags = Object.freeze([ '--input', '--module', + '--revision-input', + '--revision-module', '--profile', '--samples', '--source-commit-sha', '--artifact-sha256', + '--revision-artifact-sha256', '--runtime-id', '--reference-hardware-id', '--output', @@ -26,14 +29,47 @@ function resolveArguments(argv) { expectedFlags.some((_, index) => argv[index * 2 + 1]?.length === 0) ) { throw new Error( - 'Usage: node benchmarks/run-current-suite.mjs --input --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', + 'Usage: node benchmarks/run-current-suite.mjs --input --module --revision-input --revision-module --profile --samples --source-commit-sha --artifact-sha256 --revision-artifact-sha256 --runtime-id --reference-hardware-id --output ', ); } + const values = Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); + const sharedArguments = Object.freeze([ + '--profile', + values['--profile'], + '--samples', + values['--samples'], + '--source-commit-sha', + values['--source-commit-sha'], + '--runtime-id', + values['--runtime-id'], + '--reference-hardware-id', + values['--reference-hardware-id'], + ]); + return Object.freeze({ - documentProfile: argv[5], - forwardedArguments: Object.freeze(argv.slice(0, -2)), - outputDirectory: resolve(argv[17]), + documentProfile: values['--profile'], + markdownArguments: Object.freeze([ + '--input', + values['--input'], + '--module', + values['--module'], + ...sharedArguments, + '--artifact-sha256', + values['--artifact-sha256'], + ]), + revisionArguments: Object.freeze([ + '--input', + values['--revision-input'], + '--module', + values['--revision-module'], + ...sharedArguments, + '--artifact-sha256', + values['--revision-artifact-sha256'], + ]), + outputDirectory: resolve(values['--output']), }); } @@ -103,31 +139,79 @@ function runBoundedNodeScript(scriptName, args, failureMessage) { } } -function main(argv) { - const args = resolveArguments(argv); - prepareOutputDirectory(args.outputDirectory); - const samplesPath = resolve(args.outputDirectory, 'samples.json'); - const summaryDirectory = resolve(args.outputDirectory, 'summary'); - +function runMeasurementAndSummary({ + measurementScript, + measurementArguments, + samplesPath, + summaryDirectory, + measurementFailure, + summaryFailure, +}) { runBoundedNodeScript( - 'measure-markdown.mjs', - [...args.forwardedArguments, '--output', samplesPath], - 'Benchmark suite measurement failed.', + measurementScript, + [...measurementArguments, '--output', samplesPath], + measurementFailure, ); runBoundedNodeScript( 'summarize-samples.mjs', ['--input', samplesPath, '--output', summaryDirectory], - 'Benchmark suite summary failed.', + summaryFailure, ); +} + +function main(argv) { + const args = resolveArguments(argv); + prepareOutputDirectory(args.outputDirectory); + + const markdownSamplesPath = resolve( + args.outputDirectory, + 'markdown', + 'samples.json', + ); + const markdownSummaryDirectory = resolve( + args.outputDirectory, + 'markdown', + 'summary', + ); + const revisionSamplesPath = resolve( + args.outputDirectory, + 'revision', + 'samples.json', + ); + const revisionSummaryDirectory = resolve( + args.outputDirectory, + 'revision', + 'summary', + ); + + runMeasurementAndSummary({ + measurementScript: 'measure-markdown.mjs', + measurementArguments: args.markdownArguments, + samplesPath: markdownSamplesPath, + summaryDirectory: markdownSummaryDirectory, + measurementFailure: 'Benchmark suite Markdown measurement failed.', + summaryFailure: 'Benchmark suite Markdown summary failed.', + }); + runMeasurementAndSummary({ + measurementScript: 'measure-revision-evidence.mjs', + measurementArguments: args.revisionArguments, + samplesPath: revisionSamplesPath, + summaryDirectory: revisionSummaryDirectory, + measurementFailure: 'Benchmark suite revision measurement failed.', + summaryFailure: 'Benchmark suite revision summary failed.', + }); process.stdout.write( `${JSON.stringify({ contractVersion: 1, documentProfile: args.documentProfile, - samples: 'samples.json', + markdownSamples: 'markdown/samples.json', + markdownSummaryJson: 'markdown/summary/summary.json', + markdownSummaryText: 'markdown/summary/summary.txt', + revisionSamples: 'revision/samples.json', + revisionSummaryJson: 'revision/summary/summary.json', + revisionSummaryText: 'revision/summary/summary.txt', status: 'completed', - summaryJson: 'summary/summary.json', - summaryText: 'summary/summary.txt', })}\n`, ); } From 2614a79480597f7fe5b77488b2738201289b19c6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 18:13:38 -0700 Subject: [PATCH 148/209] fix(perf): preserve measured-script argument order --- benchmarks/run-current-suite.mjs | 34 +++++++++++++++++++------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 58c07229..d28b5bd2 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -36,18 +36,6 @@ function resolveArguments(argv) { const values = Object.fromEntries( expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), ); - const sharedArguments = Object.freeze([ - '--profile', - values['--profile'], - '--samples', - values['--samples'], - '--source-commit-sha', - values['--source-commit-sha'], - '--runtime-id', - values['--runtime-id'], - '--reference-hardware-id', - values['--reference-hardware-id'], - ]); return Object.freeze({ documentProfile: values['--profile'], @@ -56,18 +44,36 @@ function resolveArguments(argv) { values['--input'], '--module', values['--module'], - ...sharedArguments, + '--profile', + values['--profile'], + '--samples', + values['--samples'], + '--source-commit-sha', + values['--source-commit-sha'], '--artifact-sha256', values['--artifact-sha256'], + '--runtime-id', + values['--runtime-id'], + '--reference-hardware-id', + values['--reference-hardware-id'], ]), revisionArguments: Object.freeze([ '--input', values['--revision-input'], '--module', values['--revision-module'], - ...sharedArguments, + '--profile', + values['--profile'], + '--samples', + values['--samples'], + '--source-commit-sha', + values['--source-commit-sha'], '--artifact-sha256', values['--revision-artifact-sha256'], + '--runtime-id', + values['--runtime-id'], + '--reference-hardware-id', + values['--reference-hardware-id'], ]), outputDirectory: resolve(values['--output']), }); From 7ae80074210e7660ba28cdce89e5e4400bf022d5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 18:16:55 -0700 Subject: [PATCH 149/209] test(perf): keep suite symlink regression on current CLI --- src/performanceMeasurementSuiteAncestorSymlink.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/performanceMeasurementSuiteAncestorSymlink.test.ts b/src/performanceMeasurementSuiteAncestorSymlink.test.ts index 7b61c4b5..5ea36b6d 100644 --- a/src/performanceMeasurementSuiteAncestorSymlink.test.ts +++ b/src/performanceMeasurementSuiteAncestorSymlink.test.ts @@ -30,6 +30,10 @@ describe('benchmark suite output path ancestry', () => { join(root, 'unused.md'), '--module', join(root, 'unused.mjs'), + '--revision-input', + join(root, 'unused-envelope.json'), + '--revision-module', + join(root, 'unused-revision.mjs'), '--profile', 'large', '--samples', @@ -38,6 +42,8 @@ describe('benchmark suite output path ancestry', () => { 'a'.repeat(40), '--artifact-sha256', 'b'.repeat(64), + '--revision-artifact-sha256', + 'c'.repeat(64), '--runtime-id', 'node-22.18.0', '--reference-hardware-id', From eb6e8acf1145077ea2e2b40ed72a9660577dfcdf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 18:20:34 -0700 Subject: [PATCH 150/209] test(perf): require atomic suite evidence publication --- ...formanceSingleCommandSuiteContract.test.ts | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/performanceSingleCommandSuiteContract.test.ts b/src/performanceSingleCommandSuiteContract.test.ts index ac90dfbd..c87a73b9 100644 --- a/src/performanceSingleCommandSuiteContract.test.ts +++ b/src/performanceSingleCommandSuiteContract.test.ts @@ -184,6 +184,49 @@ describe('single-command benchmark suite contract', () => { ).toContain('revision-evidence-small'); }); + it('removes partial suite evidence when a downstream measurement fails', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); + temporaryDirectories.push(directory); + const outputDirectory = join(directory, 'evidence'); + const inputs = writeBenchmarkInputs(directory); + const failingRevisionModuleSource = + "export async function createDocumentEnvelopeRevisionEvidenceBytes() { throw new Error('private benchmark failure'); }\n"; + writeFileSync( + inputs.revisionModulePath, + failingRevisionModuleSource, + 'utf8', + ); + const failingRevisionArtifactSha256 = createHash('sha256') + .update(failingRevisionModuleSource) + .digest('hex'); + + const result = spawnSync( + process.execPath, + benchmarkArguments( + inputs.markdownInputPath, + inputs.markdownModulePath, + inputs.markdownArtifactSha256, + inputs.revisionInputPath, + inputs.revisionModulePath, + failingRevisionArtifactSha256, + outputDirectory, + ), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).not.toBe(0); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite revision measurement failed.\n', + ); + expect(existsSync(outputDirectory)).toBe(false); + }); + it('fails closed before writing evidence through a symlink output directory', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); temporaryDirectories.push(directory); From 0af333b74ad9ec619025f7bbb743c6b0cdb75c8d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 18:24:40 -0700 Subject: [PATCH 151/209] fix(perf): remove partial suite evidence on failure --- benchmarks/run-current-suite.mjs | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index d28b5bd2..275d53ac 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -1,5 +1,5 @@ import { spawnSync } from 'node:child_process'; -import { lstatSync, mkdirSync } from 'node:fs'; +import { lstatSync, mkdirSync, rmSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -107,7 +107,7 @@ function prepareOutputDirectory(path) { if (!existing.isDirectory()) { throw new Error(OUTPUT_DIRECTORY_ERROR); } - return; + return false; } try { @@ -121,6 +121,15 @@ function prepareOutputDirectory(path) { if (created === undefined || !created.isDirectory()) { throw new Error(OUTPUT_DIRECTORY_ERROR); } + return true; +} + +function removePartialOutputDirectory(path) { + try { + rmSync(path, { recursive: true, force: true }); + } catch { + throw new Error('Benchmark suite partial evidence could not be removed.'); + } } function runBoundedNodeScript(scriptName, args, failureMessage) { @@ -165,10 +174,7 @@ function runMeasurementAndSummary({ ); } -function main(argv) { - const args = resolveArguments(argv); - prepareOutputDirectory(args.outputDirectory); - +function runSuite(args) { const markdownSamplesPath = resolve( args.outputDirectory, 'markdown', @@ -206,6 +212,20 @@ function main(argv) { measurementFailure: 'Benchmark suite revision measurement failed.', summaryFailure: 'Benchmark suite revision summary failed.', }); +} + +function main(argv) { + const args = resolveArguments(argv); + const createdOutputDirectory = prepareOutputDirectory(args.outputDirectory); + + try { + runSuite(args); + } catch (error) { + if (createdOutputDirectory) { + removePartialOutputDirectory(args.outputDirectory); + } + throw error; + } process.stdout.write( `${JSON.stringify({ From d609faa683c742dfd757dc7d54bd664c9c676901 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 22:07:12 -0700 Subject: [PATCH 152/209] test(perf): require packed artifact benchmark provenance --- ...ormancePackedArtifactSuiteContract.test.ts | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 src/performancePackedArtifactSuiteContract.test.ts diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts new file mode 100644 index 00000000..db4933dc --- /dev/null +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -0,0 +1,146 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function sha256(bytes: Buffer | string): string { + return createHash('sha256').update(bytes).digest('hex'); +} + +function createPackedBenchmarkFixture(directory: string): { + packageSha256: string; + tarballPath: string; +} { + const packageDirectory = join(directory, 'package-source'); + const distDirectory = join(packageDirectory, 'dist'); + const packDirectory = join(directory, 'packed'); + mkdirSync(distDirectory, { recursive: true }); + mkdirSync(packDirectory, { recursive: true }); + + writeFileSync( + join(packageDirectory, 'package.json'), + `${JSON.stringify( + { + name: '@contextualwisdomlab/cwl-editor', + version: '0.0.0-benchmark-fixture', + type: 'module', + files: ['dist'], + }, + null, + 2, + )}\n`, + 'utf8', + ); + writeFileSync( + join(distDirectory, 'cwl-markdown.js'), + "export function markdownToHtml(source) { return `

${source}

`; }\n", + 'utf8', + ); + writeFileSync( + join(distDirectory, 'cwl-revision-evidence.js'), + `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'c'.repeat(64)}' } }; }\n`, + 'utf8', + ); + + const packResult = JSON.parse( + execFileSync( + 'npm', + [ + 'pack', + '--json', + '--ignore-scripts', + '--pack-destination', + packDirectory, + ], + { + cwd: packageDirectory, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, + ), + )[0] as { filename: string }; + const tarballPath = join(packDirectory, packResult.filename); + return { + packageSha256: sha256(readFileSync(tarballPath)), + tarballPath, + }; +} + +describe('packed artifact benchmark suite contract', () => { + it('binds one-command benchmark evidence to a packed npm artifact digest', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-benchmark-')); + temporaryDirectories.push(directory); + const markdownInputPath = join(directory, 'input.md'); + const revisionInputPath = join(directory, 'document-envelope.json'); + const outputDirectory = join(directory, 'evidence'); + writeFileSync(markdownInputPath, '# Packed buyer benchmark\n', 'utf8'); + writeFileSync( + revisionInputPath, + '{"contractVersion":1,"mode":"markdown","document":"# Packed buyer benchmark"}\n', + 'utf8', + ); + const packed = createPackedBenchmarkFixture(directory); + + const result = spawnSync( + process.execPath, + [ + suitePath, + '--input', + markdownInputPath, + '--revision-input', + revisionInputPath, + '--package-tarball', + packed.tarballPath, + '--package-sha256', + packed.packageSha256, + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + 'a'.repeat(40), + '--runtime-id', + 'node-22.0.0', + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + outputDirectory, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 15_000, + }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout.trim())).toMatchObject({ + contractVersion: 1, + packageName: '@contextualwisdomlab/cwl-editor', + packageVersion: '0.0.0-benchmark-fixture', + packageSha256: packed.packageSha256, + status: 'completed', + }); + }); +}); From a49ea9defd5e66eb467bd71e15b2d31eb8c6261e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 22:11:05 -0700 Subject: [PATCH 153/209] fix(perf): bind suite to packed npm artifact --- benchmarks/run-current-suite.mjs | 467 ++++++++++++++++++++++++++----- 1 file changed, 397 insertions(+), 70 deletions(-) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 275d53ac..64ed643e 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -1,11 +1,24 @@ +import { createHash } from 'node:crypto'; import { spawnSync } from 'node:child_process'; -import { lstatSync, mkdirSync, rmSync } from 'node:fs'; -import { dirname, resolve } from 'node:path'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdirSync, + mkdtempSync, + openSync, + readSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); const repositoryRoot = resolve(benchmarkDirectory, '..'); -const expectedFlags = Object.freeze([ +const legacyFlags = Object.freeze([ '--input', '--module', '--revision-input', @@ -19,66 +32,129 @@ const expectedFlags = Object.freeze([ '--reference-hardware-id', '--output', ]); +const packedFlags = Object.freeze([ + '--input', + '--revision-input', + '--package-tarball', + '--package-sha256', + '--profile', + '--samples', + '--source-commit-sha', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; +const MAX_PACKAGE_INDEX_BYTES = 1024 * 1024; +const MAX_PACKAGE_MANIFEST_BYTES = 1024 * 1024; +const MAX_MODULE_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const EXPECTED_PACKAGE_NAME = '@contextualwisdomlab/cwl-editor'; +const PACKAGE_MANIFEST_ENTRY = 'package/package.json'; +const MARKDOWN_MODULE_ENTRY = 'package/dist/cwl-markdown.js'; +const REVISION_MODULE_ENTRY = 'package/dist/cwl-revision-evidence.js'; const OUTPUT_DIRECTORY_ERROR = 'Benchmark suite output directory must be a non-symlink directory.'; -function resolveArguments(argv) { - if ( - argv.length !== expectedFlags.length * 2 || - expectedFlags.some((flag, index) => argv[index * 2] !== flag) || - expectedFlags.some((_, index) => argv[index * 2 + 1]?.length === 0) - ) { - throw new Error( - 'Usage: node benchmarks/run-current-suite.mjs --input --module --revision-input --revision-module --profile --samples --source-commit-sha --artifact-sha256 --revision-artifact-sha256 --runtime-id --reference-hardware-id --output ', - ); - } +function matchesArguments(argv, expectedFlags) { + return ( + argv.length === expectedFlags.length * 2 && + expectedFlags.every((flag, index) => argv[index * 2] === flag) && + expectedFlags.every((_, index) => argv[index * 2 + 1]?.length > 0) + ); +} - const values = Object.fromEntries( +function valuesForArguments(argv, expectedFlags) { + return Object.fromEntries( expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), ); +} +function sharedArguments(values) { return Object.freeze({ documentProfile: values['--profile'], - markdownArguments: Object.freeze([ - '--input', - values['--input'], - '--module', - values['--module'], - '--profile', - values['--profile'], - '--samples', - values['--samples'], - '--source-commit-sha', - values['--source-commit-sha'], - '--artifact-sha256', - values['--artifact-sha256'], - '--runtime-id', - values['--runtime-id'], - '--reference-hardware-id', - values['--reference-hardware-id'], - ]), - revisionArguments: Object.freeze([ - '--input', - values['--revision-input'], - '--module', - values['--revision-module'], - '--profile', - values['--profile'], - '--samples', - values['--samples'], - '--source-commit-sha', - values['--source-commit-sha'], - '--artifact-sha256', - values['--revision-artifact-sha256'], - '--runtime-id', - values['--runtime-id'], - '--reference-hardware-id', - values['--reference-hardware-id'], - ]), + sampleCount: values['--samples'], + sourceCommitSha: values['--source-commit-sha'], + runtimeId: values['--runtime-id'], + referenceHardwareId: values['--reference-hardware-id'], + markdownInputPath: values['--input'], + revisionInputPath: values['--revision-input'], outputDirectory: resolve(values['--output']), }); } +function measurementArguments({ + inputPath, + modulePath, + artifactSha256, + shared, +}) { + return Object.freeze([ + '--input', + inputPath, + '--module', + modulePath, + '--profile', + shared.documentProfile, + '--samples', + shared.sampleCount, + '--source-commit-sha', + shared.sourceCommitSha, + '--artifact-sha256', + artifactSha256, + '--runtime-id', + shared.runtimeId, + '--reference-hardware-id', + shared.referenceHardwareId, + ]); +} + +function resolveArguments(argv) { + if (matchesArguments(argv, packedFlags)) { + const values = valuesForArguments(argv, packedFlags); + const packageSha256 = values['--package-sha256']; + if (!SHA256_PATTERN.test(packageSha256)) { + throw new Error( + 'Benchmark suite package digest must be a lowercase 64-character SHA-256.', + ); + } + return Object.freeze({ + mode: 'packed', + shared: sharedArguments(values), + packageTarballPath: resolve(values['--package-tarball']), + packageSha256, + }); + } + + if (matchesArguments(argv, legacyFlags)) { + const values = valuesForArguments(argv, legacyFlags); + const shared = sharedArguments(values); + return Object.freeze({ + mode: 'module', + shared, + markdownArguments: measurementArguments({ + inputPath: shared.markdownInputPath, + modulePath: values['--module'], + artifactSha256: values['--artifact-sha256'], + shared, + }), + revisionArguments: measurementArguments({ + inputPath: shared.revisionInputPath, + modulePath: values['--revision-module'], + artifactSha256: values['--revision-artifact-sha256'], + shared, + }), + packageEvidence: null, + }); + } + + throw new Error( + 'Usage: node benchmarks/run-current-suite.mjs --input --revision-input --package-tarball --package-sha256 --profile --samples --source-commit-sha --runtime-id --reference-hardware-id --output ', + ); +} + function inspectOutputDirectory(path) { try { return lstatSync(path, { throwIfNoEntry: false }); @@ -132,6 +208,215 @@ function removePartialOutputDirectory(path) { } } +function readBoundedRegularFile(path, maximumBytes, invalidMessage, oversizedMessage) { + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error(invalidMessage); + } + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error(invalidMessage); + } + + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error(invalidMessage); + } + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) throw new Error(invalidMessage); + if (metadata.size > maximumBytes) throw new Error(oversizedMessage); + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= maximumBytes) { + const remainingBudget = maximumBytes + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > maximumBytes) throw new Error(oversizedMessage); + chunks.push(chunk.subarray(0, bytesRead)); + } + return Buffer.concat(chunks, totalBytes); + } finally { + closeSync(descriptor); + } +} + +function packageTarballBytes(path) { + return readBoundedRegularFile( + path, + MAX_PACKAGE_BYTES, + 'Benchmark suite package tarball must be a regular non-symlink file.', + 'Benchmark suite package tarball exceeds the supported size.', + ); +} + +function verifyPackageDigest(path, expectedSha256) { + const actualSha256 = createHash('sha256') + .update(packageTarballBytes(path)) + .digest('hex'); + if (actualSha256 !== expectedSha256) { + throw new Error('Benchmark suite package digest does not match the packed artifact.'); + } +} + +function runTar(argumentsList, maximumBytes, failureMessage) { + const result = spawnSync('tar', argumentsList, { + cwd: repositoryRoot, + maxBuffer: maximumBytes, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 30_000, + }); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !Buffer.isBuffer(result.stdout) || + result.stdout.byteLength > maximumBytes + ) { + throw new Error(failureMessage); + } + return result.stdout; +} + +function decodeUtf8(bytes, failureMessage) { + try { + return new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch { + throw new Error(failureMessage); + } +} + +function listPackedEntries(tarballPath) { + const listing = decodeUtf8( + runTar( + ['-tzf', tarballPath], + MAX_PACKAGE_INDEX_BYTES, + 'Benchmark suite package index could not be read.', + ), + 'Benchmark suite package index must be valid UTF-8.', + ); + return listing.split('\n').filter((entry) => entry.length > 0); +} + +function assertUniquePackedEntry(entries, expectedEntry) { + if (entries.filter((entry) => entry === expectedEntry).length !== 1) { + throw new Error('Benchmark suite package is missing a unique required artifact.'); + } +} + +function readPackedEntry(tarballPath, entry, maximumBytes) { + return runTar( + ['-xOzf', tarballPath, entry], + maximumBytes, + 'Benchmark suite package artifact could not be read.', + ); +} + +function parsePackageManifest(bytes) { + let manifest; + try { + manifest = JSON.parse( + decodeUtf8(bytes, 'Benchmark suite package manifest must be valid UTF-8.'), + ); + } catch (error) { + if (error instanceof Error && error.message.includes('valid UTF-8')) throw error; + throw new Error('Benchmark suite package manifest must be valid JSON.'); + } + if ( + manifest === null || + typeof manifest !== 'object' || + Array.isArray(manifest) || + manifest.name !== EXPECTED_PACKAGE_NAME || + typeof manifest.version !== 'string' || + manifest.version.length === 0 || + manifest.version.length > 128 + ) { + throw new Error('Benchmark suite package identity is invalid.'); + } + return Object.freeze({ name: manifest.name, version: manifest.version }); +} + +function moduleSha256(bytes) { + return createHash('sha256').update(bytes).digest('hex'); +} + +function preparePackedBenchmarkModules(args) { + verifyPackageDigest(args.packageTarballPath, args.packageSha256); + const entries = listPackedEntries(args.packageTarballPath); + for (const entry of [ + PACKAGE_MANIFEST_ENTRY, + MARKDOWN_MODULE_ENTRY, + REVISION_MODULE_ENTRY, + ]) { + assertUniquePackedEntry(entries, entry); + } + + const manifestBytes = readPackedEntry( + args.packageTarballPath, + PACKAGE_MANIFEST_ENTRY, + MAX_PACKAGE_MANIFEST_BYTES, + ); + const manifest = parsePackageManifest(manifestBytes); + const markdownModuleBytes = readPackedEntry( + args.packageTarballPath, + MARKDOWN_MODULE_ENTRY, + MAX_MODULE_BYTES, + ); + const revisionModuleBytes = readPackedEntry( + args.packageTarballPath, + REVISION_MODULE_ENTRY, + MAX_MODULE_BYTES, + ); + verifyPackageDigest(args.packageTarballPath, args.packageSha256); + + const temporaryDirectory = mkdtempSync( + join(tmpdir(), 'inkspan-packed-benchmark-'), + ); + const markdownModulePath = join(temporaryDirectory, 'cwl-markdown.mjs'); + const revisionModulePath = join( + temporaryDirectory, + 'cwl-revision-evidence.mjs', + ); + try { + writeFileSync(markdownModulePath, markdownModuleBytes); + writeFileSync(revisionModulePath, revisionModuleBytes); + } catch { + rmSync(temporaryDirectory, { recursive: true, force: true }); + throw new Error('Benchmark suite package modules could not be prepared.'); + } + + return Object.freeze({ + temporaryDirectory, + markdownModulePath, + markdownArtifactSha256: moduleSha256(markdownModuleBytes), + revisionModulePath, + revisionArtifactSha256: moduleSha256(revisionModuleBytes), + packageEvidence: Object.freeze({ + packageName: manifest.name, + packageVersion: manifest.version, + packageSha256: args.packageSha256, + }), + }); +} + function runBoundedNodeScript(scriptName, args, failureMessage) { const result = spawnSync( process.execPath, @@ -156,7 +441,7 @@ function runBoundedNodeScript(scriptName, args, failureMessage) { function runMeasurementAndSummary({ measurementScript, - measurementArguments, + measurementArguments: argumentsList, samplesPath, summaryDirectory, measurementFailure, @@ -164,7 +449,7 @@ function runMeasurementAndSummary({ }) { runBoundedNodeScript( measurementScript, - [...measurementArguments, '--output', samplesPath], + [...argumentsList, '--output', samplesPath], measurementFailure, ); runBoundedNodeScript( @@ -174,7 +459,7 @@ function runMeasurementAndSummary({ ); } -function runSuite(args) { +function runSuite(args, markdownArguments, revisionArguments) { const markdownSamplesPath = resolve( args.outputDirectory, 'markdown', @@ -198,7 +483,7 @@ function runSuite(args) { runMeasurementAndSummary({ measurementScript: 'measure-markdown.mjs', - measurementArguments: args.markdownArguments, + measurementArguments: markdownArguments, samplesPath: markdownSamplesPath, summaryDirectory: markdownSummaryDirectory, measurementFailure: 'Benchmark suite Markdown measurement failed.', @@ -206,7 +491,7 @@ function runSuite(args) { }); runMeasurementAndSummary({ measurementScript: 'measure-revision-evidence.mjs', - measurementArguments: args.revisionArguments, + measurementArguments: revisionArguments, samplesPath: revisionSamplesPath, summaryDirectory: revisionSummaryDirectory, measurementFailure: 'Benchmark suite revision measurement failed.', @@ -214,31 +499,73 @@ function runSuite(args) { }); } +function suiteManifest(args, packageEvidence) { + return Object.freeze({ + contractVersion: 1, + documentProfile: args.documentProfile, + ...(packageEvidence ?? {}), + markdownSamples: 'markdown/samples.json', + markdownSummaryJson: 'markdown/summary/summary.json', + markdownSummaryText: 'markdown/summary/summary.txt', + revisionSamples: 'revision/samples.json', + revisionSummaryJson: 'revision/summary/summary.json', + revisionSummaryText: 'revision/summary/summary.txt', + status: 'completed', + }); +} + function main(argv) { - const args = resolveArguments(argv); - const createdOutputDirectory = prepareOutputDirectory(args.outputDirectory); + const resolved = resolveArguments(argv); + const shared = resolved.shared; + let preparedPackage; + if (resolved.mode === 'packed') { + preparedPackage = preparePackedBenchmarkModules(resolved); + } + const markdownArguments = + resolved.mode === 'packed' + ? measurementArguments({ + inputPath: shared.markdownInputPath, + modulePath: preparedPackage.markdownModulePath, + artifactSha256: preparedPackage.markdownArtifactSha256, + shared, + }) + : resolved.markdownArguments; + const revisionArguments = + resolved.mode === 'packed' + ? measurementArguments({ + inputPath: shared.revisionInputPath, + modulePath: preparedPackage.revisionModulePath, + artifactSha256: preparedPackage.revisionArtifactSha256, + shared, + }) + : resolved.revisionArguments; + const packageEvidence = + resolved.mode === 'packed' ? preparedPackage.packageEvidence : null; + + let createdOutputDirectory = false; try { - runSuite(args); + createdOutputDirectory = prepareOutputDirectory(shared.outputDirectory); + runSuite(shared, markdownArguments, revisionArguments); + if (resolved.mode === 'packed') { + verifyPackageDigest(resolved.packageTarballPath, resolved.packageSha256); + } } catch (error) { if (createdOutputDirectory) { - removePartialOutputDirectory(args.outputDirectory); + removePartialOutputDirectory(shared.outputDirectory); } throw error; + } finally { + if (preparedPackage !== undefined) { + rmSync(preparedPackage.temporaryDirectory, { + recursive: true, + force: true, + }); + } } process.stdout.write( - `${JSON.stringify({ - contractVersion: 1, - documentProfile: args.documentProfile, - markdownSamples: 'markdown/samples.json', - markdownSummaryJson: 'markdown/summary/summary.json', - markdownSummaryText: 'markdown/summary/summary.txt', - revisionSamples: 'revision/samples.json', - revisionSummaryJson: 'revision/summary/summary.json', - revisionSummaryText: 'revision/summary/summary.txt', - status: 'completed', - })}\n`, + `${JSON.stringify(suiteManifest(shared, packageEvidence))}\n`, ); } From f0c9d8f5fb5ad498577c5b4fe2919a6a51dacf9a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 22:13:55 -0700 Subject: [PATCH 154/209] test(perf): reject mislabeled benchmark runtime provenance --- ...ormancePackedArtifactSuiteContract.test.ts | 106 ++++++++++++------ 1 file changed, 74 insertions(+), 32 deletions(-) diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts index db4933dc..4d1f3949 100644 --- a/src/performancePackedArtifactSuiteContract.test.ts +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -15,6 +15,7 @@ import { afterEach, describe, expect, it } from 'vitest'; const repositoryRoot = process.cwd(); const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); const temporaryDirectories: string[] = []; +const activeRuntimeId = `node-${process.versions.node}`; afterEach(() => { for (const directory of temporaryDirectories.splice(0)) { @@ -85,46 +86,59 @@ function createPackedBenchmarkFixture(directory: string): { }; } +function packedSuiteArguments(options: { + directory: string; + packageSha256: string; + runtimeId: string; + tarballPath: string; +}): string[] { + const markdownInputPath = join(options.directory, 'input.md'); + const revisionInputPath = join(options.directory, 'document-envelope.json'); + writeFileSync(markdownInputPath, '# Packed buyer benchmark\n', 'utf8'); + writeFileSync( + revisionInputPath, + '{"contractVersion":1,"mode":"markdown","document":"# Packed buyer benchmark"}\n', + 'utf8', + ); + return [ + suitePath, + '--input', + markdownInputPath, + '--revision-input', + revisionInputPath, + '--package-tarball', + options.tarballPath, + '--package-sha256', + options.packageSha256, + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + 'a'.repeat(40), + '--runtime-id', + options.runtimeId, + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + join(options.directory, 'evidence'), + ]; +} + describe('packed artifact benchmark suite contract', () => { it('binds one-command benchmark evidence to a packed npm artifact digest', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-benchmark-')); temporaryDirectories.push(directory); - const markdownInputPath = join(directory, 'input.md'); - const revisionInputPath = join(directory, 'document-envelope.json'); - const outputDirectory = join(directory, 'evidence'); - writeFileSync(markdownInputPath, '# Packed buyer benchmark\n', 'utf8'); - writeFileSync( - revisionInputPath, - '{"contractVersion":1,"mode":"markdown","document":"# Packed buyer benchmark"}\n', - 'utf8', - ); const packed = createPackedBenchmarkFixture(directory); const result = spawnSync( process.execPath, - [ - suitePath, - '--input', - markdownInputPath, - '--revision-input', - revisionInputPath, - '--package-tarball', - packed.tarballPath, - '--package-sha256', - packed.packageSha256, - '--profile', - 'small', - '--samples', - '2', - '--source-commit-sha', - 'a'.repeat(40), - '--runtime-id', - 'node-22.0.0', - '--reference-hardware-id', - `refhw-sha256-${'b'.repeat(64)}`, - '--output', - outputDirectory, - ], + packedSuiteArguments({ + directory, + packageSha256: packed.packageSha256, + runtimeId: activeRuntimeId, + tarballPath: packed.tarballPath, + }), { cwd: repositoryRoot, encoding: 'utf8', @@ -143,4 +157,32 @@ describe('packed artifact benchmark suite contract', () => { status: 'completed', }); }); + + it('rejects a runtime identifier that does not match the active Node process', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-runtime-')); + temporaryDirectories.push(directory); + const packed = createPackedBenchmarkFixture(directory); + + const result = spawnSync( + process.execPath, + packedSuiteArguments({ + directory, + packageSha256: packed.packageSha256, + runtimeId: 'node-0.0.0', + tarballPath: packed.tarballPath, + }), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 15_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite runtime ID must match the active Node runtime.\n', + ); + }); }); From 0c9f3f3e4e2c1c30e9ed0f3adc8469de13417602 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 22:18:27 -0700 Subject: [PATCH 155/209] fix(perf): attest active benchmark runtime --- benchmarks/run-current-suite.mjs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 64ed643e..e63398cf 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -120,6 +120,12 @@ function resolveArguments(argv) { 'Benchmark suite package digest must be a lowercase 64-character SHA-256.', ); } + const activeRuntimeId = `node-${process.versions.node}`; + if (values['--runtime-id'] !== activeRuntimeId) { + throw new Error( + 'Benchmark suite runtime ID must match the active Node runtime.', + ); + } return Object.freeze({ mode: 'packed', shared: sharedArguments(values), From 5c8f6f8670c6906ffef648470c763202aa16ee11 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 24 Aug 2026 07:28:35 -0700 Subject: [PATCH 156/209] test(perf): bind suite manifest to run provenance --- src/performanceSingleCommandSuiteContract.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/performanceSingleCommandSuiteContract.test.ts b/src/performanceSingleCommandSuiteContract.test.ts index c87a73b9..0eae1cc5 100644 --- a/src/performanceSingleCommandSuiteContract.test.ts +++ b/src/performanceSingleCommandSuiteContract.test.ts @@ -130,6 +130,10 @@ describe('single-command benchmark suite contract', () => { expect(JSON.parse(output.trim())).toEqual({ contractVersion: 1, documentProfile: 'small', + sampleCount: 2, + sourceCommitSha: 'a'.repeat(40), + runtimeId: 'node-22.0.0', + referenceHardwareId: `refhw-sha256-${'b'.repeat(64)}`, markdownSamples: 'markdown/samples.json', markdownSummaryJson: 'markdown/summary/summary.json', markdownSummaryText: 'markdown/summary/summary.txt', From 946c7150ad1bd7bcd8e30268bb7ec1d40f8a814e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 24 Aug 2026 07:30:12 -0700 Subject: [PATCH 157/209] fix(perf): bind suite manifest to run provenance --- benchmarks/run-current-suite.mjs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index e63398cf..4d9b912a 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -509,6 +509,10 @@ function suiteManifest(args, packageEvidence) { return Object.freeze({ contractVersion: 1, documentProfile: args.documentProfile, + sampleCount: Number(args.sampleCount), + sourceCommitSha: args.sourceCommitSha, + runtimeId: args.runtimeId, + referenceHardwareId: args.referenceHardwareId, ...(packageEvidence ?? {}), markdownSamples: 'markdown/samples.json', markdownSummaryJson: 'markdown/summary/summary.json', From 413b0f392da15e1bf672f0177950a3654e5271af Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 24 Aug 2026 22:14:56 -0700 Subject: [PATCH 158/209] test(perf): attest packed suite run provenance --- src/performancePackedArtifactSuiteContract.test.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts index 4d1f3949..d0a8db8a 100644 --- a/src/performancePackedArtifactSuiteContract.test.ts +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -16,6 +16,8 @@ const repositoryRoot = process.cwd(); const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); const temporaryDirectories: string[] = []; const activeRuntimeId = `node-${process.versions.node}`; +const sourceCommitSha = 'a'.repeat(40); +const referenceHardwareId = `refhw-sha256-${'b'.repeat(64)}`; afterEach(() => { for (const directory of temporaryDirectories.splice(0)) { @@ -115,18 +117,18 @@ function packedSuiteArguments(options: { '--samples', '2', '--source-commit-sha', - 'a'.repeat(40), + sourceCommitSha, '--runtime-id', options.runtimeId, '--reference-hardware-id', - `refhw-sha256-${'b'.repeat(64)}`, + referenceHardwareId, '--output', join(options.directory, 'evidence'), ]; } describe('packed artifact benchmark suite contract', () => { - it('binds one-command benchmark evidence to a packed npm artifact digest', () => { + it('binds one-command benchmark evidence to packed artifact and run provenance', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-benchmark-')); temporaryDirectories.push(directory); const packed = createPackedBenchmarkFixture(directory); @@ -151,6 +153,11 @@ describe('packed artifact benchmark suite contract', () => { expect(result.stderr).toBe(''); expect(JSON.parse(result.stdout.trim())).toMatchObject({ contractVersion: 1, + documentProfile: 'small', + sampleCount: 2, + sourceCommitSha, + runtimeId: activeRuntimeId, + referenceHardwareId, packageName: '@contextualwisdomlab/cwl-editor', packageVersion: '0.0.0-benchmark-fixture', packageSha256: packed.packageSha256, From f136388a7909cf31305ff02f84324eb942d2ceed Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 03:08:59 -0700 Subject: [PATCH 159/209] test(perf): bind benchmark source provenance to checkout --- ...ormancePackedArtifactSuiteContract.test.ts | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts index d0a8db8a..35098ef8 100644 --- a/src/performancePackedArtifactSuiteContract.test.ts +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -16,7 +16,11 @@ const repositoryRoot = process.cwd(); const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); const temporaryDirectories: string[] = []; const activeRuntimeId = `node-${process.versions.node}`; -const sourceCommitSha = 'a'.repeat(40); +const sourceCommitSha = execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], +}).trim(); const referenceHardwareId = `refhw-sha256-${'b'.repeat(64)}`; afterEach(() => { @@ -92,6 +96,7 @@ function packedSuiteArguments(options: { directory: string; packageSha256: string; runtimeId: string; + sourceCommitSha?: string; tarballPath: string; }): string[] { const markdownInputPath = join(options.directory, 'input.md'); @@ -117,7 +122,7 @@ function packedSuiteArguments(options: { '--samples', '2', '--source-commit-sha', - sourceCommitSha, + options.sourceCommitSha ?? sourceCommitSha, '--runtime-id', options.runtimeId, '--reference-hardware-id', @@ -192,4 +197,33 @@ describe('packed artifact benchmark suite contract', () => { 'Benchmark suite runtime ID must match the active Node runtime.\n', ); }); + + it('rejects source provenance that does not match the benchmark checkout', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-source-')); + temporaryDirectories.push(directory); + const packed = createPackedBenchmarkFixture(directory); + + const result = spawnSync( + process.execPath, + packedSuiteArguments({ + directory, + packageSha256: packed.packageSha256, + runtimeId: activeRuntimeId, + sourceCommitSha: '0'.repeat(40), + tarballPath: packed.tarballPath, + }), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 15_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite source commit SHA must match the current benchmark checkout.\n', + ); + }); }); From 9876531b0688be8b974b010d37ebdd191cdfd389 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 03:14:16 -0700 Subject: [PATCH 160/209] fix(perf): verify benchmark source checkout provenance --- benchmarks/run-current-suite.mjs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 4d9b912a..d9717fc3 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -45,6 +45,7 @@ const packedFlags = Object.freeze([ '--output', ]); const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const COMMIT_SHA_PATTERN = /^[0-9a-f]{40}$/u; const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; const MAX_PACKAGE_INDEX_BYTES = 1024 * 1024; const MAX_PACKAGE_MANIFEST_BYTES = 1024 * 1024; @@ -111,6 +112,28 @@ function measurementArguments({ ]); } +function currentCheckoutSha() { + const result = spawnSync('git', ['rev-parse', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + const checkoutSha = result.stdout?.trim(); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !COMMIT_SHA_PATTERN.test(checkoutSha ?? '') + ) { + throw new Error( + 'Benchmark suite source commit SHA could not be verified against the current checkout.', + ); + } + return checkoutSha; +} + function resolveArguments(argv) { if (matchesArguments(argv, packedFlags)) { const values = valuesForArguments(argv, packedFlags); @@ -126,6 +149,11 @@ function resolveArguments(argv) { 'Benchmark suite runtime ID must match the active Node runtime.', ); } + if (values['--source-commit-sha'] !== currentCheckoutSha()) { + throw new Error( + 'Benchmark suite source commit SHA must match the current benchmark checkout.', + ); + } return Object.freeze({ mode: 'packed', shared: sharedArguments(values), From 4a9661f1d0f6675473fe1c44a05949ff588c1083 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 04:12:07 -0700 Subject: [PATCH 161/209] fix(perf): reject dirty benchmark provenance --- benchmarks/run-current-suite-core.mjs | 617 ++++++++++++++++++ benchmarks/run-current-suite.mjs | 607 +---------------- benchmarks/source-checkout-provenance.mjs | 34 + ...ceSourceCheckoutProvenanceContract.test.ts | 40 ++ 4 files changed, 703 insertions(+), 595 deletions(-) create mode 100644 benchmarks/run-current-suite-core.mjs create mode 100644 benchmarks/source-checkout-provenance.mjs create mode 100644 src/performanceSourceCheckoutProvenanceContract.test.ts diff --git a/benchmarks/run-current-suite-core.mjs b/benchmarks/run-current-suite-core.mjs new file mode 100644 index 00000000..d9717fc3 --- /dev/null +++ b/benchmarks/run-current-suite-core.mjs @@ -0,0 +1,617 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdirSync, + mkdtempSync, + openSync, + readSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(benchmarkDirectory, '..'); +const legacyFlags = Object.freeze([ + '--input', + '--module', + '--revision-input', + '--revision-module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--revision-artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const packedFlags = Object.freeze([ + '--input', + '--revision-input', + '--package-tarball', + '--package-sha256', + '--profile', + '--samples', + '--source-commit-sha', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const COMMIT_SHA_PATTERN = /^[0-9a-f]{40}$/u; +const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; +const MAX_PACKAGE_INDEX_BYTES = 1024 * 1024; +const MAX_PACKAGE_MANIFEST_BYTES = 1024 * 1024; +const MAX_MODULE_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const EXPECTED_PACKAGE_NAME = '@contextualwisdomlab/cwl-editor'; +const PACKAGE_MANIFEST_ENTRY = 'package/package.json'; +const MARKDOWN_MODULE_ENTRY = 'package/dist/cwl-markdown.js'; +const REVISION_MODULE_ENTRY = 'package/dist/cwl-revision-evidence.js'; +const OUTPUT_DIRECTORY_ERROR = + 'Benchmark suite output directory must be a non-symlink directory.'; + +function matchesArguments(argv, expectedFlags) { + return ( + argv.length === expectedFlags.length * 2 && + expectedFlags.every((flag, index) => argv[index * 2] === flag) && + expectedFlags.every((_, index) => argv[index * 2 + 1]?.length > 0) + ); +} + +function valuesForArguments(argv, expectedFlags) { + return Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); +} + +function sharedArguments(values) { + return Object.freeze({ + documentProfile: values['--profile'], + sampleCount: values['--samples'], + sourceCommitSha: values['--source-commit-sha'], + runtimeId: values['--runtime-id'], + referenceHardwareId: values['--reference-hardware-id'], + markdownInputPath: values['--input'], + revisionInputPath: values['--revision-input'], + outputDirectory: resolve(values['--output']), + }); +} + +function measurementArguments({ + inputPath, + modulePath, + artifactSha256, + shared, +}) { + return Object.freeze([ + '--input', + inputPath, + '--module', + modulePath, + '--profile', + shared.documentProfile, + '--samples', + shared.sampleCount, + '--source-commit-sha', + shared.sourceCommitSha, + '--artifact-sha256', + artifactSha256, + '--runtime-id', + shared.runtimeId, + '--reference-hardware-id', + shared.referenceHardwareId, + ]); +} + +function currentCheckoutSha() { + const result = spawnSync('git', ['rev-parse', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + const checkoutSha = result.stdout?.trim(); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !COMMIT_SHA_PATTERN.test(checkoutSha ?? '') + ) { + throw new Error( + 'Benchmark suite source commit SHA could not be verified against the current checkout.', + ); + } + return checkoutSha; +} + +function resolveArguments(argv) { + if (matchesArguments(argv, packedFlags)) { + const values = valuesForArguments(argv, packedFlags); + const packageSha256 = values['--package-sha256']; + if (!SHA256_PATTERN.test(packageSha256)) { + throw new Error( + 'Benchmark suite package digest must be a lowercase 64-character SHA-256.', + ); + } + const activeRuntimeId = `node-${process.versions.node}`; + if (values['--runtime-id'] !== activeRuntimeId) { + throw new Error( + 'Benchmark suite runtime ID must match the active Node runtime.', + ); + } + if (values['--source-commit-sha'] !== currentCheckoutSha()) { + throw new Error( + 'Benchmark suite source commit SHA must match the current benchmark checkout.', + ); + } + return Object.freeze({ + mode: 'packed', + shared: sharedArguments(values), + packageTarballPath: resolve(values['--package-tarball']), + packageSha256, + }); + } + + if (matchesArguments(argv, legacyFlags)) { + const values = valuesForArguments(argv, legacyFlags); + const shared = sharedArguments(values); + return Object.freeze({ + mode: 'module', + shared, + markdownArguments: measurementArguments({ + inputPath: shared.markdownInputPath, + modulePath: values['--module'], + artifactSha256: values['--artifact-sha256'], + shared, + }), + revisionArguments: measurementArguments({ + inputPath: shared.revisionInputPath, + modulePath: values['--revision-module'], + artifactSha256: values['--revision-artifact-sha256'], + shared, + }), + packageEvidence: null, + }); + } + + throw new Error( + 'Usage: node benchmarks/run-current-suite.mjs --input --revision-input --package-tarball --package-sha256 --profile --samples --source-commit-sha --runtime-id --reference-hardware-id --output ', + ); +} + +function inspectOutputDirectory(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark suite output directory could not be inspected.'); + } +} + +function assertNoSymlinkDirectoryComponents(path) { + let current = path; + while (true) { + const metadata = inspectOutputDirectory(current); + if (metadata?.isSymbolicLink()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + +function prepareOutputDirectory(path) { + assertNoSymlinkDirectoryComponents(path); + const existing = inspectOutputDirectory(path); + if (existing !== undefined) { + if (!existing.isDirectory()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + return false; + } + + try { + mkdirSync(path, { recursive: true }); + } catch { + throw new Error('Benchmark suite output directory could not be prepared.'); + } + + assertNoSymlinkDirectoryComponents(path); + const created = inspectOutputDirectory(path); + if (created === undefined || !created.isDirectory()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + return true; +} + +function removePartialOutputDirectory(path) { + try { + rmSync(path, { recursive: true, force: true }); + } catch { + throw new Error('Benchmark suite partial evidence could not be removed.'); + } +} + +function readBoundedRegularFile(path, maximumBytes, invalidMessage, oversizedMessage) { + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error(invalidMessage); + } + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error(invalidMessage); + } + + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error(invalidMessage); + } + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) throw new Error(invalidMessage); + if (metadata.size > maximumBytes) throw new Error(oversizedMessage); + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= maximumBytes) { + const remainingBudget = maximumBytes + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > maximumBytes) throw new Error(oversizedMessage); + chunks.push(chunk.subarray(0, bytesRead)); + } + return Buffer.concat(chunks, totalBytes); + } finally { + closeSync(descriptor); + } +} + +function packageTarballBytes(path) { + return readBoundedRegularFile( + path, + MAX_PACKAGE_BYTES, + 'Benchmark suite package tarball must be a regular non-symlink file.', + 'Benchmark suite package tarball exceeds the supported size.', + ); +} + +function verifyPackageDigest(path, expectedSha256) { + const actualSha256 = createHash('sha256') + .update(packageTarballBytes(path)) + .digest('hex'); + if (actualSha256 !== expectedSha256) { + throw new Error('Benchmark suite package digest does not match the packed artifact.'); + } +} + +function runTar(argumentsList, maximumBytes, failureMessage) { + const result = spawnSync('tar', argumentsList, { + cwd: repositoryRoot, + maxBuffer: maximumBytes, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 30_000, + }); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !Buffer.isBuffer(result.stdout) || + result.stdout.byteLength > maximumBytes + ) { + throw new Error(failureMessage); + } + return result.stdout; +} + +function decodeUtf8(bytes, failureMessage) { + try { + return new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch { + throw new Error(failureMessage); + } +} + +function listPackedEntries(tarballPath) { + const listing = decodeUtf8( + runTar( + ['-tzf', tarballPath], + MAX_PACKAGE_INDEX_BYTES, + 'Benchmark suite package index could not be read.', + ), + 'Benchmark suite package index must be valid UTF-8.', + ); + return listing.split('\n').filter((entry) => entry.length > 0); +} + +function assertUniquePackedEntry(entries, expectedEntry) { + if (entries.filter((entry) => entry === expectedEntry).length !== 1) { + throw new Error('Benchmark suite package is missing a unique required artifact.'); + } +} + +function readPackedEntry(tarballPath, entry, maximumBytes) { + return runTar( + ['-xOzf', tarballPath, entry], + maximumBytes, + 'Benchmark suite package artifact could not be read.', + ); +} + +function parsePackageManifest(bytes) { + let manifest; + try { + manifest = JSON.parse( + decodeUtf8(bytes, 'Benchmark suite package manifest must be valid UTF-8.'), + ); + } catch (error) { + if (error instanceof Error && error.message.includes('valid UTF-8')) throw error; + throw new Error('Benchmark suite package manifest must be valid JSON.'); + } + if ( + manifest === null || + typeof manifest !== 'object' || + Array.isArray(manifest) || + manifest.name !== EXPECTED_PACKAGE_NAME || + typeof manifest.version !== 'string' || + manifest.version.length === 0 || + manifest.version.length > 128 + ) { + throw new Error('Benchmark suite package identity is invalid.'); + } + return Object.freeze({ name: manifest.name, version: manifest.version }); +} + +function moduleSha256(bytes) { + return createHash('sha256').update(bytes).digest('hex'); +} + +function preparePackedBenchmarkModules(args) { + verifyPackageDigest(args.packageTarballPath, args.packageSha256); + const entries = listPackedEntries(args.packageTarballPath); + for (const entry of [ + PACKAGE_MANIFEST_ENTRY, + MARKDOWN_MODULE_ENTRY, + REVISION_MODULE_ENTRY, + ]) { + assertUniquePackedEntry(entries, entry); + } + + const manifestBytes = readPackedEntry( + args.packageTarballPath, + PACKAGE_MANIFEST_ENTRY, + MAX_PACKAGE_MANIFEST_BYTES, + ); + const manifest = parsePackageManifest(manifestBytes); + const markdownModuleBytes = readPackedEntry( + args.packageTarballPath, + MARKDOWN_MODULE_ENTRY, + MAX_MODULE_BYTES, + ); + const revisionModuleBytes = readPackedEntry( + args.packageTarballPath, + REVISION_MODULE_ENTRY, + MAX_MODULE_BYTES, + ); + verifyPackageDigest(args.packageTarballPath, args.packageSha256); + + const temporaryDirectory = mkdtempSync( + join(tmpdir(), 'inkspan-packed-benchmark-'), + ); + const markdownModulePath = join(temporaryDirectory, 'cwl-markdown.mjs'); + const revisionModulePath = join( + temporaryDirectory, + 'cwl-revision-evidence.mjs', + ); + try { + writeFileSync(markdownModulePath, markdownModuleBytes); + writeFileSync(revisionModulePath, revisionModuleBytes); + } catch { + rmSync(temporaryDirectory, { recursive: true, force: true }); + throw new Error('Benchmark suite package modules could not be prepared.'); + } + + return Object.freeze({ + temporaryDirectory, + markdownModulePath, + markdownArtifactSha256: moduleSha256(markdownModuleBytes), + revisionModulePath, + revisionArtifactSha256: moduleSha256(revisionModuleBytes), + packageEvidence: Object.freeze({ + packageName: manifest.name, + packageVersion: manifest.version, + packageSha256: args.packageSha256, + }), + }); +} + +function runBoundedNodeScript(scriptName, args, failureMessage) { + const result = spawnSync( + process.execPath, + [resolve(benchmarkDirectory, scriptName), ...args], + { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 4 * 1024 * 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 120_000, + }, + ); + + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 + ) { + throw new Error(failureMessage); + } +} + +function runMeasurementAndSummary({ + measurementScript, + measurementArguments: argumentsList, + samplesPath, + summaryDirectory, + measurementFailure, + summaryFailure, +}) { + runBoundedNodeScript( + measurementScript, + [...argumentsList, '--output', samplesPath], + measurementFailure, + ); + runBoundedNodeScript( + 'summarize-samples.mjs', + ['--input', samplesPath, '--output', summaryDirectory], + summaryFailure, + ); +} + +function runSuite(args, markdownArguments, revisionArguments) { + const markdownSamplesPath = resolve( + args.outputDirectory, + 'markdown', + 'samples.json', + ); + const markdownSummaryDirectory = resolve( + args.outputDirectory, + 'markdown', + 'summary', + ); + const revisionSamplesPath = resolve( + args.outputDirectory, + 'revision', + 'samples.json', + ); + const revisionSummaryDirectory = resolve( + args.outputDirectory, + 'revision', + 'summary', + ); + + runMeasurementAndSummary({ + measurementScript: 'measure-markdown.mjs', + measurementArguments: markdownArguments, + samplesPath: markdownSamplesPath, + summaryDirectory: markdownSummaryDirectory, + measurementFailure: 'Benchmark suite Markdown measurement failed.', + summaryFailure: 'Benchmark suite Markdown summary failed.', + }); + runMeasurementAndSummary({ + measurementScript: 'measure-revision-evidence.mjs', + measurementArguments: revisionArguments, + samplesPath: revisionSamplesPath, + summaryDirectory: revisionSummaryDirectory, + measurementFailure: 'Benchmark suite revision measurement failed.', + summaryFailure: 'Benchmark suite revision summary failed.', + }); +} + +function suiteManifest(args, packageEvidence) { + return Object.freeze({ + contractVersion: 1, + documentProfile: args.documentProfile, + sampleCount: Number(args.sampleCount), + sourceCommitSha: args.sourceCommitSha, + runtimeId: args.runtimeId, + referenceHardwareId: args.referenceHardwareId, + ...(packageEvidence ?? {}), + markdownSamples: 'markdown/samples.json', + markdownSummaryJson: 'markdown/summary/summary.json', + markdownSummaryText: 'markdown/summary/summary.txt', + revisionSamples: 'revision/samples.json', + revisionSummaryJson: 'revision/summary/summary.json', + revisionSummaryText: 'revision/summary/summary.txt', + status: 'completed', + }); +} + +function main(argv) { + const resolved = resolveArguments(argv); + const shared = resolved.shared; + let preparedPackage; + if (resolved.mode === 'packed') { + preparedPackage = preparePackedBenchmarkModules(resolved); + } + + const markdownArguments = + resolved.mode === 'packed' + ? measurementArguments({ + inputPath: shared.markdownInputPath, + modulePath: preparedPackage.markdownModulePath, + artifactSha256: preparedPackage.markdownArtifactSha256, + shared, + }) + : resolved.markdownArguments; + const revisionArguments = + resolved.mode === 'packed' + ? measurementArguments({ + inputPath: shared.revisionInputPath, + modulePath: preparedPackage.revisionModulePath, + artifactSha256: preparedPackage.revisionArtifactSha256, + shared, + }) + : resolved.revisionArguments; + const packageEvidence = + resolved.mode === 'packed' ? preparedPackage.packageEvidence : null; + + let createdOutputDirectory = false; + try { + createdOutputDirectory = prepareOutputDirectory(shared.outputDirectory); + runSuite(shared, markdownArguments, revisionArguments); + if (resolved.mode === 'packed') { + verifyPackageDigest(resolved.packageTarballPath, resolved.packageSha256); + } + } catch (error) { + if (createdOutputDirectory) { + removePartialOutputDirectory(shared.outputDirectory); + } + throw error; + } finally { + if (preparedPackage !== undefined) { + rmSync(preparedPackage.temporaryDirectory, { + recursive: true, + force: true, + }); + } + } + + process.stdout.write( + `${JSON.stringify(suiteManifest(shared, packageEvidence))}\n`, + ); +} + +try { + main(process.argv.slice(2)); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Benchmark suite failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index d9717fc3..cc87a585 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -1,610 +1,27 @@ -import { createHash } from 'node:crypto'; import { spawnSync } from 'node:child_process'; -import { - closeSync, - constants, - fstatSync, - lstatSync, - mkdirSync, - mkdtempSync, - openSync, - readSync, - rmSync, - writeFileSync, -} from 'node:fs'; -import { tmpdir } from 'node:os'; -import { dirname, join, resolve } from 'node:path'; +import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { assertCleanSourceCheckout } from './source-checkout-provenance.mjs'; + const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); const repositoryRoot = resolve(benchmarkDirectory, '..'); -const legacyFlags = Object.freeze([ - '--input', - '--module', - '--revision-input', - '--revision-module', - '--profile', - '--samples', - '--source-commit-sha', - '--artifact-sha256', - '--revision-artifact-sha256', - '--runtime-id', - '--reference-hardware-id', - '--output', -]); -const packedFlags = Object.freeze([ - '--input', - '--revision-input', - '--package-tarball', - '--package-sha256', - '--profile', - '--samples', - '--source-commit-sha', - '--runtime-id', - '--reference-hardware-id', - '--output', -]); -const SHA256_PATTERN = /^[0-9a-f]{64}$/u; -const COMMIT_SHA_PATTERN = /^[0-9a-f]{40}$/u; -const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; -const MAX_PACKAGE_INDEX_BYTES = 1024 * 1024; -const MAX_PACKAGE_MANIFEST_BYTES = 1024 * 1024; -const MAX_MODULE_BYTES = 16 * 1024 * 1024; -const READ_CHUNK_BYTES = 64 * 1024; -const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); -const EXPECTED_PACKAGE_NAME = '@contextualwisdomlab/cwl-editor'; -const PACKAGE_MANIFEST_ENTRY = 'package/package.json'; -const MARKDOWN_MODULE_ENTRY = 'package/dist/cwl-markdown.js'; -const REVISION_MODULE_ENTRY = 'package/dist/cwl-revision-evidence.js'; -const OUTPUT_DIRECTORY_ERROR = - 'Benchmark suite output directory must be a non-symlink directory.'; - -function matchesArguments(argv, expectedFlags) { - return ( - argv.length === expectedFlags.length * 2 && - expectedFlags.every((flag, index) => argv[index * 2] === flag) && - expectedFlags.every((_, index) => argv[index * 2 + 1]?.length > 0) - ); -} - -function valuesForArguments(argv, expectedFlags) { - return Object.fromEntries( - expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), - ); -} - -function sharedArguments(values) { - return Object.freeze({ - documentProfile: values['--profile'], - sampleCount: values['--samples'], - sourceCommitSha: values['--source-commit-sha'], - runtimeId: values['--runtime-id'], - referenceHardwareId: values['--reference-hardware-id'], - markdownInputPath: values['--input'], - revisionInputPath: values['--revision-input'], - outputDirectory: resolve(values['--output']), - }); -} - -function measurementArguments({ - inputPath, - modulePath, - artifactSha256, - shared, -}) { - return Object.freeze([ - '--input', - inputPath, - '--module', - modulePath, - '--profile', - shared.documentProfile, - '--samples', - shared.sampleCount, - '--source-commit-sha', - shared.sourceCommitSha, - '--artifact-sha256', - artifactSha256, - '--runtime-id', - shared.runtimeId, - '--reference-hardware-id', - shared.referenceHardwareId, - ]); -} - -function currentCheckoutSha() { - const result = spawnSync('git', ['rev-parse', 'HEAD'], { - cwd: repositoryRoot, - encoding: 'utf8', - maxBuffer: 1024, - stdio: ['ignore', 'pipe', 'pipe'], - timeout: 10_000, - }); - const checkoutSha = result.stdout?.trim(); - if ( - result.error !== undefined || - result.signal !== null || - result.status !== 0 || - !COMMIT_SHA_PATTERN.test(checkoutSha ?? '') - ) { - throw new Error( - 'Benchmark suite source commit SHA could not be verified against the current checkout.', - ); - } - return checkoutSha; -} - -function resolveArguments(argv) { - if (matchesArguments(argv, packedFlags)) { - const values = valuesForArguments(argv, packedFlags); - const packageSha256 = values['--package-sha256']; - if (!SHA256_PATTERN.test(packageSha256)) { - throw new Error( - 'Benchmark suite package digest must be a lowercase 64-character SHA-256.', - ); - } - const activeRuntimeId = `node-${process.versions.node}`; - if (values['--runtime-id'] !== activeRuntimeId) { - throw new Error( - 'Benchmark suite runtime ID must match the active Node runtime.', - ); - } - if (values['--source-commit-sha'] !== currentCheckoutSha()) { - throw new Error( - 'Benchmark suite source commit SHA must match the current benchmark checkout.', - ); - } - return Object.freeze({ - mode: 'packed', - shared: sharedArguments(values), - packageTarballPath: resolve(values['--package-tarball']), - packageSha256, - }); - } - - if (matchesArguments(argv, legacyFlags)) { - const values = valuesForArguments(argv, legacyFlags); - const shared = sharedArguments(values); - return Object.freeze({ - mode: 'module', - shared, - markdownArguments: measurementArguments({ - inputPath: shared.markdownInputPath, - modulePath: values['--module'], - artifactSha256: values['--artifact-sha256'], - shared, - }), - revisionArguments: measurementArguments({ - inputPath: shared.revisionInputPath, - modulePath: values['--revision-module'], - artifactSha256: values['--revision-artifact-sha256'], - shared, - }), - packageEvidence: null, - }); - } - - throw new Error( - 'Usage: node benchmarks/run-current-suite.mjs --input --revision-input --package-tarball --package-sha256 --profile --samples --source-commit-sha --runtime-id --reference-hardware-id --output ', - ); -} - -function inspectOutputDirectory(path) { - try { - return lstatSync(path, { throwIfNoEntry: false }); - } catch { - throw new Error('Benchmark suite output directory could not be inspected.'); - } -} - -function assertNoSymlinkDirectoryComponents(path) { - let current = path; - while (true) { - const metadata = inspectOutputDirectory(current); - if (metadata?.isSymbolicLink()) { - throw new Error(OUTPUT_DIRECTORY_ERROR); - } - const parent = dirname(current); - if (parent === current) return; - current = parent; - } -} - -function prepareOutputDirectory(path) { - assertNoSymlinkDirectoryComponents(path); - const existing = inspectOutputDirectory(path); - if (existing !== undefined) { - if (!existing.isDirectory()) { - throw new Error(OUTPUT_DIRECTORY_ERROR); - } - return false; - } - - try { - mkdirSync(path, { recursive: true }); - } catch { - throw new Error('Benchmark suite output directory could not be prepared.'); - } - - assertNoSymlinkDirectoryComponents(path); - const created = inspectOutputDirectory(path); - if (created === undefined || !created.isDirectory()) { - throw new Error(OUTPUT_DIRECTORY_ERROR); - } - return true; -} - -function removePartialOutputDirectory(path) { - try { - rmSync(path, { recursive: true, force: true }); - } catch { - throw new Error('Benchmark suite partial evidence could not be removed.'); - } -} - -function readBoundedRegularFile(path, maximumBytes, invalidMessage, oversizedMessage) { - let pathMetadata; - try { - pathMetadata = lstatSync(path, { throwIfNoEntry: false }); - } catch { - throw new Error(invalidMessage); - } - if ( - pathMetadata === undefined || - pathMetadata.isSymbolicLink() || - !pathMetadata.isFile() - ) { - throw new Error(invalidMessage); - } - - let descriptor; - try { - descriptor = openSync(path, READ_ONLY_NOFOLLOW); - } catch { - throw new Error(invalidMessage); - } - try { - const metadata = fstatSync(descriptor); - if (!metadata.isFile()) throw new Error(invalidMessage); - if (metadata.size > maximumBytes) throw new Error(oversizedMessage); - - const chunks = []; - let totalBytes = 0; - while (totalBytes <= maximumBytes) { - const remainingBudget = maximumBytes + 1 - totalBytes; - const chunk = Buffer.allocUnsafe( - Math.min(READ_CHUNK_BYTES, remainingBudget), - ); - const bytesRead = readSync( - descriptor, - chunk, - 0, - chunk.byteLength, - null, - ); - if (bytesRead === 0) break; - totalBytes += bytesRead; - if (totalBytes > maximumBytes) throw new Error(oversizedMessage); - chunks.push(chunk.subarray(0, bytesRead)); - } - return Buffer.concat(chunks, totalBytes); - } finally { - closeSync(descriptor); - } -} - -function packageTarballBytes(path) { - return readBoundedRegularFile( - path, - MAX_PACKAGE_BYTES, - 'Benchmark suite package tarball must be a regular non-symlink file.', - 'Benchmark suite package tarball exceeds the supported size.', - ); -} +const coreRunnerPath = resolve(benchmarkDirectory, 'run-current-suite-core.mjs'); -function verifyPackageDigest(path, expectedSha256) { - const actualSha256 = createHash('sha256') - .update(packageTarballBytes(path)) - .digest('hex'); - if (actualSha256 !== expectedSha256) { - throw new Error('Benchmark suite package digest does not match the packed artifact.'); - } -} +function main(argv) { + assertCleanSourceCheckout(repositoryRoot); -function runTar(argumentsList, maximumBytes, failureMessage) { - const result = spawnSync('tar', argumentsList, { + const result = spawnSync(process.execPath, [coreRunnerPath, ...argv], { cwd: repositoryRoot, - maxBuffer: maximumBytes, - stdio: ['ignore', 'pipe', 'pipe'], - timeout: 30_000, + stdio: 'inherit', + timeout: 600_000, }); - if ( - result.error !== undefined || - result.signal !== null || - result.status !== 0 || - !Buffer.isBuffer(result.stdout) || - result.stdout.byteLength > maximumBytes - ) { - throw new Error(failureMessage); - } - return result.stdout; -} - -function decodeUtf8(bytes, failureMessage) { - try { - return new TextDecoder('utf-8', { fatal: true }).decode(bytes); - } catch { - throw new Error(failureMessage); - } -} - -function listPackedEntries(tarballPath) { - const listing = decodeUtf8( - runTar( - ['-tzf', tarballPath], - MAX_PACKAGE_INDEX_BYTES, - 'Benchmark suite package index could not be read.', - ), - 'Benchmark suite package index must be valid UTF-8.', - ); - return listing.split('\n').filter((entry) => entry.length > 0); -} - -function assertUniquePackedEntry(entries, expectedEntry) { - if (entries.filter((entry) => entry === expectedEntry).length !== 1) { - throw new Error('Benchmark suite package is missing a unique required artifact.'); - } -} - -function readPackedEntry(tarballPath, entry, maximumBytes) { - return runTar( - ['-xOzf', tarballPath, entry], - maximumBytes, - 'Benchmark suite package artifact could not be read.', - ); -} - -function parsePackageManifest(bytes) { - let manifest; - try { - manifest = JSON.parse( - decodeUtf8(bytes, 'Benchmark suite package manifest must be valid UTF-8.'), - ); - } catch (error) { - if (error instanceof Error && error.message.includes('valid UTF-8')) throw error; - throw new Error('Benchmark suite package manifest must be valid JSON.'); - } - if ( - manifest === null || - typeof manifest !== 'object' || - Array.isArray(manifest) || - manifest.name !== EXPECTED_PACKAGE_NAME || - typeof manifest.version !== 'string' || - manifest.version.length === 0 || - manifest.version.length > 128 - ) { - throw new Error('Benchmark suite package identity is invalid.'); - } - return Object.freeze({ name: manifest.name, version: manifest.version }); -} - -function moduleSha256(bytes) { - return createHash('sha256').update(bytes).digest('hex'); -} - -function preparePackedBenchmarkModules(args) { - verifyPackageDigest(args.packageTarballPath, args.packageSha256); - const entries = listPackedEntries(args.packageTarballPath); - for (const entry of [ - PACKAGE_MANIFEST_ENTRY, - MARKDOWN_MODULE_ENTRY, - REVISION_MODULE_ENTRY, - ]) { - assertUniquePackedEntry(entries, entry); - } - - const manifestBytes = readPackedEntry( - args.packageTarballPath, - PACKAGE_MANIFEST_ENTRY, - MAX_PACKAGE_MANIFEST_BYTES, - ); - const manifest = parsePackageManifest(manifestBytes); - const markdownModuleBytes = readPackedEntry( - args.packageTarballPath, - MARKDOWN_MODULE_ENTRY, - MAX_MODULE_BYTES, - ); - const revisionModuleBytes = readPackedEntry( - args.packageTarballPath, - REVISION_MODULE_ENTRY, - MAX_MODULE_BYTES, - ); - verifyPackageDigest(args.packageTarballPath, args.packageSha256); - - const temporaryDirectory = mkdtempSync( - join(tmpdir(), 'inkspan-packed-benchmark-'), - ); - const markdownModulePath = join(temporaryDirectory, 'cwl-markdown.mjs'); - const revisionModulePath = join( - temporaryDirectory, - 'cwl-revision-evidence.mjs', - ); - try { - writeFileSync(markdownModulePath, markdownModuleBytes); - writeFileSync(revisionModulePath, revisionModuleBytes); - } catch { - rmSync(temporaryDirectory, { recursive: true, force: true }); - throw new Error('Benchmark suite package modules could not be prepared.'); - } - - return Object.freeze({ - temporaryDirectory, - markdownModulePath, - markdownArtifactSha256: moduleSha256(markdownModuleBytes), - revisionModulePath, - revisionArtifactSha256: moduleSha256(revisionModuleBytes), - packageEvidence: Object.freeze({ - packageName: manifest.name, - packageVersion: manifest.version, - packageSha256: args.packageSha256, - }), - }); -} - -function runBoundedNodeScript(scriptName, args, failureMessage) { - const result = spawnSync( - process.execPath, - [resolve(benchmarkDirectory, scriptName), ...args], - { - cwd: repositoryRoot, - encoding: 'utf8', - maxBuffer: 4 * 1024 * 1024, - stdio: ['ignore', 'pipe', 'pipe'], - timeout: 120_000, - }, - ); - - if ( - result.error !== undefined || - result.signal !== null || - result.status !== 0 - ) { - throw new Error(failureMessage); - } -} - -function runMeasurementAndSummary({ - measurementScript, - measurementArguments: argumentsList, - samplesPath, - summaryDirectory, - measurementFailure, - summaryFailure, -}) { - runBoundedNodeScript( - measurementScript, - [...argumentsList, '--output', samplesPath], - measurementFailure, - ); - runBoundedNodeScript( - 'summarize-samples.mjs', - ['--input', samplesPath, '--output', summaryDirectory], - summaryFailure, - ); -} - -function runSuite(args, markdownArguments, revisionArguments) { - const markdownSamplesPath = resolve( - args.outputDirectory, - 'markdown', - 'samples.json', - ); - const markdownSummaryDirectory = resolve( - args.outputDirectory, - 'markdown', - 'summary', - ); - const revisionSamplesPath = resolve( - args.outputDirectory, - 'revision', - 'samples.json', - ); - const revisionSummaryDirectory = resolve( - args.outputDirectory, - 'revision', - 'summary', - ); - - runMeasurementAndSummary({ - measurementScript: 'measure-markdown.mjs', - measurementArguments: markdownArguments, - samplesPath: markdownSamplesPath, - summaryDirectory: markdownSummaryDirectory, - measurementFailure: 'Benchmark suite Markdown measurement failed.', - summaryFailure: 'Benchmark suite Markdown summary failed.', - }); - runMeasurementAndSummary({ - measurementScript: 'measure-revision-evidence.mjs', - measurementArguments: revisionArguments, - samplesPath: revisionSamplesPath, - summaryDirectory: revisionSummaryDirectory, - measurementFailure: 'Benchmark suite revision measurement failed.', - summaryFailure: 'Benchmark suite revision summary failed.', - }); -} - -function suiteManifest(args, packageEvidence) { - return Object.freeze({ - contractVersion: 1, - documentProfile: args.documentProfile, - sampleCount: Number(args.sampleCount), - sourceCommitSha: args.sourceCommitSha, - runtimeId: args.runtimeId, - referenceHardwareId: args.referenceHardwareId, - ...(packageEvidence ?? {}), - markdownSamples: 'markdown/samples.json', - markdownSummaryJson: 'markdown/summary/summary.json', - markdownSummaryText: 'markdown/summary/summary.txt', - revisionSamples: 'revision/samples.json', - revisionSummaryJson: 'revision/summary/summary.json', - revisionSummaryText: 'revision/summary/summary.txt', - status: 'completed', - }); -} - -function main(argv) { - const resolved = resolveArguments(argv); - const shared = resolved.shared; - let preparedPackage; - if (resolved.mode === 'packed') { - preparedPackage = preparePackedBenchmarkModules(resolved); - } - - const markdownArguments = - resolved.mode === 'packed' - ? measurementArguments({ - inputPath: shared.markdownInputPath, - modulePath: preparedPackage.markdownModulePath, - artifactSha256: preparedPackage.markdownArtifactSha256, - shared, - }) - : resolved.markdownArguments; - const revisionArguments = - resolved.mode === 'packed' - ? measurementArguments({ - inputPath: shared.revisionInputPath, - modulePath: preparedPackage.revisionModulePath, - artifactSha256: preparedPackage.revisionArtifactSha256, - shared, - }) - : resolved.revisionArguments; - const packageEvidence = - resolved.mode === 'packed' ? preparedPackage.packageEvidence : null; - let createdOutputDirectory = false; - try { - createdOutputDirectory = prepareOutputDirectory(shared.outputDirectory); - runSuite(shared, markdownArguments, revisionArguments); - if (resolved.mode === 'packed') { - verifyPackageDigest(resolved.packageTarballPath, resolved.packageSha256); - } - } catch (error) { - if (createdOutputDirectory) { - removePartialOutputDirectory(shared.outputDirectory); - } - throw error; - } finally { - if (preparedPackage !== undefined) { - rmSync(preparedPackage.temporaryDirectory, { - recursive: true, - force: true, - }); - } + if (result.error !== undefined || result.signal !== null) { + throw new Error('Benchmark suite internal runner could not complete.'); } - process.stdout.write( - `${JSON.stringify(suiteManifest(shared, packageEvidence))}\n`, - ); + process.exitCode = result.status ?? 1; } try { diff --git a/benchmarks/source-checkout-provenance.mjs b/benchmarks/source-checkout-provenance.mjs new file mode 100644 index 00000000..e2362381 --- /dev/null +++ b/benchmarks/source-checkout-provenance.mjs @@ -0,0 +1,34 @@ +import { spawnSync } from 'node:child_process'; + +const MAX_STATUS_BYTES = 1024 * 1024; + +export function assertCleanSourceCheckout(repositoryRoot) { + const result = spawnSync( + 'git', + ['status', '--porcelain=v1', '--untracked-files=all'], + { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: MAX_STATUS_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + typeof result.stdout !== 'string' + ) { + throw new Error( + 'Benchmark suite source checkout cleanliness could not be verified.', + ); + } + + if (result.stdout.length !== 0) { + throw new Error( + 'Benchmark suite source checkout must be clean before acquisition evidence is recorded.', + ); + } +} diff --git a/src/performanceSourceCheckoutProvenanceContract.test.ts b/src/performanceSourceCheckoutProvenanceContract.test.ts new file mode 100644 index 00000000..52ed1e3f --- /dev/null +++ b/src/performanceSourceCheckoutProvenanceContract.test.ts @@ -0,0 +1,40 @@ +import { spawnSync } from 'node:child_process'; +import { rmSync, writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const dirtySentinelPath = resolve( + repositoryRoot, + `.inkspan-benchmark-dirty-provenance-${process.pid}`, +); + +afterEach(() => { + rmSync(dirtySentinelPath, { force: true }); +}); + +describe('benchmark source checkout provenance', () => { + it('rejects untracked source state before acquisition evidence can run', () => { + writeFileSync(dirtySentinelPath, 'untracked benchmark provenance sentinel\n'); + + const result = spawnSync(process.execPath, [suitePath], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024 * 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + + expect(result.error).toBeUndefined(); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite source checkout must be clean before acquisition evidence is recorded.\n', + ); + expect(result.stderr).not.toContain('Usage:'); + expect(result.stderr).not.toContain(dirtySentinelPath); + }); +}); From b376877eb971f0070956c3bf8ef8c88dc0b71150 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 04:14:43 -0700 Subject: [PATCH 162/209] test(perf): isolate checkout provenance regression --- ...ceSourceCheckoutProvenanceContract.test.ts | 83 +++++++++++++++---- 1 file changed, 67 insertions(+), 16 deletions(-) diff --git a/src/performanceSourceCheckoutProvenanceContract.test.ts b/src/performanceSourceCheckoutProvenanceContract.test.ts index 52ed1e3f..94f37cc2 100644 --- a/src/performanceSourceCheckoutProvenanceContract.test.ts +++ b/src/performanceSourceCheckoutProvenanceContract.test.ts @@ -1,31 +1,82 @@ -import { spawnSync } from 'node:child_process'; -import { rmSync, writeFileSync } from 'node:fs'; -import { resolve } from 'node:path'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; import { afterEach, describe, expect, it } from 'vitest'; const repositoryRoot = process.cwd(); -const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); -const dirtySentinelPath = resolve( - repositoryRoot, - `.inkspan-benchmark-dirty-provenance-${process.pid}`, -); +const helperUrl = pathToFileURL( + resolve(repositoryRoot, 'benchmarks/source-checkout-provenance.mjs'), +).href; +const temporaryDirectories: string[] = []; + +const probe = ` +import { assertCleanSourceCheckout } from ${JSON.stringify(helperUrl)}; +try { + assertCleanSourceCheckout(process.argv[1]); + process.stdout.write('clean\\n'); +} catch (error) { + process.stderr.write(\`${'${error instanceof Error ? error.message : "verification failed"}'}\\n\`); + process.exitCode = 1; +} +`; afterEach(() => { - rmSync(dirtySentinelPath, { force: true }); + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } }); -describe('benchmark source checkout provenance', () => { - it('rejects untracked source state before acquisition evidence can run', () => { - writeFileSync(dirtySentinelPath, 'untracked benchmark provenance sentinel\n'); +function createRepository(): string { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-provenance-')); + temporaryDirectories.push(directory); + execFileSync('git', ['init', '--quiet'], { cwd: directory }); + execFileSync('git', ['config', 'user.email', 'test@example.invalid'], { + cwd: directory, + }); + execFileSync('git', ['config', 'user.name', 'Inkspan Test'], { + cwd: directory, + }); + writeFileSync(join(directory, 'tracked.txt'), 'committed\n'); + execFileSync('git', ['add', 'tracked.txt'], { cwd: directory }); + execFileSync('git', ['commit', '--quiet', '-m', 'fixture'], { cwd: directory }); + return directory; +} - const result = spawnSync(process.execPath, [suitePath], { +function probeCheckout(directory: string) { + return spawnSync( + process.execPath, + ['--input-type=module', '--eval', probe, directory], + { cwd: repositoryRoot, encoding: 'utf8', maxBuffer: 1024 * 1024, stdio: ['ignore', 'pipe', 'pipe'], timeout: 10_000, - }); + }, + ); +} + +describe('benchmark source checkout provenance', () => { + it('accepts a clean source checkout', () => { + const directory = createRepository(); + const result = probeCheckout(directory); + + expect(result.error).toBeUndefined(); + expect(result.signal).toBeNull(); + expect(result.status).toBe(0); + expect(result.stdout).toBe('clean\n'); + expect(result.stderr).toBe(''); + }); + + it('rejects untracked source state without disclosing paths', () => { + const directory = createRepository(); + const untrackedPath = join(directory, 'untracked-secret-name.txt'); + writeFileSync(untrackedPath, 'not part of the committed source\n'); + + const result = probeCheckout(directory); expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); @@ -34,7 +85,7 @@ describe('benchmark source checkout provenance', () => { expect(result.stderr).toBe( 'Benchmark suite source checkout must be clean before acquisition evidence is recorded.\n', ); - expect(result.stderr).not.toContain('Usage:'); - expect(result.stderr).not.toContain(dirtySentinelPath); + expect(result.stderr).not.toContain(untrackedPath); + expect(result.stderr).not.toContain('untracked-secret-name.txt'); }); }); From 8b7b6df06f4d3d8574460acdbd6993c5ee0a203e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 10:40:19 -0700 Subject: [PATCH 163/209] test(perf): require Office duration and RSS evidence --- office/tests/test_performance_measurement.py | 111 +++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 office/tests/test_performance_measurement.py diff --git a/office/tests/test_performance_measurement.py b/office/tests/test_performance_measurement.py new file mode 100644 index 00000000..94c14d7d --- /dev/null +++ b/office/tests/test_performance_measurement.py @@ -0,0 +1,111 @@ +"""Contract tests for privacy-safe Office render performance evidence.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + + +def test_measure_office_render_records_duration_peak_rss_and_provenance(tmp_path: Path) -> None: + """Measure repeated synthetic renders without copying document content into evidence.""" + + sentinel = "PRIVATE-BENCHMARK-PAYLOAD-SENTINEL" + request_path = tmp_path / "buyer-private-name.json" + request_path.write_text( + json.dumps( + { + "format": "docx", + "title": "Synthetic benchmark fixture", + "blocks": [ + {"type": "heading", "level": 1, "text": "Synthetic heading"}, + {"type": "paragraph", "text": sentinel}, + ], + } + ), + encoding="utf-8", + ) + + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + completed = subprocess.run( + [ + sys.executable, + str(script), + "--input", + str(request_path), + "--profile", + "docx-small", + "--iterations", + "2", + "--reference-hardware", + "pytest-reference", + ], + check=False, + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + text=True, + ) + + assert completed.returncode == 0, completed.stderr + evidence = json.loads(completed.stdout) + assert evidence["contractVersion"] == 1 + assert evidence["synthetic"] is True + assert evidence["operation"] == "office_render" + assert evidence["profile"] == "docx-small" + assert evidence["format"] == "docx" + assert evidence["iterations"] == 2 + assert evidence["referenceHardware"] == "pytest-reference" + assert len(evidence["fixtureSha256"]) == 64 + assert len(evidence["sourceSha"]) == 40 + assert evidence["runtime"]["implementation"] + assert evidence["runtime"]["python"] + assert evidence["runtime"]["platform"] + assert len(evidence["samples"]) == 2 + for sample in evidence["samples"]: + assert isinstance(sample["durationMs"], (int, float)) + assert sample["durationMs"] >= 0 + assert isinstance(sample["peakRssBytes"], int) + assert sample["peakRssBytes"] > 0 + assert evidence["summary"]["durationMs"]["p50"] >= 0 + assert evidence["summary"]["durationMs"]["p75"] >= 0 + assert evidence["summary"]["durationMs"]["p95"] >= 0 + assert evidence["summary"]["durationMs"]["max"] >= 0 + assert evidence["summary"]["peakRssBytes"]["max"] > 0 + + combined_output = completed.stdout + completed.stderr + assert sentinel not in combined_output + assert str(request_path) not in combined_output + + +def test_measure_office_render_rejects_unbounded_iteration_counts_without_reading_input( + tmp_path: Path, +) -> None: + """Reject impossible benchmark work before inspecting the caller-selected request path.""" + + request_path = tmp_path / "must-not-be-read.json" + request_path.write_text("PRIVATE-ITERATION-SENTINEL", encoding="utf-8") + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + completed = subprocess.run( + [ + sys.executable, + str(script), + "--input", + str(request_path), + "--profile", + "docx-small", + "--iterations", + "1001", + "--reference-hardware", + "pytest-reference", + ], + check=False, + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + text=True, + ) + + assert completed.returncode != 0 + assert "iterations must be between 1 and 100" in completed.stderr + assert str(request_path) not in completed.stderr + assert "PRIVATE-ITERATION-SENTINEL" not in completed.stderr From 38a287cc44ffa7ad6893c1a9f3cd496f82ae98a0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 10:42:25 -0700 Subject: [PATCH 164/209] test(perf): bind Office evidence to canonical fixtures --- office/tests/test_performance_measurement.py | 172 +++++++++++++------ 1 file changed, 124 insertions(+), 48 deletions(-) diff --git a/office/tests/test_performance_measurement.py b/office/tests/test_performance_measurement.py index 94c14d7d..15fc5219 100644 --- a/office/tests/test_performance_measurement.py +++ b/office/tests/test_performance_measurement.py @@ -8,39 +8,94 @@ from pathlib import Path -def test_measure_office_render_records_duration_peak_rss_and_provenance(tmp_path: Path) -> None: - """Measure repeated synthetic renders without copying document content into evidence.""" +MULTILINGUAL_PARAGRAPH = ( + "English: deterministic Office rendering fixture. 한국어: 합성 성능 문서입니다. " + "日本語: 合成性能文書です。 中文: 这是合成性能文档。 " + "Tiếng Việt: Đây là tài liệu hiệu năng tổng hợp." +) - sentinel = "PRIVATE-BENCHMARK-PAYLOAD-SENTINEL" - request_path = tmp_path / "buyer-private-name.json" - request_path.write_text( - json.dumps( - { - "format": "docx", - "title": "Synthetic benchmark fixture", - "blocks": [ - {"type": "heading", "level": 1, "text": "Synthetic heading"}, - {"type": "paragraph", "text": sentinel}, - ], - } - ), - encoding="utf-8", - ) +def _docx_page(page_number: int) -> list[dict[str, object]]: + page = str(page_number).zfill(3) + return [ + {"type": "heading", "level": 1, "text": f"Synthetic page {page}"}, + { + "type": "paragraph", + "text": f"{MULTILINGUAL_PARAGRAPH} Page {page}.", + "alignment": "justify", + }, + { + "type": "rich_paragraph", + "runs": [ + {"text": f"Page {page} summary: ", "bold": True}, + {"text": "deterministic ", "italic": True}, + {"text": "Office rendering fixture.", "underline": True}, + ], + }, + { + "type": "bullet_list", + "ordered": False, + "items": [ + f"page {page} item A", + f"page {page} item B", + f"page {page} item C", + ], + }, + { + "type": "table", + "headers": ["Page", "Metric", "Value"], + "rows": [ + [page, "latency-sample", page_number], + [page, "memory-sample", page_number * 2], + [page, "revision-sample", page_number * 3], + [page, "render-sample", page_number * 4], + ], + }, + ] + + +def _canonical_docx_small_fixture_bytes() -> bytes: + blocks: list[dict[str, object]] = [] + for page_number in range(1, 3): + blocks.extend(_docx_page(page_number)) + if page_number < 2: + blocks.append({"type": "page_break"}) + request = { + "format": "docx", + "title": "Inkspan synthetic DOCX benchmark: small", + "author": "Inkspan synthetic benchmark", + "subject": "Deterministic synthetic performance fixture", + "blocks": blocks, + } + return (json.dumps(request, ensure_ascii=False, indent=2) + "\n").encode() + + +def _measure_command(script: Path, request_path: Path, iterations: str) -> list[str]: + return [ + sys.executable, + str(script), + "--input", + str(request_path), + "--format", + "docx", + "--fixture-profile", + "small", + "--iterations", + iterations, + "--reference-hardware", + "pytest-reference", + ] + + +def test_measure_office_render_records_duration_peak_rss_and_provenance(tmp_path: Path) -> None: + """Measure a lock-verified synthetic fixture without copying its document body into evidence.""" + + request_path = tmp_path / "synthetic-docx-small.json" + request_bytes = _canonical_docx_small_fixture_bytes() + request_path.write_bytes(request_bytes) script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" completed = subprocess.run( - [ - sys.executable, - str(script), - "--input", - str(request_path), - "--profile", - "docx-small", - "--iterations", - "2", - "--reference-hardware", - "pytest-reference", - ], + _measure_command(script, request_path, "2"), check=False, cwd=Path(__file__).resolve().parents[2], capture_output=True, @@ -52,10 +107,12 @@ def test_measure_office_render_records_duration_peak_rss_and_provenance(tmp_path assert evidence["contractVersion"] == 1 assert evidence["synthetic"] is True assert evidence["operation"] == "office_render" - assert evidence["profile"] == "docx-small" + assert evidence["fixtureId"] == "docx.small" + assert evidence["profile"] == "small" assert evidence["format"] == "docx" assert evidence["iterations"] == 2 assert evidence["referenceHardware"] == "pytest-reference" + assert evidence["fixtureBytes"] == len(request_bytes) assert len(evidence["fixtureSha256"]) == 64 assert len(evidence["sourceSha"]) == 40 assert evidence["runtime"]["implementation"] @@ -67,17 +124,47 @@ def test_measure_office_render_records_duration_peak_rss_and_provenance(tmp_path assert sample["durationMs"] >= 0 assert isinstance(sample["peakRssBytes"], int) assert sample["peakRssBytes"] > 0 - assert evidence["summary"]["durationMs"]["p50"] >= 0 - assert evidence["summary"]["durationMs"]["p75"] >= 0 - assert evidence["summary"]["durationMs"]["p95"] >= 0 - assert evidence["summary"]["durationMs"]["max"] >= 0 - assert evidence["summary"]["peakRssBytes"]["max"] > 0 + for percentile in ("p50", "p75", "p95", "max"): + assert evidence["summary"]["durationMs"][percentile] >= 0 + assert evidence["summary"]["peakRssBytes"][percentile] > 0 combined_output = completed.stdout + completed.stderr - assert sentinel not in combined_output + assert MULTILINGUAL_PARAGRAPH not in combined_output assert str(request_path) not in combined_output +def test_measure_office_render_rejects_noncanonical_content_without_leaking_it( + tmp_path: Path, +) -> None: + """Do not label arbitrary document content as canonical synthetic benchmark evidence.""" + + sentinel = "PRIVATE-NONCANONICAL-DOCUMENT-SENTINEL" + request_path = tmp_path / "private-customer-name.json" + request_path.write_text( + json.dumps( + { + "format": "docx", + "title": "private customer document", + "blocks": [{"type": "paragraph", "text": sentinel}], + } + ), + encoding="utf-8", + ) + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + completed = subprocess.run( + _measure_command(script, request_path, "1"), + check=False, + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + text=True, + ) + + assert completed.returncode != 0 + assert "input does not match the canonical synthetic Office fixture" in completed.stderr + assert sentinel not in completed.stderr + assert str(request_path) not in completed.stderr + + def test_measure_office_render_rejects_unbounded_iteration_counts_without_reading_input( tmp_path: Path, ) -> None: @@ -87,18 +174,7 @@ def test_measure_office_render_rejects_unbounded_iteration_counts_without_readin request_path.write_text("PRIVATE-ITERATION-SENTINEL", encoding="utf-8") script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" completed = subprocess.run( - [ - sys.executable, - str(script), - "--input", - str(request_path), - "--profile", - "docx-small", - "--iterations", - "1001", - "--reference-hardware", - "pytest-reference", - ], + _measure_command(script, request_path, "1001"), check=False, cwd=Path(__file__).resolve().parents[2], capture_output=True, From a5275f9ac6aa0956628674ba39f9a6e74d1a455f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 10:43:42 -0700 Subject: [PATCH 165/209] feat(perf): measure canonical Office render latency and RSS --- office/benchmarks/measure_render.py | 305 ++++++++++++++++++++++++++++ 1 file changed, 305 insertions(+) create mode 100644 office/benchmarks/measure_render.py diff --git a/office/benchmarks/measure_render.py b/office/benchmarks/measure_render.py new file mode 100644 index 00000000..96a75f89 --- /dev/null +++ b/office/benchmarks/measure_render.py @@ -0,0 +1,305 @@ +"""Produce privacy-safe timing and peak-RSS evidence for canonical Office fixtures.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import platform +import re +import stat +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +OFFICE_ROOT = Path(__file__).resolve().parents[1] +REPOSITORY_ROOT = OFFICE_ROOT.parent +LOCK_PATH = REPOSITORY_ROOT / "benchmarks" / "office-fixtures.lock.json" +SOURCE_ROOT = OFFICE_ROOT / "src" +if str(SOURCE_ROOT) not in sys.path: + sys.path.insert(0, str(SOURCE_ROOT)) + +from inkspan_office.safe_renderer import write_office_document # noqa: E402 + +MAX_ITERATIONS = 100 +MAX_TOKEN_CODE_UNITS = 128 +TOKEN_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:-]*\Z") +SHA256_PATTERN = re.compile(r"[0-9a-f]{64}\Z") +GIT_SHA_PATTERN = re.compile(r"[0-9a-f]{40}\Z") +SUPPORTED_FORMATS = {"docx", "xlsx", "pptx"} + + +class BenchmarkContractError(Exception): + """Raised when benchmark evidence cannot be produced safely and truthfully.""" + + +def _metadata_token(value: str, label: str) -> str: + if ( + not isinstance(value, str) + or len(value) > MAX_TOKEN_CODE_UNITS + or not TOKEN_PATTERN.fullmatch(value) + ): + raise BenchmarkContractError(f"{label} must be a bounded metadata token") + return value + + +def _positive_iterations(value: int) -> int: + if not isinstance(value, int) or isinstance(value, bool) or not 1 <= value <= MAX_ITERATIONS: + raise BenchmarkContractError("iterations must be between 1 and 100") + return value + + +def _load_fixture_contract(format_name: str, profile: str) -> tuple[int, str]: + try: + lock = json.loads(LOCK_PATH.read_text(encoding="utf-8")) + if lock.get("contractVersion") != 1 or lock.get("synthetic") is not True: + raise BenchmarkContractError("canonical Office fixture lock is invalid") + record = lock["formats"][format_name][profile] + expected_bytes = record["bytes"] + expected_sha256 = record["sha256"] + except (OSError, KeyError, TypeError, json.JSONDecodeError) as exc: + raise BenchmarkContractError("canonical Office fixture lock is invalid") from exc + if ( + not isinstance(expected_bytes, int) + or isinstance(expected_bytes, bool) + or expected_bytes <= 0 + or not isinstance(expected_sha256, str) + or not SHA256_PATTERN.fullmatch(expected_sha256) + ): + raise BenchmarkContractError("canonical Office fixture lock is invalid") + return expected_bytes, expected_sha256 + + +def _read_exact_regular_fixture(path: Path, expected_bytes: int, expected_sha256: str) -> bytes: + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise BenchmarkContractError("Office benchmark input could not be opened safely") from exc + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size != expected_bytes: + raise BenchmarkContractError( + "input does not match the canonical synthetic Office fixture" + ) + chunks: list[bytes] = [] + remaining = expected_bytes + while remaining: + chunk = os.read(descriptor, min(remaining, 1024 * 1024)) + if not chunk: + raise BenchmarkContractError( + "input does not match the canonical synthetic Office fixture" + ) + chunks.append(chunk) + remaining -= len(chunk) + if os.read(descriptor, 1): + raise BenchmarkContractError( + "input does not match the canonical synthetic Office fixture" + ) + payload = b"".join(chunks) + except OSError as exc: + raise BenchmarkContractError("Office benchmark input could not be read safely") from exc + finally: + os.close(descriptor) + if hashlib.sha256(payload).hexdigest() != expected_sha256: + raise BenchmarkContractError("input does not match the canonical synthetic Office fixture") + return payload + + +def _source_sha() -> str: + status = subprocess.run( + ["git", "-C", str(REPOSITORY_ROOT), "status", "--porcelain", "--untracked-files=all"], + check=False, + capture_output=True, + text=True, + ) + if status.returncode != 0 or status.stdout: + raise BenchmarkContractError("benchmark checkout must be clean") + revision = subprocess.run( + ["git", "-C", str(REPOSITORY_ROOT), "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + ) + sha = revision.stdout.strip() + if revision.returncode != 0 or not GIT_SHA_PATTERN.fullmatch(sha): + raise BenchmarkContractError("benchmark source revision could not be verified") + return sha + + +def _peak_rss_bytes() -> int: + try: + import resource + except ImportError as exc: # pragma: no cover - benchmark CI is POSIX + raise BenchmarkContractError("peak RSS measurement is unavailable on this runtime") from exc + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + if peak <= 0: + raise BenchmarkContractError("peak RSS measurement is unavailable on this runtime") + if sys.platform == "darwin": + return int(peak) + return int(peak) * 1024 + + +def _child_measure() -> int: + try: + payload_bytes = sys.stdin.buffer.read() + payload = json.loads(payload_bytes.decode("utf-8")) + if not isinstance(payload, dict): + raise BenchmarkContractError("canonical Office fixture must contain an object") + format_name = payload.get("format") + if format_name not in SUPPORTED_FORMATS: + raise BenchmarkContractError("canonical Office fixture format is unsupported") + with tempfile.TemporaryDirectory(prefix="inkspan-office-benchmark-") as directory: + output_path = Path(directory) / f"render.{format_name}" + started = time.perf_counter_ns() + write_office_document(payload, output_path) + duration_ms = (time.perf_counter_ns() - started) / 1_000_000 + peak_rss_bytes = _peak_rss_bytes() + print( + json.dumps( + { + "format": format_name, + "durationMs": round(duration_ms, 6), + "peakRssBytes": peak_rss_bytes, + }, + separators=(",", ":"), + sort_keys=True, + ) + ) + return 0 + except Exception: + print("Office benchmark render sample failed.", file=sys.stderr) + return 2 + + +def _run_sample(payload_bytes: bytes) -> dict[str, Any]: + completed = subprocess.run( + [sys.executable, str(Path(__file__).resolve()), "--child"], + input=payload_bytes, + check=False, + cwd=REPOSITORY_ROOT, + capture_output=True, + ) + if completed.returncode != 0: + raise BenchmarkContractError("Office benchmark render sample failed") + try: + sample = json.loads(completed.stdout.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise BenchmarkContractError("Office benchmark render sample was invalid") from exc + if not isinstance(sample, dict): + raise BenchmarkContractError("Office benchmark render sample was invalid") + duration = sample.get("durationMs") + peak_rss = sample.get("peakRssBytes") + format_name = sample.get("format") + if ( + format_name not in SUPPORTED_FORMATS + or not isinstance(duration, (int, float)) + or isinstance(duration, bool) + or not math.isfinite(duration) + or duration < 0 + or not isinstance(peak_rss, int) + or isinstance(peak_rss, bool) + or peak_rss <= 0 + ): + raise BenchmarkContractError("Office benchmark render sample was invalid") + return {"format": format_name, "durationMs": duration, "peakRssBytes": peak_rss} + + +def _percentile(values: list[float], quantile: float) -> float: + ordered = sorted(values) + if len(ordered) == 1: + return ordered[0] + position = (len(ordered) - 1) * quantile + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + weight = position - lower + return ordered[lower] * (1 - weight) + ordered[upper] * weight + + +def _summary(values: list[float], *, integral: bool) -> dict[str, int | float]: + result: dict[str, int | float] = {} + for name, quantile in (("p50", 0.50), ("p75", 0.75), ("p95", 0.95)): + value = _percentile(values, quantile) + result[name] = int(round(value)) if integral else round(value, 6) + maximum = max(values) + result["max"] = int(maximum) if integral else round(maximum, 6) + return result + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Measure a canonical synthetic Inkspan Office fixture") + parser.add_argument("--input", required=True, help="canonical generated Office fixture path") + parser.add_argument("--format", required=True, choices=sorted(SUPPORTED_FORMATS)) + parser.add_argument("--fixture-profile", required=True) + parser.add_argument("--iterations", type=int, default=5) + parser.add_argument("--reference-hardware", required=True) + return parser + + +def _main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + iterations = _positive_iterations(args.iterations) + profile = _metadata_token(args.fixture_profile, "fixture profile") + reference_hardware = _metadata_token(args.reference_hardware, "reference hardware") + expected_bytes, expected_sha256 = _load_fixture_contract(args.format, profile) + source_sha = _source_sha() + payload_bytes = _read_exact_regular_fixture( + Path(args.input), expected_bytes, expected_sha256 + ) + samples = [_run_sample(payload_bytes) for _ in range(iterations)] + observed_formats = {sample["format"] for sample in samples} + if observed_formats != {args.format}: + raise BenchmarkContractError("Office benchmark render format was inconsistent") + duration_values = [float(sample["durationMs"]) for sample in samples] + rss_values = [float(sample["peakRssBytes"]) for sample in samples] + evidence = { + "contractVersion": 1, + "synthetic": True, + "operation": "office_render", + "fixtureId": f"{args.format}.{profile}", + "format": args.format, + "profile": profile, + "iterations": iterations, + "referenceHardware": reference_hardware, + "fixtureBytes": expected_bytes, + "fixtureSha256": expected_sha256, + "sourceSha": source_sha, + "runtime": { + "implementation": platform.python_implementation(), + "python": platform.python_version(), + "platform": sys.platform, + }, + "samples": samples, + "summary": { + "durationMs": _summary(duration_values, integral=False), + "peakRssBytes": _summary(rss_values, integral=True), + }, + } + print(json.dumps(evidence, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + except BenchmarkContractError as exc: + print(str(exc), file=sys.stderr) + return 2 + except Exception: + print("Office benchmark measurement failed.", file=sys.stderr) + return 2 + + +def main() -> int: + """Run the benchmark command or its isolated one-render child process.""" + + if sys.argv[1:] == ["--child"]: + return _child_measure() + return _main() + + +if __name__ == "__main__": + raise SystemExit(main()) From e6598e866a84d6b1139c74a8bd978566863e878d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 11:34:16 -0700 Subject: [PATCH 166/209] test(perf): bound hung Office render sample RED --- office/tests/test_performance_measurement.py | 29 ++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/office/tests/test_performance_measurement.py b/office/tests/test_performance_measurement.py index 15fc5219..2307d09b 100644 --- a/office/tests/test_performance_measurement.py +++ b/office/tests/test_performance_measurement.py @@ -3,10 +3,13 @@ from __future__ import annotations import json +import runpy import subprocess import sys from pathlib import Path +import pytest + MULTILINGUAL_PARAGRAPH = ( "English: deterministic Office rendering fixture. 한국어: 합성 성능 문서입니다. " @@ -185,3 +188,29 @@ def test_measure_office_render_rejects_unbounded_iteration_counts_without_readin assert "iterations must be between 1 and 100" in completed.stderr assert str(request_path) not in completed.stderr assert "PRIVATE-ITERATION-SENTINEL" not in completed.stderr + + +def test_office_render_sample_times_out_without_leaking_payload( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Bound a hung renderer child and normalize timeout failure without exposing document data.""" + + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + namespace = runpy.run_path(str(script), run_name="inkspan_measure_render_test") + benchmark_error = namespace["BenchmarkContractError"] + run_sample = namespace["_run_sample"] + observed_timeout: list[float | None] = [] + sentinel = b'PRIVATE-HUNG-RENDER-SENTINEL' + + def _timeout(*args: object, **kwargs: object) -> subprocess.CompletedProcess[bytes]: + timeout = kwargs.get("timeout") + observed_timeout.append(timeout if isinstance(timeout, (int, float)) else None) + raise subprocess.TimeoutExpired(args[0] if args else "renderer", timeout or 0) + + monkeypatch.setattr(subprocess, "run", _timeout) + + with pytest.raises(benchmark_error, match="Office benchmark render sample timed out") as exc_info: + run_sample(sentinel) + + assert observed_timeout == [120] + assert sentinel.decode() not in str(exc_info.value) From 7128b6c1e0c130b26993466bddd18fac5e2ebc60 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 11:36:01 -0700 Subject: [PATCH 167/209] fix(perf): bound Office render sample duration --- office/benchmarks/measure_render.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/office/benchmarks/measure_render.py b/office/benchmarks/measure_render.py index 96a75f89..9b092eae 100644 --- a/office/benchmarks/measure_render.py +++ b/office/benchmarks/measure_render.py @@ -28,6 +28,7 @@ MAX_ITERATIONS = 100 MAX_TOKEN_CODE_UNITS = 128 +SAMPLE_TIMEOUT_SECONDS = 120 TOKEN_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:-]*\Z") SHA256_PATTERN = re.compile(r"[0-9a-f]{64}\Z") GIT_SHA_PATTERN = re.compile(r"[0-9a-f]{40}\Z") @@ -178,13 +179,17 @@ def _child_measure() -> int: def _run_sample(payload_bytes: bytes) -> dict[str, Any]: - completed = subprocess.run( - [sys.executable, str(Path(__file__).resolve()), "--child"], - input=payload_bytes, - check=False, - cwd=REPOSITORY_ROOT, - capture_output=True, - ) + try: + completed = subprocess.run( + [sys.executable, str(Path(__file__).resolve()), "--child"], + input=payload_bytes, + check=False, + cwd=REPOSITORY_ROOT, + capture_output=True, + timeout=SAMPLE_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired: + raise BenchmarkContractError("Office benchmark render sample timed out") from None if completed.returncode != 0: raise BenchmarkContractError("Office benchmark render sample failed") try: From 1572af1c985a87452785c865d0a1c405d0631990 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 12:26:19 -0700 Subject: [PATCH 168/209] test(perf): require child fixture re-verification --- office/tests/test_performance_measurement.py | 33 ++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/office/tests/test_performance_measurement.py b/office/tests/test_performance_measurement.py index 2307d09b..c0b21277 100644 --- a/office/tests/test_performance_measurement.py +++ b/office/tests/test_performance_measurement.py @@ -214,3 +214,36 @@ def _timeout(*args: object, **kwargs: object) -> subprocess.CompletedProcess[byt assert observed_timeout == [120] assert sentinel.decode() not in str(exc_info.value) + + +def test_office_render_child_rejects_unverified_private_payload() -> None: + """Require the isolated child to re-verify canonical fixture identity before rendering.""" + + sentinel = "PRIVATE-DIRECT-CHILD-DOCUMENT-SENTINEL" + payload = json.dumps( + { + "format": "docx", + "title": "private direct child document", + "blocks": [{"type": "paragraph", "text": sentinel}], + } + ).encode() + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + completed = subprocess.run( + [ + sys.executable, + str(script), + "--child", + "--format", + "docx", + "--fixture-profile", + "small", + ], + input=payload, + check=False, + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + ) + + assert completed.returncode != 0 + assert b"input does not match the canonical synthetic Office fixture" in completed.stderr + assert sentinel.encode() not in completed.stderr From 28053fd4e00a8c7dc439e30e2e203d4be208ce98 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 12:30:07 -0700 Subject: [PATCH 169/209] fix(perf): bind Office child to canonical fixture --- office/benchmarks/measure_render.py | 56 ++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/office/benchmarks/measure_render.py b/office/benchmarks/measure_render.py index 9b092eae..4789d264 100644 --- a/office/benchmarks/measure_render.py +++ b/office/benchmarks/measure_render.py @@ -112,6 +112,13 @@ def _read_exact_regular_fixture(path: Path, expected_bytes: int, expected_sha256 return payload +def _read_exact_stdin_fixture(expected_bytes: int, expected_sha256: str) -> bytes: + payload = sys.stdin.buffer.read(expected_bytes + 1) + if len(payload) != expected_bytes or hashlib.sha256(payload).hexdigest() != expected_sha256: + raise BenchmarkContractError("input does not match the canonical synthetic Office fixture") + return payload + + def _source_sha() -> str: status = subprocess.run( ["git", "-C", str(REPOSITORY_ROOT), "status", "--porcelain", "--untracked-files=all"], @@ -146,15 +153,16 @@ def _peak_rss_bytes() -> int: return int(peak) * 1024 -def _child_measure() -> int: +def _child_measure(format_name: str, profile: str) -> int: try: - payload_bytes = sys.stdin.buffer.read() + profile = _metadata_token(profile, "fixture profile") + expected_bytes, expected_sha256 = _load_fixture_contract(format_name, profile) + payload_bytes = _read_exact_stdin_fixture(expected_bytes, expected_sha256) payload = json.loads(payload_bytes.decode("utf-8")) if not isinstance(payload, dict): raise BenchmarkContractError("canonical Office fixture must contain an object") - format_name = payload.get("format") - if format_name not in SUPPORTED_FORMATS: - raise BenchmarkContractError("canonical Office fixture format is unsupported") + if payload.get("format") != format_name: + raise BenchmarkContractError("canonical Office fixture format is inconsistent") with tempfile.TemporaryDirectory(prefix="inkspan-office-benchmark-") as directory: output_path = Path(directory) / f"render.{format_name}" started = time.perf_counter_ns() @@ -173,15 +181,26 @@ def _child_measure() -> int: ) ) return 0 + except BenchmarkContractError as exc: + print(str(exc), file=sys.stderr) + return 2 except Exception: print("Office benchmark render sample failed.", file=sys.stderr) return 2 -def _run_sample(payload_bytes: bytes) -> dict[str, Any]: +def _run_sample(payload_bytes: bytes, format_name: str, profile: str) -> dict[str, Any]: try: completed = subprocess.run( - [sys.executable, str(Path(__file__).resolve()), "--child"], + [ + sys.executable, + str(Path(__file__).resolve()), + "--child", + "--format", + format_name, + "--fixture-profile", + profile, + ], input=payload_bytes, check=False, cwd=REPOSITORY_ROOT, @@ -200,9 +219,9 @@ def _run_sample(payload_bytes: bytes) -> dict[str, Any]: raise BenchmarkContractError("Office benchmark render sample was invalid") duration = sample.get("durationMs") peak_rss = sample.get("peakRssBytes") - format_name = sample.get("format") + observed_format = sample.get("format") if ( - format_name not in SUPPORTED_FORMATS + observed_format != format_name or not isinstance(duration, (int, float)) or isinstance(duration, bool) or not math.isfinite(duration) @@ -212,7 +231,7 @@ def _run_sample(payload_bytes: bytes) -> dict[str, Any]: or peak_rss <= 0 ): raise BenchmarkContractError("Office benchmark render sample was invalid") - return {"format": format_name, "durationMs": duration, "peakRssBytes": peak_rss} + return {"format": observed_format, "durationMs": duration, "peakRssBytes": peak_rss} def _percentile(values: list[float], quantile: float) -> float: @@ -248,6 +267,13 @@ def _parser() -> argparse.ArgumentParser: return parser +def _child_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Render one verified synthetic Inkspan Office fixture") + parser.add_argument("--format", required=True, choices=sorted(SUPPORTED_FORMATS)) + parser.add_argument("--fixture-profile", required=True) + return parser + + def _main(argv: list[str] | None = None) -> int: args = _parser().parse_args(argv) try: @@ -259,10 +285,7 @@ def _main(argv: list[str] | None = None) -> int: payload_bytes = _read_exact_regular_fixture( Path(args.input), expected_bytes, expected_sha256 ) - samples = [_run_sample(payload_bytes) for _ in range(iterations)] - observed_formats = {sample["format"] for sample in samples} - if observed_formats != {args.format}: - raise BenchmarkContractError("Office benchmark render format was inconsistent") + samples = [_run_sample(payload_bytes, args.format, profile) for _ in range(iterations)] duration_values = [float(sample["durationMs"]) for sample in samples] rss_values = [float(sample["peakRssBytes"]) for sample in samples] evidence = { @@ -301,8 +324,9 @@ def _main(argv: list[str] | None = None) -> int: def main() -> int: """Run the benchmark command or its isolated one-render child process.""" - if sys.argv[1:] == ["--child"]: - return _child_measure() + if sys.argv[1:2] == ["--child"]: + args = _child_parser().parse_args(sys.argv[2:]) + return _child_measure(args.format, args.fixture_profile) return _main() From a6ab00f3be35ded4db6707dd6c8e8d15ea73f642 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 12:30:51 -0700 Subject: [PATCH 170/209] test(perf): pass canonical child identity --- office/tests/test_performance_measurement.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/office/tests/test_performance_measurement.py b/office/tests/test_performance_measurement.py index c0b21277..b9e6c88f 100644 --- a/office/tests/test_performance_measurement.py +++ b/office/tests/test_performance_measurement.py @@ -210,7 +210,7 @@ def _timeout(*args: object, **kwargs: object) -> subprocess.CompletedProcess[byt monkeypatch.setattr(subprocess, "run", _timeout) with pytest.raises(benchmark_error, match="Office benchmark render sample timed out") as exc_info: - run_sample(sentinel) + run_sample(sentinel, "docx", "small") assert observed_timeout == [120] assert sentinel.decode() not in str(exc_info.value) From 2251be8a734799527f8dce87fcb5a9cc291a5b9d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 14:08:02 -0700 Subject: [PATCH 171/209] test(perf): bind packed benchmark to verified tarball bytes --- ...ormancePackedArtifactPathStability.test.ts | 201 ++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 src/performancePackedArtifactPathStability.test.ts diff --git a/src/performancePackedArtifactPathStability.test.ts b/src/performancePackedArtifactPathStability.test.ts new file mode 100644 index 00000000..659d9ed7 --- /dev/null +++ b/src/performancePackedArtifactPathStability.test.ts @@ -0,0 +1,201 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + chmodSync, + copyFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { delimiter, join, resolve } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const temporaryDirectories: string[] = []; +const activeRuntimeId = `node-${process.versions.node}`; +const sourceCommitSha = execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], +}).trim(); +const referenceHardwareId = `refhw-sha256-${'d'.repeat(64)}`; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function sha256(bytes: Buffer | string): string { + return createHash('sha256').update(bytes).digest('hex'); +} + +function createPackedBenchmarkFixture( + directory: string, + name: string, + markdownMarker: string, +): { + markdownModuleSha256: string; + packageSha256: string; + tarballPath: string; +} { + const packageDirectory = join(directory, `${name}-package-source`); + const distDirectory = join(packageDirectory, 'dist'); + const packDirectory = join(directory, `${name}-packed`); + mkdirSync(distDirectory, { recursive: true }); + mkdirSync(packDirectory, { recursive: true }); + + const markdownModule = `export function markdownToHtml(source) { return \`

${source}

\`; }\n`; + writeFileSync( + join(packageDirectory, 'package.json'), + `${JSON.stringify( + { + name: '@contextualwisdomlab/cwl-editor', + version: '0.0.0-benchmark-fixture', + type: 'module', + files: ['dist'], + }, + null, + 2, + )}\n`, + 'utf8', + ); + writeFileSync( + join(distDirectory, 'cwl-markdown.js'), + markdownModule, + 'utf8', + ); + writeFileSync( + join(distDirectory, 'cwl-revision-evidence.js'), + `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'e'.repeat(64)}' } }; }\n`, + 'utf8', + ); + + const packResult = JSON.parse( + execFileSync( + 'npm', + [ + 'pack', + '--json', + '--ignore-scripts', + '--pack-destination', + packDirectory, + ], + { + cwd: packageDirectory, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, + ), + )[0] as { filename: string }; + const tarballPath = join(packDirectory, packResult.filename); + return { + markdownModuleSha256: sha256(markdownModule), + packageSha256: sha256(readFileSync(tarballPath)), + tarballPath, + }; +} + +function createTarInterpositionShim( + directory: string, + originalTarballPath: string, + adversarialTarballPath: string, +): { environment: NodeJS.ProcessEnv; originalBackupPath: string } { + const shimDirectory = join(directory, 'shim'); + mkdirSync(shimDirectory, { recursive: true }); + const originalBackupPath = join(directory, 'original-package-backup.tgz'); + copyFileSync(originalTarballPath, originalBackupPath); + const realTar = execFileSync('sh', ['-c', 'command -v tar'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }).trim(); + const shimPath = join(shimDirectory, 'tar'); + writeFileSync( + shimPath, + `#!/usr/bin/env node\nimport { copyFileSync } from 'node:fs';\nimport { spawnSync } from 'node:child_process';\n\nconst args = process.argv.slice(2);\nconst target = args.at(-1);\nconst original = process.env.INKSPAN_TEST_ORIGINAL_TARBALL;\nif (target === original) copyFileSync(process.env.INKSPAN_TEST_ADVERSARIAL_TARBALL, original);\ntry {\n const result = spawnSync(process.env.INKSPAN_TEST_REAL_TAR, args, { stdio: ['ignore', 'pipe', 'pipe'] });\n if (result.stdout) process.stdout.write(result.stdout);\n if (result.stderr) process.stderr.write(result.stderr);\n process.exitCode = result.status ?? 1;\n} finally {\n if (target === original) copyFileSync(process.env.INKSPAN_TEST_ORIGINAL_BACKUP, original);\n}\n`, + 'utf8', + ); + chmodSync(shimPath, 0o755); + return { + originalBackupPath, + environment: { + ...process.env, + PATH: `${shimDirectory}${delimiter}${process.env.PATH ?? ''}`, + INKSPAN_TEST_REAL_TAR: realTar, + INKSPAN_TEST_ORIGINAL_TARBALL: originalTarballPath, + INKSPAN_TEST_ADVERSARIAL_TARBALL: adversarialTarballPath, + INKSPAN_TEST_ORIGINAL_BACKUP: originalBackupPath, + }, + }; +} + +describe('packed artifact benchmark path stability', () => { + it('measures the same tarball bytes whose package digest was verified', () => { + if (process.platform === 'win32') return; + + const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-path-stability-')); + temporaryDirectories.push(directory); + const original = createPackedBenchmarkFixture(directory, 'original', 'verified'); + const adversarial = createPackedBenchmarkFixture(directory, 'adversarial', 'interposed'); + const { environment } = createTarInterpositionShim( + directory, + original.tarballPath, + adversarial.tarballPath, + ); + const markdownInputPath = join(directory, 'input.md'); + const revisionInputPath = join(directory, 'document-envelope.json'); + const outputDirectory = join(directory, 'evidence'); + writeFileSync(markdownInputPath, '# Stable packed artifact\n', 'utf8'); + writeFileSync( + revisionInputPath, + '{"contractVersion":1,"mode":"markdown","document":"# Stable packed artifact"}\n', + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + suitePath, + '--input', + markdownInputPath, + '--revision-input', + revisionInputPath, + '--package-tarball', + original.tarballPath, + '--package-sha256', + original.packageSha256, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + sourceCommitSha, + '--runtime-id', + activeRuntimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + outputDirectory, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + env: environment, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 15_000, + }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + const markdownEvidence = JSON.parse( + readFileSync(join(outputDirectory, 'markdown', 'samples.json'), 'utf8'), + ) as { artifactSha256: string }; + expect(markdownEvidence.artifactSha256).toBe(original.markdownModuleSha256); + }); +}); From 0055f240dc86763008c59228737519cf3a4b55d0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 14:09:48 -0700 Subject: [PATCH 172/209] test(perf): exercise packed tarball interposition --- src/performancePackedArtifactPathStability.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/performancePackedArtifactPathStability.test.ts b/src/performancePackedArtifactPathStability.test.ts index 659d9ed7..ae659c42 100644 --- a/src/performancePackedArtifactPathStability.test.ts +++ b/src/performancePackedArtifactPathStability.test.ts @@ -117,7 +117,7 @@ function createTarInterpositionShim( const shimPath = join(shimDirectory, 'tar'); writeFileSync( shimPath, - `#!/usr/bin/env node\nimport { copyFileSync } from 'node:fs';\nimport { spawnSync } from 'node:child_process';\n\nconst args = process.argv.slice(2);\nconst target = args.at(-1);\nconst original = process.env.INKSPAN_TEST_ORIGINAL_TARBALL;\nif (target === original) copyFileSync(process.env.INKSPAN_TEST_ADVERSARIAL_TARBALL, original);\ntry {\n const result = spawnSync(process.env.INKSPAN_TEST_REAL_TAR, args, { stdio: ['ignore', 'pipe', 'pipe'] });\n if (result.stdout) process.stdout.write(result.stdout);\n if (result.stderr) process.stderr.write(result.stderr);\n process.exitCode = result.status ?? 1;\n} finally {\n if (target === original) copyFileSync(process.env.INKSPAN_TEST_ORIGINAL_BACKUP, original);\n}\n`, + `#!/usr/bin/env node\nimport { copyFileSync } from 'node:fs';\nimport { spawnSync } from 'node:child_process';\n\nconst args = process.argv.slice(2);\nconst target = args[1];\nconst original = process.env.INKSPAN_TEST_ORIGINAL_TARBALL;\nif (target === original) copyFileSync(process.env.INKSPAN_TEST_ADVERSARIAL_TARBALL, original);\ntry {\n const result = spawnSync(process.env.INKSPAN_TEST_REAL_TAR, args, { stdio: ['ignore', 'pipe', 'pipe'] });\n if (result.stdout) process.stdout.write(result.stdout);\n if (result.stderr) process.stderr.write(result.stderr);\n process.exitCode = result.status ?? 1;\n} finally {\n if (target === original) copyFileSync(process.env.INKSPAN_TEST_ORIGINAL_BACKUP, original);\n}\n`, 'utf8', ); chmodSync(shimPath, 0o755); From 4033e2c73a4d68fb26c37c6932bd69c156ce2401 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 14:10:20 -0700 Subject: [PATCH 173/209] fix(perf): snapshot verified packed artifact before extraction --- benchmarks/run-current-suite.mjs | 162 +++++++++++++++++++++++++++++-- 1 file changed, 152 insertions(+), 10 deletions(-) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index cc87a585..030f33e1 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -1,5 +1,17 @@ import { spawnSync } from 'node:child_process'; -import { dirname, resolve } from 'node:path'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdtempSync, + openSync, + readSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { assertCleanSourceCheckout } from './source-checkout-provenance.mjs'; @@ -7,21 +19,151 @@ import { assertCleanSourceCheckout } from './source-checkout-provenance.mjs'; const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); const repositoryRoot = resolve(benchmarkDirectory, '..'); const coreRunnerPath = resolve(benchmarkDirectory, 'run-current-suite-core.mjs'); +const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const packedFlags = Object.freeze([ + '--input', + '--revision-input', + '--package-tarball', + '--package-sha256', + '--profile', + '--samples', + '--source-commit-sha', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const packageTarballValueIndex = + packedFlags.indexOf('--package-tarball') * 2 + 1; + +function matchesArguments(argv, expectedFlags) { + return ( + argv.length === expectedFlags.length * 2 && + expectedFlags.every((flag, index) => argv[index * 2] === flag) && + expectedFlags.every((_, index) => argv[index * 2 + 1]?.length > 0) + ); +} + +function readPackedTarballSnapshot(path) { + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error( + 'Benchmark suite package tarball must be a regular non-symlink file.', + ); + } + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error( + 'Benchmark suite package tarball must be a regular non-symlink file.', + ); + } + + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error( + 'Benchmark suite package tarball must be a regular non-symlink file.', + ); + } + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error( + 'Benchmark suite package tarball must be a regular non-symlink file.', + ); + } + if (metadata.size > MAX_PACKAGE_BYTES) { + throw new Error('Benchmark suite package tarball exceeds the supported size.'); + } + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= MAX_PACKAGE_BYTES) { + const remainingBudget = MAX_PACKAGE_BYTES + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > MAX_PACKAGE_BYTES) { + throw new Error('Benchmark suite package tarball exceeds the supported size.'); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + return Buffer.concat(chunks, totalBytes); + } finally { + closeSync(descriptor); + } +} + +function snapshotPackedArguments(argv) { + if (!matchesArguments(argv, packedFlags)) { + return Object.freeze({ argv, temporaryDirectory: null }); + } + + const packageTarballPath = resolve(argv[packageTarballValueIndex]); + const packageBytes = readPackedTarballSnapshot(packageTarballPath); + const temporaryDirectory = mkdtempSync( + join(tmpdir(), 'inkspan-packed-suite-snapshot-'), + ); + const snapshotPath = join(temporaryDirectory, 'package.tgz'); + try { + writeFileSync(snapshotPath, packageBytes, { mode: 0o600 }); + } catch { + rmSync(temporaryDirectory, { recursive: true, force: true }); + throw new Error('Benchmark suite package tarball snapshot could not be prepared.'); + } + + const snapshottedArguments = [...argv]; + snapshottedArguments[packageTarballValueIndex] = snapshotPath; + return Object.freeze({ + argv: snapshottedArguments, + temporaryDirectory, + }); +} function main(argv) { assertCleanSourceCheckout(repositoryRoot); + const snapshotted = snapshotPackedArguments(argv); - const result = spawnSync(process.execPath, [coreRunnerPath, ...argv], { - cwd: repositoryRoot, - stdio: 'inherit', - timeout: 600_000, - }); + try { + const result = spawnSync( + process.execPath, + [coreRunnerPath, ...snapshotted.argv], + { + cwd: repositoryRoot, + stdio: 'inherit', + timeout: 600_000, + }, + ); - if (result.error !== undefined || result.signal !== null) { - throw new Error('Benchmark suite internal runner could not complete.'); - } + if (result.error !== undefined || result.signal !== null) { + throw new Error('Benchmark suite internal runner could not complete.'); + } - process.exitCode = result.status ?? 1; + process.exitCode = result.status ?? 1; + } finally { + if (snapshotted.temporaryDirectory !== null) { + rmSync(snapshotted.temporaryDirectory, { + recursive: true, + force: true, + }); + } + } } try { From c455ec147e0a3d11106a1d4e971bb36d084c5b69 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 14:12:07 -0700 Subject: [PATCH 174/209] fix(test): preserve generated benchmark source interpolation --- src/performancePackedArtifactPathStability.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/performancePackedArtifactPathStability.test.ts b/src/performancePackedArtifactPathStability.test.ts index ae659c42..7ff8c0df 100644 --- a/src/performancePackedArtifactPathStability.test.ts +++ b/src/performancePackedArtifactPathStability.test.ts @@ -50,7 +50,7 @@ function createPackedBenchmarkFixture( mkdirSync(distDirectory, { recursive: true }); mkdirSync(packDirectory, { recursive: true }); - const markdownModule = `export function markdownToHtml(source) { return \`

${source}

\`; }\n`; + const markdownModule = `export function markdownToHtml(source) { return \`

\${source}

\`; }\n`; writeFileSync( join(packageDirectory, 'package.json'), `${JSON.stringify( From 5abc23761f6285b796444f6636a37cd8a9a60238 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 14:19:26 -0700 Subject: [PATCH 175/209] test(perf): reject mutable existing suite evidence targets --- ...rmanceSuiteExistingOutputAtomicity.test.ts | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 src/performanceSuiteExistingOutputAtomicity.test.ts diff --git a/src/performanceSuiteExistingOutputAtomicity.test.ts b/src/performanceSuiteExistingOutputAtomicity.test.ts new file mode 100644 index 00000000..a1f6cdfb --- /dev/null +++ b/src/performanceSuiteExistingOutputAtomicity.test.ts @@ -0,0 +1,104 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function sha256(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +describe('benchmark suite existing-output atomicity', () => { + it('rejects an existing evidence directory before mutating prior evidence', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-suite-existing-output-')); + temporaryDirectories.push(directory); + const markdownInputPath = join(directory, 'input.md'); + const revisionInputPath = join(directory, 'document-envelope.json'); + const markdownModulePath = join(directory, 'markdown.mjs'); + const revisionModulePath = join(directory, 'revision.mjs'); + const outputDirectory = join(directory, 'evidence'); + const priorEvidencePath = join(outputDirectory, 'accepted-evidence.json'); + const markdownModuleSource = + "export function markdownToHtml(source) { return `

${source}

`; }\n"; + const revisionModuleSource = + "export async function createDocumentEnvelopeRevisionEvidenceBytes() { throw new Error('private downstream failure'); }\n"; + + writeFileSync(markdownInputPath, '# Existing evidence must stay immutable\n', 'utf8'); + writeFileSync( + revisionInputPath, + '{"contractVersion":1,"mode":"markdown","document":"# Existing evidence must stay immutable"}\n', + 'utf8', + ); + writeFileSync(markdownModulePath, markdownModuleSource, 'utf8'); + writeFileSync(revisionModulePath, revisionModuleSource, 'utf8'); + mkdirSync(outputDirectory); + writeFileSync(priorEvidencePath, '{"status":"accepted"}\n', 'utf8'); + + const result = spawnSync( + process.execPath, + [ + suitePath, + '--input', + markdownInputPath, + '--module', + markdownModulePath, + '--revision-input', + revisionInputPath, + '--revision-module', + revisionModulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + sha256(markdownModuleSource), + '--revision-artifact-sha256', + sha256(revisionModuleSource), + '--runtime-id', + 'node-22.0.0', + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + outputDirectory, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite output directory must not already exist.\n', + ); + expect(readFileSync(priorEvidencePath, 'utf8')).toBe( + '{"status":"accepted"}\n', + ); + expect(existsSync(join(outputDirectory, 'markdown'))).toBe(false); + expect(existsSync(join(outputDirectory, 'revision'))).toBe(false); + }); +}); From 962113322857faa596ff16adc0f618cdb43e1049 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 14:24:25 -0700 Subject: [PATCH 176/209] fix(perf): preserve existing benchmark evidence directories --- benchmarks/run-current-suite.mjs | 37 ++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 030f33e1..21b21403 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -22,6 +22,20 @@ const coreRunnerPath = resolve(benchmarkDirectory, 'run-current-suite-core.mjs') const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const legacyFlags = Object.freeze([ + '--input', + '--module', + '--revision-input', + '--revision-module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--revision-artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); const packedFlags = Object.freeze([ '--input', '--revision-input', @@ -45,6 +59,28 @@ function matchesArguments(argv, expectedFlags) { ); } +function matchingFlags(argv) { + if (matchesArguments(argv, packedFlags)) return packedFlags; + if (matchesArguments(argv, legacyFlags)) return legacyFlags; + return null; +} + +function assertFreshOutputDirectory(argv) { + const flags = matchingFlags(argv); + if (flags === null) return; + const outputValueIndex = flags.indexOf('--output') * 2 + 1; + const outputDirectory = resolve(repositoryRoot, argv[outputValueIndex]); + let metadata; + try { + metadata = lstatSync(outputDirectory, { throwIfNoEntry: false }); + } catch { + return; + } + if (metadata?.isDirectory()) { + throw new Error('Benchmark suite output directory must not already exist.'); + } +} + function readPackedTarballSnapshot(path) { let pathMetadata; try { @@ -138,6 +174,7 @@ function snapshotPackedArguments(argv) { function main(argv) { assertCleanSourceCheckout(repositoryRoot); + assertFreshOutputDirectory(argv); const snapshotted = snapshotPackedArguments(argv); try { From 1949f18581bc76551ef73d08202342e37329829f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 15:06:56 -0700 Subject: [PATCH 177/209] test(perf): cover core existing-output race boundary --- ...rmanceSuiteExistingOutputAtomicity.test.ts | 177 +++++++++++++----- 1 file changed, 126 insertions(+), 51 deletions(-) diff --git a/src/performanceSuiteExistingOutputAtomicity.test.ts b/src/performanceSuiteExistingOutputAtomicity.test.ts index a1f6cdfb..063b0ce4 100644 --- a/src/performanceSuiteExistingOutputAtomicity.test.ts +++ b/src/performanceSuiteExistingOutputAtomicity.test.ts @@ -15,6 +15,10 @@ import { afterEach, describe, expect, it } from 'vitest'; const repositoryRoot = process.cwd(); const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const coreSuitePath = resolve( + repositoryRoot, + 'benchmarks/run-current-suite-core.mjs', +); const temporaryDirectories: string[] = []; afterEach(() => { @@ -27,61 +31,136 @@ function sha256(source: string): string { return createHash('sha256').update(source).digest('hex'); } +function existingOutputArguments({ + suite, + markdownInputPath, + markdownModulePath, + revisionInputPath, + revisionModulePath, + markdownModuleSource, + revisionModuleSource, + outputDirectory, +}: { + suite: string; + markdownInputPath: string; + markdownModulePath: string; + revisionInputPath: string; + revisionModulePath: string; + markdownModuleSource: string; + revisionModuleSource: string; + outputDirectory: string; +}) { + return [ + suite, + '--input', + markdownInputPath, + '--module', + markdownModulePath, + '--revision-input', + revisionInputPath, + '--revision-module', + revisionModulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + sha256(markdownModuleSource), + '--revision-artifact-sha256', + sha256(revisionModuleSource), + '--runtime-id', + 'node-22.0.0', + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + outputDirectory, + ]; +} + +function makeExistingOutputFixture(directory: string) { + const markdownInputPath = join(directory, 'input.md'); + const revisionInputPath = join(directory, 'document-envelope.json'); + const markdownModulePath = join(directory, 'markdown.mjs'); + const revisionModulePath = join(directory, 'revision.mjs'); + const outputDirectory = join(directory, 'evidence'); + const priorEvidencePath = join(outputDirectory, 'accepted-evidence.json'); + const markdownModuleSource = + "export function markdownToHtml(source) { return `

${source}

`; }\n"; + const revisionModuleSource = + "export async function createDocumentEnvelopeRevisionEvidenceBytes() { throw new Error('private downstream failure'); }\n"; + + writeFileSync(markdownInputPath, '# Existing evidence must stay immutable\n', 'utf8'); + writeFileSync( + revisionInputPath, + '{"contractVersion":1,"mode":"markdown","document":"# Existing evidence must stay immutable"}\n', + 'utf8', + ); + writeFileSync(markdownModulePath, markdownModuleSource, 'utf8'); + writeFileSync(revisionModulePath, revisionModuleSource, 'utf8'); + mkdirSync(outputDirectory); + writeFileSync(priorEvidencePath, '{"status":"accepted"}\n', 'utf8'); + + return { + markdownInputPath, + revisionInputPath, + markdownModulePath, + revisionModulePath, + outputDirectory, + priorEvidencePath, + markdownModuleSource, + revisionModuleSource, + }; +} + +function expectExistingEvidenceUntouched({ + outputDirectory, + priorEvidencePath, +}: { + outputDirectory: string; + priorEvidencePath: string; +}) { + expect(readFileSync(priorEvidencePath, 'utf8')).toBe( + '{"status":"accepted"}\n', + ); + expect(existsSync(join(outputDirectory, 'markdown'))).toBe(false); + expect(existsSync(join(outputDirectory, 'revision'))).toBe(false); +} + describe('benchmark suite existing-output atomicity', () => { it('rejects an existing evidence directory before mutating prior evidence', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-suite-existing-output-')); temporaryDirectories.push(directory); - const markdownInputPath = join(directory, 'input.md'); - const revisionInputPath = join(directory, 'document-envelope.json'); - const markdownModulePath = join(directory, 'markdown.mjs'); - const revisionModulePath = join(directory, 'revision.mjs'); - const outputDirectory = join(directory, 'evidence'); - const priorEvidencePath = join(outputDirectory, 'accepted-evidence.json'); - const markdownModuleSource = - "export function markdownToHtml(source) { return `

${source}

`; }\n"; - const revisionModuleSource = - "export async function createDocumentEnvelopeRevisionEvidenceBytes() { throw new Error('private downstream failure'); }\n"; + const fixture = makeExistingOutputFixture(directory); + + const result = spawnSync( + process.execPath, + existingOutputArguments({ suite: suitePath, ...fixture }), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); - writeFileSync(markdownInputPath, '# Existing evidence must stay immutable\n', 'utf8'); - writeFileSync( - revisionInputPath, - '{"contractVersion":1,"mode":"markdown","document":"# Existing evidence must stay immutable"}\n', - 'utf8', + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite output directory must not already exist.\n', ); - writeFileSync(markdownModulePath, markdownModuleSource, 'utf8'); - writeFileSync(revisionModulePath, revisionModuleSource, 'utf8'); - mkdirSync(outputDirectory); - writeFileSync(priorEvidencePath, '{"status":"accepted"}\n', 'utf8'); + expectExistingEvidenceUntouched(fixture); + }); + + it('rejects an output directory created after the wrapper preflight', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-suite-core-existing-output-')); + temporaryDirectories.push(directory); + const fixture = makeExistingOutputFixture(directory); const result = spawnSync( process.execPath, - [ - suitePath, - '--input', - markdownInputPath, - '--module', - markdownModulePath, - '--revision-input', - revisionInputPath, - '--revision-module', - revisionModulePath, - '--profile', - 'small', - '--samples', - '1', - '--source-commit-sha', - 'a'.repeat(40), - '--artifact-sha256', - sha256(markdownModuleSource), - '--revision-artifact-sha256', - sha256(revisionModuleSource), - '--runtime-id', - 'node-22.0.0', - '--reference-hardware-id', - `refhw-sha256-${'b'.repeat(64)}`, - '--output', - outputDirectory, - ], + existingOutputArguments({ suite: coreSuitePath, ...fixture }), { cwd: repositoryRoot, encoding: 'utf8', @@ -95,10 +174,6 @@ describe('benchmark suite existing-output atomicity', () => { expect(result.stderr).toBe( 'Benchmark suite output directory must not already exist.\n', ); - expect(readFileSync(priorEvidencePath, 'utf8')).toBe( - '{"status":"accepted"}\n', - ); - expect(existsSync(join(outputDirectory, 'markdown'))).toBe(false); - expect(existsSync(join(outputDirectory, 'revision'))).toBe(false); + expectExistingEvidenceUntouched(fixture); }); }); From 92683c28e68ac588327cb21fb851a30bfe260150 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 15:10:35 -0700 Subject: [PATCH 178/209] fix(perf): fail closed on existing evidence output --- benchmarks/run-current-suite-core.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/run-current-suite-core.mjs b/benchmarks/run-current-suite-core.mjs index d9717fc3..a18479e8 100644 --- a/benchmarks/run-current-suite-core.mjs +++ b/benchmarks/run-current-suite-core.mjs @@ -58,6 +58,8 @@ const MARKDOWN_MODULE_ENTRY = 'package/dist/cwl-markdown.js'; const REVISION_MODULE_ENTRY = 'package/dist/cwl-revision-evidence.js'; const OUTPUT_DIRECTORY_ERROR = 'Benchmark suite output directory must be a non-symlink directory.'; +const OUTPUT_DIRECTORY_EXISTS_ERROR = + 'Benchmark suite output directory must not already exist.'; function matchesArguments(argv, expectedFlags) { return ( @@ -217,7 +219,7 @@ function prepareOutputDirectory(path) { if (!existing.isDirectory()) { throw new Error(OUTPUT_DIRECTORY_ERROR); } - return false; + throw new Error(OUTPUT_DIRECTORY_EXISTS_ERROR); } try { From c16fe9c471e221f8a8d04df8149404b8b895bb44 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 15:13:32 -0700 Subject: [PATCH 179/209] test(perf): preserve existing measurement evidence --- ...eMeasurementProducerOutputHardlink.test.ts | 59 +++++++++++++++++-- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/src/performanceMeasurementProducerOutputHardlink.test.ts b/src/performanceMeasurementProducerOutputHardlink.test.ts index e3210db3..5aeea7a0 100644 --- a/src/performanceMeasurementProducerOutputHardlink.test.ts +++ b/src/performanceMeasurementProducerOutputHardlink.test.ts @@ -52,7 +52,7 @@ function commonArguments( ]; } -function expectHardlinkFailure( +function expectOutputPreservedFailure( script: string, args: string[], sentinel: string, @@ -70,7 +70,7 @@ function expectHardlinkFailure( expect(readFileSync(sentinel, 'utf8')).toBe(originalSentinel); } -describe('benchmark producer output hard-link safety', () => { +describe('benchmark producer output immutability', () => { it('fails closed before Markdown measurement overwrites an unrelated hard link', () => { const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-hardlink-')); const input = join(root, 'document.md'); @@ -85,7 +85,7 @@ describe('benchmark producer output hard-link safety', () => { writeFileSync(sentinel, 'buyer-owned-content\n', 'utf8'); linkSync(sentinel, output); - expectHardlinkFailure( + expectOutputPreservedFailure( markdownScript, commonArguments(input, module, sha256(moduleSource), output), sentinel, @@ -115,7 +115,7 @@ describe('benchmark producer output hard-link safety', () => { writeFileSync(sentinel, 'buyer-owned-content\n', 'utf8'); linkSync(sentinel, output); - expectHardlinkFailure( + expectOutputPreservedFailure( revisionScript, commonArguments(input, module, sha256(moduleSource), output), sentinel, @@ -125,4 +125,55 @@ describe('benchmark producer output hard-link safety', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('fails closed before Markdown measurement overwrites existing evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-existing-output-')); + const input = join(root, 'document.md'); + const module = join(root, 'markdown-module.mjs'); + const output = join(root, 'samples.json'); + const moduleSource = 'export const markdownToHtml = (source) => `

${source}

`;\n'; + + try { + writeFileSync(input, '# Hello\n', 'utf8'); + writeFileSync(module, moduleSource, 'utf8'); + writeFileSync(output, '{"status":"accepted"}\n', 'utf8'); + + expectOutputPreservedFailure( + markdownScript, + commonArguments(input, module, sha256(moduleSource), output), + output, + 'Markdown benchmark output must not already exist.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before revision measurement overwrites existing evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-existing-output-')); + const input = join(root, 'document-envelope.json'); + const module = join(root, 'revision-module.mjs'); + const output = join(root, 'samples.json'); + const moduleSource = [ + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) {', + " return { revision: { digestHex: String(source.byteLength).padStart(64, '0') } };", + '}', + '', + ].join('\n'); + + try { + writeFileSync(input, '{"contractVersion":1}\n', 'utf8'); + writeFileSync(module, moduleSource, 'utf8'); + writeFileSync(output, '{"status":"accepted"}\n', 'utf8'); + + expectOutputPreservedFailure( + revisionScript, + commonArguments(input, module, sha256(moduleSource), output), + output, + 'Revision benchmark output must not already exist.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); From 856c1e9932daf84290136db4cb4cc692ce73e864 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 15:15:41 -0700 Subject: [PATCH 180/209] fix(perf): preserve existing markdown evidence --- benchmarks/measure-markdown.mjs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 68f1cae1..51d437ec 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -30,6 +30,8 @@ const REFERENCE_HARDWARE_ID_PATTERN = /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; const OUTPUT_DIRECTORY_ERROR = 'Markdown benchmark output directory must be a non-symlink directory.'; +const OUTPUT_EXISTS_ERROR = + 'Markdown benchmark output must not already exist.'; function resolveArguments(argv) { const expectedFlags = [ @@ -301,7 +303,7 @@ function writeMeasurementOutput(path, content) { } assertNoSymlinkOutputAncestors(path); try { - writeFileSync(path, content, 'utf8'); + writeFileSync(path, content, { encoding: 'utf8', flag: 'wx' }); } catch { throw new Error('Markdown benchmark output could not be written.'); } @@ -368,6 +370,9 @@ async function main() { if (outputMetadata !== undefined && outputMetadata.nlink !== 1) { throw new Error('Markdown benchmark output must not be multiply linked.'); } + if (outputMetadata !== undefined) { + throw new Error(OUTPUT_EXISTS_ERROR); + } writeMeasurementOutput( args.outputPath, `${JSON.stringify( From 1b7710a5e8bf3300182b6f5ec4bc05be840c57f1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 15:16:57 -0700 Subject: [PATCH 181/209] fix(perf): preserve existing revision evidence --- benchmarks/measure-revision-evidence.mjs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index a209e3e0..5a69b2b6 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -29,6 +29,8 @@ const REFERENCE_HARDWARE_ID_PATTERN = /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; const OUTPUT_DIRECTORY_ERROR = 'Revision benchmark output directory must be a non-symlink directory.'; +const OUTPUT_EXISTS_ERROR = + 'Revision benchmark output must not already exist.'; function resolveArguments(argv) { const expectedFlags = [ @@ -285,7 +287,7 @@ function writeMeasurementOutput(path, content) { } assertNoSymlinkOutputAncestors(path); try { - writeFileSync(path, content, 'utf8'); + writeFileSync(path, content, { encoding: 'utf8', flag: 'wx' }); } catch { throw new Error('Revision benchmark output could not be written.'); } @@ -377,6 +379,9 @@ async function main() { if (outputMetadata !== undefined && outputMetadata.nlink !== 1) { throw new Error('Revision benchmark output must not be multiply linked.'); } + if (outputMetadata !== undefined) { + throw new Error(OUTPUT_EXISTS_ERROR); + } writeMeasurementOutput( args.outputPath, `${JSON.stringify( From 71b1fc5f7aa528450d6270c7949fcdc95e313af7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:09:16 -0700 Subject: [PATCH 182/209] test(perf): require HTML serialization measurement --- ...rmanceHtmlSerializationMeasurement.test.ts | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 src/performanceHtmlSerializationMeasurement.test.ts diff --git a/src/performanceHtmlSerializationMeasurement.test.ts b/src/performanceHtmlSerializationMeasurement.test.ts new file mode 100644 index 00000000..e7d016c2 --- /dev/null +++ b/src/performanceHtmlSerializationMeasurement.test.ts @@ -0,0 +1,95 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-markdown.mjs', +); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const RUNTIME_ID = 'node-22.18.0'; +const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function sha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +describe('HTML serialization performance measurement', () => { + it('measures packed htmlToMarkdown without falling back to markdownToHtml', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-html-serialization-')); + const input = join(root, 'small.html'); + const modulePath = join(root, 'packed-markdown.mjs'); + const output = join(root, 'samples.json'); + + try { + writeFileSync(input, '

Synthetic benchmark fixture

\n', 'utf8'); + writeFileSync( + modulePath, + [ + "export function markdownToHtml() { throw new Error('wrong serialization direction'); }", + "export function htmlToMarkdown(source) { return source.replace(/<[^>]+>/gu, '').trim(); }", + '', + ].join('\n'), + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--operation', + 'html-to-markdown', + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + sha256(modulePath), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + REFERENCE_HARDWARE_ID, + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + + const evidence = JSON.parse(readFileSync(output, 'utf8')) as { + benchmarkId: string; + unit: string; + documentProfile: string; + samples: unknown[]; + }; + expect(evidence.benchmarkId).toBe('html-serialization-small'); + expect(evidence.unit).toBe('ms'); + expect(evidence.documentProfile).toBe('small'); + expect(evidence.samples).toHaveLength(2); + expect( + evidence.samples.every( + (sample) => + typeof sample === 'number' && Number.isFinite(sample) && sample >= 0, + ), + ).toBe(true); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 8851d5dd0e9fbd62bf365fb7f23d86aafe5c0560 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:12:23 -0700 Subject: [PATCH 183/209] feat(perf): measure HTML serialization direction --- benchmarks/measure-markdown.mjs | 117 ++++++++++++++++++++++++-------- 1 file changed, 87 insertions(+), 30 deletions(-) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 51d437ec..424e5a22 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -22,6 +22,10 @@ const MAX_SAMPLES = 1_000; const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const SERIALIZATION_OPERATIONS = new Set([ + 'markdown-to-html', + 'html-to-markdown', +]); const SHA1_PATTERN = /^[0-9a-f]{40}$/u; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; const RUNTIME_ID_PATTERN = @@ -32,32 +36,61 @@ const OUTPUT_DIRECTORY_ERROR = 'Markdown benchmark output directory must be a non-symlink directory.'; const OUTPUT_EXISTS_ERROR = 'Markdown benchmark output must not already exist.'; +const LEGACY_FLAGS = Object.freeze([ + '--input', + '--module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const OPERATION_FLAGS = Object.freeze([ + '--input', + '--module', + '--operation', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); + +function matchesArguments(argv, expectedFlags) { + return ( + argv.length === expectedFlags.length * 2 && + expectedFlags.every((flag, index) => argv[index * 2] === flag) && + expectedFlags.every((_, index) => argv[index * 2 + 1]?.length > 0) + ); +} + +function valuesForArguments(argv, expectedFlags) { + return Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); +} function resolveArguments(argv) { - const expectedFlags = [ - '--input', - '--module', - '--profile', - '--samples', - '--source-commit-sha', - '--artifact-sha256', - '--runtime-id', - '--reference-hardware-id', - '--output', - ]; - if ( - argv.length !== expectedFlags.length * 2 || - expectedFlags.some((flag, index) => argv[index * 2] !== flag) || - expectedFlags.some((_, index) => argv[index * 2 + 1]?.length === 0) - ) { + let values; + let operation = 'markdown-to-html'; + if (matchesArguments(argv, OPERATION_FLAGS)) { + values = valuesForArguments(argv, OPERATION_FLAGS); + operation = values['--operation']; + if (!SERIALIZATION_OPERATIONS.has(operation)) { + throw new Error('Markdown benchmark serialization operation is invalid.'); + } + } else if (matchesArguments(argv, LEGACY_FLAGS)) { + values = valuesForArguments(argv, LEGACY_FLAGS); + } else { throw new Error( 'Usage: node benchmarks/measure-markdown.mjs --input --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', ); } - const values = Object.fromEntries( - expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), - ); const profile = values['--profile']; if (!DOCUMENT_PROFILES.has(profile)) { throw new Error('Markdown benchmark profile is invalid.'); @@ -94,6 +127,7 @@ function resolveArguments(argv) { return Object.freeze({ inputPath: resolve(values['--input']), modulePath: values['--module'], + operation, profile, sampleCount, sourceCommitSha, @@ -286,11 +320,28 @@ async function loadMeasuredModule(modulePath) { } } -function runMeasuredMarkdownToHtml(markdownToHtml, source) { +function serializationContract(operation) { + if (operation === 'html-to-markdown') { + return Object.freeze({ + exportName: 'htmlToMarkdown', + benchmarkPrefix: 'html-serialization', + executionFailure: 'Measured htmlToMarkdown() execution failed.', + returnFailure: 'Measured htmlToMarkdown() must return a string.', + }); + } + return Object.freeze({ + exportName: 'markdownToHtml', + benchmarkPrefix: 'markdown-serialization', + executionFailure: 'Measured markdownToHtml() execution failed.', + returnFailure: 'Measured markdownToHtml() must return a string.', + }); +} + +function runMeasuredSerialization(serializer, source, failureMessage) { try { - return markdownToHtml(source); + return serializer(source); } catch { - throw new Error('Measured markdownToHtml() execution failed.'); + throw new Error(failureMessage); } } @@ -331,24 +382,30 @@ async function main() { verifyMeasuredModuleDigest(modulePath, args.artifactSha256); const measuredModule = await loadMeasuredModule(modulePath); - if (typeof measuredModule.markdownToHtml !== 'function') { - throw new Error('Measured Markdown module must export markdownToHtml().'); + const contract = serializationContract(args.operation); + const serializer = measuredModule[contract.exportName]; + if (typeof serializer !== 'function') { + throw new Error( + `Measured Markdown module must export ${contract.exportName}().`, + ); } - const warmup = runMeasuredMarkdownToHtml( - measuredModule.markdownToHtml, + const warmup = runMeasuredSerialization( + serializer, source, + contract.executionFailure, ); if (typeof warmup !== 'string') { - throw new Error('Measured markdownToHtml() must return a string.'); + throw new Error(contract.returnFailure); } const samples = []; for (let index = 0; index < args.sampleCount; index += 1) { const start = performance.now(); - const output = runMeasuredMarkdownToHtml( - measuredModule.markdownToHtml, + const output = runMeasuredSerialization( + serializer, source, + contract.executionFailure, ); const elapsed = performance.now() - start; if ( @@ -378,7 +435,7 @@ async function main() { `${JSON.stringify( { contractVersion: 1, - benchmarkId: `markdown-serialization-${args.profile}`, + benchmarkId: `${contract.benchmarkPrefix}-${args.profile}`, unit: 'ms', sourceCommitSha: args.sourceCommitSha, artifactSha256: args.artifactSha256, From cf8fd2e3dd917c1874781cfb1000dcc1d0f71c1e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:19:58 -0700 Subject: [PATCH 184/209] test(perf): require HTML serialization in benchmark suite --- ...anceHtmlSerializationSuiteContract.test.ts | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 src/performanceHtmlSerializationSuiteContract.test.ts diff --git a/src/performanceHtmlSerializationSuiteContract.test.ts b/src/performanceHtmlSerializationSuiteContract.test.ts new file mode 100644 index 00000000..4a401681 --- /dev/null +++ b/src/performanceHtmlSerializationSuiteContract.test.ts @@ -0,0 +1,140 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); + +function sha256(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +describe('single-command HTML serialization benchmark contract', () => { + it('measures HTML-to-Markdown serialization alongside Markdown and revision evidence', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-html-suite-')); + const markdownInput = join(directory, 'document.md'); + const htmlInput = join(directory, 'document.html'); + const markdownModule = join(directory, 'markdown.mjs'); + const revisionInput = join(directory, 'document-envelope.json'); + const revisionModule = join(directory, 'revision.mjs'); + const outputDirectory = join(directory, 'evidence'); + const markdownModuleSource = [ + "export function markdownToHtml(source) { return `

${source}

`; }", + "export function htmlToMarkdown(source) { return source.replace(/<[^>]+>/gu, '').trim(); }", + '', + ].join('\n'); + const revisionModuleSource = `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'c'.repeat(64)}' } }; }\n`; + + try { + writeFileSync(markdownInput, '# Buyer benchmark\n', 'utf8'); + writeFileSync(htmlInput, '

Buyer benchmark

\n', 'utf8'); + writeFileSync(markdownModule, markdownModuleSource, 'utf8'); + writeFileSync( + revisionInput, + '{"contractVersion":1,"mode":"markdown","document":"# Buyer benchmark"}\n', + 'utf8', + ); + writeFileSync(revisionModule, revisionModuleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + suitePath, + '--input', + markdownInput, + '--html-input', + htmlInput, + '--module', + markdownModule, + '--revision-input', + revisionInput, + '--revision-module', + revisionModule, + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + sha256(markdownModuleSource), + '--revision-artifact-sha256', + sha256(revisionModuleSource), + '--runtime-id', + 'node-22.0.0', + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + outputDirectory, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + + const manifest = JSON.parse(result.stdout.trim()) as Record; + expect(manifest).toMatchObject({ + htmlSerializationSamples: 'html-serialization/samples.json', + htmlSerializationSummaryJson: + 'html-serialization/summary/summary.json', + htmlSerializationSummaryText: + 'html-serialization/summary/summary.txt', + status: 'completed', + }); + + const samples = JSON.parse( + readFileSync( + join(outputDirectory, 'html-serialization', 'samples.json'), + 'utf8', + ), + ) as { + benchmarkId?: unknown; + documentProfile?: unknown; + samples?: unknown[]; + }; + expect(samples.benchmarkId).toBe('html-serialization-small'); + expect(samples.documentProfile).toBe('small'); + expect(samples.samples).toHaveLength(2); + + const summary = JSON.parse( + readFileSync( + join( + outputDirectory, + 'html-serialization', + 'summary', + 'summary.json', + ), + 'utf8', + ), + ) as { benchmarkId?: unknown }; + expect(summary.benchmarkId).toBe('html-serialization-small'); + expect( + readFileSync( + join( + outputDirectory, + 'html-serialization', + 'summary', + 'summary.txt', + ), + 'utf8', + ), + ).toContain('html-serialization-small'); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); +}); From 94ad4f81791117aaf3ffd70dded27c5006f1101b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:23:56 -0700 Subject: [PATCH 185/209] feat(perf): include HTML serialization in suite --- benchmarks/run-current-suite.mjs | 157 ++++++++++++++++++++++++++++++- 1 file changed, 154 insertions(+), 3 deletions(-) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 21b21403..0eff1312 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -19,7 +19,13 @@ import { assertCleanSourceCheckout } from './source-checkout-provenance.mjs'; const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); const repositoryRoot = resolve(benchmarkDirectory, '..'); const coreRunnerPath = resolve(benchmarkDirectory, 'run-current-suite-core.mjs'); +const markdownMeasurementPath = resolve( + benchmarkDirectory, + 'measure-markdown.mjs', +); +const sampleSummaryPath = resolve(benchmarkDirectory, 'summarize-samples.mjs'); const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; +const MAX_CHILD_OUTPUT_BYTES = 4 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); const legacyFlags = Object.freeze([ @@ -36,6 +42,21 @@ const legacyFlags = Object.freeze([ '--reference-hardware-id', '--output', ]); +const htmlLegacyFlags = Object.freeze([ + '--input', + '--html-input', + '--module', + '--revision-input', + '--revision-module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--revision-artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); const packedFlags = Object.freeze([ '--input', '--revision-input', @@ -59,7 +80,18 @@ function matchesArguments(argv, expectedFlags) { ); } +function valuesForArguments(argv, expectedFlags) { + return Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); +} + +function argumentsForFlags(values, flags) { + return flags.flatMap((flag) => [flag, values[flag]]); +} + function matchingFlags(argv) { + if (matchesArguments(argv, htmlLegacyFlags)) return htmlLegacyFlags; if (matchesArguments(argv, packedFlags)) return packedFlags; if (matchesArguments(argv, legacyFlags)) return legacyFlags; return null; @@ -172,9 +204,118 @@ function snapshotPackedArguments(argv) { }); } -function main(argv) { - assertCleanSourceCheckout(repositoryRoot); - assertFreshOutputDirectory(argv); +function runBoundedNode(scriptPath, args, failureMessage) { + const result = spawnSync(process.execPath, [scriptPath, ...args], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: MAX_CHILD_OUTPUT_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 600_000, + }); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 + ) { + throw new Error(failureMessage); + } + return result.stdout; +} + +function parseCoreManifest(stdout) { + let manifest; + try { + manifest = JSON.parse(stdout.trim()); + } catch { + throw new Error('Benchmark suite internal runner returned invalid evidence.'); + } + if ( + manifest === null || + typeof manifest !== 'object' || + Array.isArray(manifest) || + manifest.status !== 'completed' + ) { + throw new Error('Benchmark suite internal runner returned invalid evidence.'); + } + return manifest; +} + +function runHtmlSerializationSuite(argv) { + const values = valuesForArguments(argv, htmlLegacyFlags); + const outputDirectory = resolve(repositoryRoot, values['--output']); + const legacyArguments = argumentsForFlags(values, legacyFlags); + let coreCompleted = false; + + try { + const coreStdout = runBoundedNode( + coreRunnerPath, + legacyArguments, + 'Benchmark suite internal runner failed.', + ); + coreCompleted = true; + const manifest = parseCoreManifest(coreStdout); + const samplesPath = resolve( + outputDirectory, + 'html-serialization', + 'samples.json', + ); + const summaryDirectory = resolve( + outputDirectory, + 'html-serialization', + 'summary', + ); + + runBoundedNode( + markdownMeasurementPath, + [ + '--input', + values['--html-input'], + '--module', + values['--module'], + '--operation', + 'html-to-markdown', + '--profile', + values['--profile'], + '--samples', + values['--samples'], + '--source-commit-sha', + values['--source-commit-sha'], + '--artifact-sha256', + values['--artifact-sha256'], + '--runtime-id', + values['--runtime-id'], + '--reference-hardware-id', + values['--reference-hardware-id'], + '--output', + samplesPath, + ], + 'Benchmark suite HTML serialization measurement failed.', + ); + runBoundedNode( + sampleSummaryPath, + ['--input', samplesPath, '--output', summaryDirectory], + 'Benchmark suite HTML serialization summary failed.', + ); + + process.stdout.write( + `${JSON.stringify({ + ...manifest, + htmlSerializationSamples: 'html-serialization/samples.json', + htmlSerializationSummaryJson: + 'html-serialization/summary/summary.json', + htmlSerializationSummaryText: + 'html-serialization/summary/summary.txt', + })}\n`, + ); + } catch (error) { + if (coreCompleted) { + rmSync(outputDirectory, { recursive: true, force: true }); + } + throw error; + } +} + +function runExistingSuite(argv) { const snapshotted = snapshotPackedArguments(argv); try { @@ -203,6 +344,16 @@ function main(argv) { } } +function main(argv) { + assertCleanSourceCheckout(repositoryRoot); + assertFreshOutputDirectory(argv); + if (matchesArguments(argv, htmlLegacyFlags)) { + runHtmlSerializationSuite(argv); + return; + } + runExistingSuite(argv); +} + try { main(process.argv.slice(2)); } catch (error) { From 0d8557daf4b5be8ced592312fe95c4fbecafba0f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:28:46 -0700 Subject: [PATCH 186/209] test(perf): require packed HTML serialization evidence --- ...ormancePackedArtifactSuiteContract.test.ts | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts index 35098ef8..5e3eb674 100644 --- a/src/performancePackedArtifactSuiteContract.test.ts +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -59,7 +59,11 @@ function createPackedBenchmarkFixture(directory: string): { ); writeFileSync( join(distDirectory, 'cwl-markdown.js'), - "export function markdownToHtml(source) { return `

${source}

`; }\n", + [ + "export function markdownToHtml(source) { return `

${source}

`; }", + "export function htmlToMarkdown(source) { return source.replace(/<[^>]+>/gu, '').trim(); }", + '', + ].join('\n'), 'utf8', ); writeFileSync( @@ -100,8 +104,10 @@ function packedSuiteArguments(options: { tarballPath: string; }): string[] { const markdownInputPath = join(options.directory, 'input.md'); + const htmlInputPath = join(options.directory, 'input.html'); const revisionInputPath = join(options.directory, 'document-envelope.json'); writeFileSync(markdownInputPath, '# Packed buyer benchmark\n', 'utf8'); + writeFileSync(htmlInputPath, '

Packed buyer benchmark

\n', 'utf8'); writeFileSync( revisionInputPath, '{"contractVersion":1,"mode":"markdown","document":"# Packed buyer benchmark"}\n', @@ -111,6 +117,8 @@ function packedSuiteArguments(options: { suitePath, '--input', markdownInputPath, + '--html-input', + htmlInputPath, '--revision-input', revisionInputPath, '--package-tarball', @@ -166,8 +174,22 @@ describe('packed artifact benchmark suite contract', () => { packageName: '@contextualwisdomlab/cwl-editor', packageVersion: '0.0.0-benchmark-fixture', packageSha256: packed.packageSha256, + htmlSerializationSamples: 'html-serialization/samples.json', + htmlSerializationSummaryJson: + 'html-serialization/summary/summary.json', + htmlSerializationSummaryText: + 'html-serialization/summary/summary.txt', status: 'completed', }); + + const htmlSamples = JSON.parse( + readFileSync( + join(directory, 'evidence', 'html-serialization', 'samples.json'), + 'utf8', + ), + ) as { benchmarkId?: unknown; samples?: unknown[] }; + expect(htmlSamples.benchmarkId).toBe('html-serialization-small'); + expect(htmlSamples.samples).toHaveLength(2); }); it('rejects a runtime identifier that does not match the active Node process', () => { From c220538e5126bcd764ec4f47a4fbb94df414e050 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:39:14 -0700 Subject: [PATCH 187/209] test(perf): reject mismatched benchmark source checkout --- ...ceSourceCheckoutProvenanceContract.test.ts | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/src/performanceSourceCheckoutProvenanceContract.test.ts b/src/performanceSourceCheckoutProvenanceContract.test.ts index 94f37cc2..34ab09d0 100644 --- a/src/performanceSourceCheckoutProvenanceContract.test.ts +++ b/src/performanceSourceCheckoutProvenanceContract.test.ts @@ -15,7 +15,7 @@ const temporaryDirectories: string[] = []; const probe = ` import { assertCleanSourceCheckout } from ${JSON.stringify(helperUrl)}; try { - assertCleanSourceCheckout(process.argv[1]); + assertCleanSourceCheckout(process.argv[1], process.argv[2]); process.stdout.write('clean\\n'); } catch (error) { process.stderr.write(\`${'${error instanceof Error ? error.message : "verification failed"}'}\\n\`); @@ -45,10 +45,17 @@ function createRepository(): string { return directory; } -function probeCheckout(directory: string) { +function headSha(directory: string): string { + return execFileSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: directory, + encoding: 'utf8', + }).trim(); +} + +function probeCheckout(directory: string, expectedCommitSha = headSha(directory)) { return spawnSync( process.execPath, - ['--input-type=module', '--eval', probe, directory], + ['--input-type=module', '--eval', probe, directory, expectedCommitSha], { cwd: repositoryRoot, encoding: 'utf8', @@ -60,7 +67,7 @@ function probeCheckout(directory: string) { } describe('benchmark source checkout provenance', () => { - it('accepts a clean source checkout', () => { + it('accepts a clean source checkout at the claimed source commit', () => { const directory = createRepository(); const result = probeCheckout(directory); @@ -71,6 +78,23 @@ describe('benchmark source checkout provenance', () => { expect(result.stderr).toBe(''); }); + it('rejects a clean checkout when the claimed source commit is not HEAD', () => { + const directory = createRepository(); + const actualHead = headSha(directory); + const mismatchedCommit = actualHead === 'f'.repeat(40) ? 'e'.repeat(40) : 'f'.repeat(40); + const result = probeCheckout(directory, mismatchedCommit); + + expect(result.error).toBeUndefined(); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite source commit does not match checked-out HEAD.\n', + ); + expect(result.stderr).not.toContain(actualHead); + expect(result.stderr).not.toContain(mismatchedCommit); + }); + it('rejects untracked source state without disclosing paths', () => { const directory = createRepository(); const untrackedPath = join(directory, 'untracked-secret-name.txt'); From 67b499c96301ae9df7b531623fee20e3caa8a53f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:39:52 -0700 Subject: [PATCH 188/209] fix(perf): bind benchmark evidence to checked-out source --- benchmarks/source-checkout-provenance.mjs | 34 ++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/benchmarks/source-checkout-provenance.mjs b/benchmarks/source-checkout-provenance.mjs index e2362381..c1cfdf79 100644 --- a/benchmarks/source-checkout-provenance.mjs +++ b/benchmarks/source-checkout-provenance.mjs @@ -2,7 +2,30 @@ import { spawnSync } from 'node:child_process'; const MAX_STATUS_BYTES = 1024 * 1024; -export function assertCleanSourceCheckout(repositoryRoot) { +function checkedOutHeadSha(repositoryRoot) { + const result = spawnSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: MAX_STATUS_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + typeof result.stdout !== 'string' + ) { + throw new Error( + 'Benchmark suite source checkout identity could not be verified.', + ); + } + + return result.stdout.trim(); +} + +export function assertCleanSourceCheckout(repositoryRoot, expectedSourceCommitSha) { const result = spawnSync( 'git', ['status', '--porcelain=v1', '--untracked-files=all'], @@ -31,4 +54,13 @@ export function assertCleanSourceCheckout(repositoryRoot) { 'Benchmark suite source checkout must be clean before acquisition evidence is recorded.', ); } + + if ( + expectedSourceCommitSha !== undefined && + checkedOutHeadSha(repositoryRoot) !== expectedSourceCommitSha + ) { + throw new Error( + 'Benchmark suite source commit does not match checked-out HEAD.', + ); + } } From fd73da75c76dd23de82f5a870200abbae1e8d62f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:43:37 -0700 Subject: [PATCH 189/209] test(perf): bind suite source provenance to checkout --- ...formanceSingleCommandSuiteContract.test.ts | 52 ++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/src/performanceSingleCommandSuiteContract.test.ts b/src/performanceSingleCommandSuiteContract.test.ts index 0eae1cc5..4c0b0dac 100644 --- a/src/performanceSingleCommandSuiteContract.test.ts +++ b/src/performanceSingleCommandSuiteContract.test.ts @@ -17,6 +17,15 @@ import { afterEach, describe, expect, it } from 'vitest'; const repositoryRoot = process.cwd(); const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); const temporaryDirectories: string[] = []; +const currentSourceCommitSha = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); afterEach(() => { for (const directory of temporaryDirectories.splice(0)) { @@ -32,6 +41,7 @@ function benchmarkArguments( revisionModulePath: string, revisionArtifactSha256: string, outputDirectory: string, + sourceCommitSha = currentSourceCommitSha, ): string[] { return [ suitePath, @@ -48,7 +58,7 @@ function benchmarkArguments( '--samples', '2', '--source-commit-sha', - 'a'.repeat(40), + sourceCommitSha, '--artifact-sha256', markdownArtifactSha256, '--revision-artifact-sha256', @@ -131,7 +141,7 @@ describe('single-command benchmark suite contract', () => { contractVersion: 1, documentProfile: 'small', sampleCount: 2, - sourceCommitSha: 'a'.repeat(40), + sourceCommitSha: currentSourceCommitSha, runtimeId: 'node-22.0.0', referenceHardwareId: `refhw-sha256-${'b'.repeat(64)}`, markdownSamples: 'markdown/samples.json', @@ -188,6 +198,44 @@ describe('single-command benchmark suite contract', () => { ).toContain('revision-evidence-small'); }); + it('rejects a claimed source commit that is not the checked-out HEAD', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-source-')); + temporaryDirectories.push(directory); + const outputDirectory = join(directory, 'evidence'); + const inputs = writeBenchmarkInputs(directory); + const mismatchedCommit = + currentSourceCommitSha === 'f'.repeat(40) ? 'e'.repeat(40) : 'f'.repeat(40); + + const result = spawnSync( + process.execPath, + benchmarkArguments( + inputs.markdownInputPath, + inputs.markdownModulePath, + inputs.markdownArtifactSha256, + inputs.revisionInputPath, + inputs.revisionModulePath, + inputs.revisionArtifactSha256, + outputDirectory, + mismatchedCommit, + ), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite source commit does not match checked-out HEAD.\n', + ); + expect(result.stderr).not.toContain(currentSourceCommitSha); + expect(result.stderr).not.toContain(mismatchedCommit); + expect(existsSync(outputDirectory)).toBe(false); + }); + it('removes partial suite evidence when a downstream measurement fails', () => { const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); temporaryDirectories.push(directory); From 5547e4d1d35c21f5f9150f96ef660688d9eb7104 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:43:59 -0700 Subject: [PATCH 190/209] test(perf): use live checkout SHA for HTML suite --- ...erformanceHtmlSerializationSuiteContract.test.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/performanceHtmlSerializationSuiteContract.test.ts b/src/performanceHtmlSerializationSuiteContract.test.ts index 4a401681..e4a6eadb 100644 --- a/src/performanceHtmlSerializationSuiteContract.test.ts +++ b/src/performanceHtmlSerializationSuiteContract.test.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { spawnSync } from 'node:child_process'; +import { execFileSync, spawnSync } from 'node:child_process'; import { mkdtempSync, readFileSync, @@ -12,6 +12,15 @@ import { describe, expect, it } from 'vitest'; const repositoryRoot = process.cwd(); const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const currentSourceCommitSha = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); function sha256(source: string): string { return createHash('sha256').update(source).digest('hex'); @@ -63,7 +72,7 @@ describe('single-command HTML serialization benchmark contract', () => { '--samples', '2', '--source-commit-sha', - 'a'.repeat(40), + currentSourceCommitSha, '--artifact-sha256', sha256(markdownModuleSource), '--revision-artifact-sha256', From aa9be79eda40c0811d0e58b4978080bdea641335 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:45:02 -0700 Subject: [PATCH 191/209] fix(perf): reject wrong source SHA before benchmark evidence --- benchmarks/run-current-suite.mjs | 48 +++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index 0eff1312..b2e405c2 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -97,20 +97,50 @@ function matchingFlags(argv) { return null; } +function inspectOutputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark suite output directory could not be inspected.'); + } +} + function assertFreshOutputDirectory(argv) { const flags = matchingFlags(argv); if (flags === null) return; const outputValueIndex = flags.indexOf('--output') * 2 + 1; const outputDirectory = resolve(repositoryRoot, argv[outputValueIndex]); - let metadata; - try { - metadata = lstatSync(outputDirectory, { throwIfNoEntry: false }); - } catch { - return; + + let current = outputDirectory; + while (true) { + const metadata = inspectOutputPath(current); + if (metadata?.isSymbolicLink()) { + throw new Error( + 'Benchmark suite output directory must be a non-symlink directory.', + ); + } + if (current === outputDirectory && metadata !== undefined) { + if (!metadata.isDirectory()) { + throw new Error( + 'Benchmark suite output directory must be a non-symlink directory.', + ); + } + throw new Error('Benchmark suite output directory must not already exist.'); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; } - if (metadata?.isDirectory()) { - throw new Error('Benchmark suite output directory must not already exist.'); +} + +function claimedLegacySourceCommitSha(argv) { + if (matchesArguments(argv, htmlLegacyFlags)) { + return valuesForArguments(argv, htmlLegacyFlags)['--source-commit-sha']; } + if (matchesArguments(argv, legacyFlags)) { + return valuesForArguments(argv, legacyFlags)['--source-commit-sha']; + } + return undefined; } function readPackedTarballSnapshot(path) { @@ -347,6 +377,10 @@ function runExistingSuite(argv) { function main(argv) { assertCleanSourceCheckout(repositoryRoot); assertFreshOutputDirectory(argv); + const expectedLegacySourceCommitSha = claimedLegacySourceCommitSha(argv); + if (expectedLegacySourceCommitSha !== undefined) { + assertCleanSourceCheckout(repositoryRoot, expectedLegacySourceCommitSha); + } if (matchesArguments(argv, htmlLegacyFlags)) { runHtmlSerializationSuite(argv); return; From 0001bb9ea99083c617854fb7e0f4a8f3c744badf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 16:51:47 -0700 Subject: [PATCH 192/209] fix(perf): restore packed HTML benchmark composition --- benchmarks/run-current-suite.mjs | 215 ++++++++++++++++++++++++++++--- 1 file changed, 194 insertions(+), 21 deletions(-) diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs index b2e405c2..21ca46e5 100644 --- a/benchmarks/run-current-suite.mjs +++ b/benchmarks/run-current-suite.mjs @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { spawnSync } from 'node:child_process'; import { closeSync, @@ -25,9 +26,11 @@ const markdownMeasurementPath = resolve( ); const sampleSummaryPath = resolve(benchmarkDirectory, 'summarize-samples.mjs'); const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; +const MAX_MODULE_BYTES = 16 * 1024 * 1024; const MAX_CHILD_OUTPUT_BYTES = 4 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const PACKED_MARKDOWN_MODULE_ENTRY = 'package/dist/cwl-markdown.js'; const legacyFlags = Object.freeze([ '--input', '--module', @@ -69,8 +72,19 @@ const packedFlags = Object.freeze([ '--reference-hardware-id', '--output', ]); -const packageTarballValueIndex = - packedFlags.indexOf('--package-tarball') * 2 + 1; +const packedHtmlFlags = Object.freeze([ + '--input', + '--html-input', + '--revision-input', + '--package-tarball', + '--package-sha256', + '--profile', + '--samples', + '--source-commit-sha', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); function matchesArguments(argv, expectedFlags) { return ( @@ -92,6 +106,7 @@ function argumentsForFlags(values, flags) { function matchingFlags(argv) { if (matchesArguments(argv, htmlLegacyFlags)) return htmlLegacyFlags; + if (matchesArguments(argv, packedHtmlFlags)) return packedHtmlFlags; if (matchesArguments(argv, packedFlags)) return packedFlags; if (matchesArguments(argv, legacyFlags)) return legacyFlags; return null; @@ -209,10 +224,17 @@ function readPackedTarballSnapshot(path) { } function snapshotPackedArguments(argv) { - if (!matchesArguments(argv, packedFlags)) { + const flags = matchesArguments(argv, packedHtmlFlags) + ? packedHtmlFlags + : matchesArguments(argv, packedFlags) + ? packedFlags + : null; + if (flags === null) { return Object.freeze({ argv, temporaryDirectory: null }); } + const packageTarballValueIndex = + flags.indexOf('--package-tarball') * 2 + 1; const packageTarballPath = resolve(argv[packageTarballValueIndex]); const packageBytes = readPackedTarballSnapshot(packageTarballPath); const temporaryDirectory = mkdtempSync( @@ -252,6 +274,26 @@ function runBoundedNode(scriptPath, args, failureMessage) { return result.stdout; } +function runCoreNodePreservingError(args) { + const result = spawnSync(process.execPath, [coreRunnerPath, ...args], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: MAX_CHILD_OUTPUT_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 600_000, + }); + if (result.error !== undefined || result.signal !== null) { + throw new Error('Benchmark suite internal runner could not complete.'); + } + if (result.status !== 0) { + const message = result.stderr.trimEnd(); + throw new Error( + message.length > 0 ? message : 'Benchmark suite internal runner failed.', + ); + } + return result.stdout; +} + function parseCoreManifest(stdout) { let manifest; try { @@ -270,6 +312,61 @@ function parseCoreManifest(stdout) { return manifest; } +function readPackedMarkdownModule(tarballPath) { + const result = spawnSync( + 'tar', + ['-xOzf', tarballPath, PACKED_MARKDOWN_MODULE_ENTRY], + { + cwd: repositoryRoot, + maxBuffer: MAX_MODULE_BYTES + 1, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 30_000, + }, + ); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !Buffer.isBuffer(result.stdout) || + result.stdout.byteLength > MAX_MODULE_BYTES + ) { + throw new Error('Benchmark suite packed Markdown module could not be read.'); + } + return result.stdout; +} + +function assertPackedSnapshotDigest(tarballPath, expectedSha256) { + const actualSha256 = createHash('sha256') + .update(readPackedTarballSnapshot(tarballPath)) + .digest('hex'); + if (actualSha256 !== expectedSha256) { + throw new Error('Benchmark suite package digest does not match the packed artifact.'); + } +} + +function htmlSerializationEvidenceArguments(values, modulePath, artifactSha256) { + return [ + '--input', + values['--html-input'], + '--module', + modulePath, + '--operation', + 'html-to-markdown', + '--profile', + values['--profile'], + '--samples', + values['--samples'], + '--source-commit-sha', + values['--source-commit-sha'], + '--artifact-sha256', + artifactSha256, + '--runtime-id', + values['--runtime-id'], + '--reference-hardware-id', + values['--reference-hardware-id'], + ]; +} + function runHtmlSerializationSuite(argv) { const values = valuesForArguments(argv, htmlLegacyFlags); const outputDirectory = resolve(repositoryRoot, values['--output']); @@ -298,24 +395,85 @@ function runHtmlSerializationSuite(argv) { runBoundedNode( markdownMeasurementPath, [ - '--input', - values['--html-input'], - '--module', - values['--module'], - '--operation', - 'html-to-markdown', - '--profile', - values['--profile'], - '--samples', - values['--samples'], - '--source-commit-sha', - values['--source-commit-sha'], - '--artifact-sha256', - values['--artifact-sha256'], - '--runtime-id', - values['--runtime-id'], - '--reference-hardware-id', - values['--reference-hardware-id'], + ...htmlSerializationEvidenceArguments( + values, + values['--module'], + values['--artifact-sha256'], + ), + '--output', + samplesPath, + ], + 'Benchmark suite HTML serialization measurement failed.', + ); + runBoundedNode( + sampleSummaryPath, + ['--input', samplesPath, '--output', summaryDirectory], + 'Benchmark suite HTML serialization summary failed.', + ); + + process.stdout.write( + `${JSON.stringify({ + ...manifest, + htmlSerializationSamples: 'html-serialization/samples.json', + htmlSerializationSummaryJson: + 'html-serialization/summary/summary.json', + htmlSerializationSummaryText: + 'html-serialization/summary/summary.txt', + })}\n`, + ); + } catch (error) { + if (coreCompleted) { + rmSync(outputDirectory, { recursive: true, force: true }); + } + throw error; + } +} + +function runPackedHtmlSerializationSuite(argv) { + const values = valuesForArguments(argv, packedHtmlFlags); + const outputDirectory = resolve(repositoryRoot, values['--output']); + const coreArguments = argumentsForFlags(values, packedFlags); + const snapshotted = snapshotPackedArguments(coreArguments); + let coreCompleted = false; + + try { + const coreStdout = runCoreNodePreservingError(snapshotted.argv); + coreCompleted = true; + const manifest = parseCoreManifest(coreStdout); + const snapshotValues = valuesForArguments(snapshotted.argv, packedFlags); + const snapshotTarballPath = snapshotValues['--package-tarball']; + const markdownModuleBytes = readPackedMarkdownModule(snapshotTarballPath); + const markdownModulePath = join( + snapshotted.temporaryDirectory, + 'cwl-markdown.mjs', + ); + try { + writeFileSync(markdownModulePath, markdownModuleBytes, { mode: 0o600 }); + } catch { + throw new Error('Benchmark suite packed Markdown module could not be prepared.'); + } + const markdownArtifactSha256 = createHash('sha256') + .update(markdownModuleBytes) + .digest('hex'); + const samplesPath = resolve( + outputDirectory, + 'html-serialization', + 'samples.json', + ); + const summaryDirectory = resolve( + outputDirectory, + 'html-serialization', + 'summary', + ); + + runBoundedNode( + markdownMeasurementPath, + [ + ...htmlSerializationEvidenceArguments( + values, + markdownModulePath, + markdownArtifactSha256, + ), '--output', samplesPath, ], @@ -326,6 +484,10 @@ function runHtmlSerializationSuite(argv) { ['--input', samplesPath, '--output', summaryDirectory], 'Benchmark suite HTML serialization summary failed.', ); + assertPackedSnapshotDigest( + snapshotTarballPath, + values['--package-sha256'], + ); process.stdout.write( `${JSON.stringify({ @@ -342,6 +504,13 @@ function runHtmlSerializationSuite(argv) { rmSync(outputDirectory, { recursive: true, force: true }); } throw error; + } finally { + if (snapshotted.temporaryDirectory !== null) { + rmSync(snapshotted.temporaryDirectory, { + recursive: true, + force: true, + }); + } } } @@ -385,6 +554,10 @@ function main(argv) { runHtmlSerializationSuite(argv); return; } + if (matchesArguments(argv, packedHtmlFlags)) { + runPackedHtmlSerializationSuite(argv); + return; + } runExistingSuite(argv); } From 48363bab55a61bff1fdece63d4322eee457691de Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 01:36:27 -0700 Subject: [PATCH 193/209] test(perf): reject false producer provenance --- ...ntProducerSourceProvenanceContract.test.ts | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 src/performanceMeasurementProducerSourceProvenanceContract.test.ts diff --git a/src/performanceMeasurementProducerSourceProvenanceContract.test.ts b/src/performanceMeasurementProducerSourceProvenanceContract.test.ts new file mode 100644 index 00000000..3d64d83a --- /dev/null +++ b/src/performanceMeasurementProducerSourceProvenanceContract.test.ts @@ -0,0 +1,180 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const markdownMeasurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-markdown.mjs', +); +const revisionMeasurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-revision-evidence.mjs', +); +const currentSourceCommitSha = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const mismatchedSourceCommitSha = + currentSourceCommitSha === 'f'.repeat(40) ? 'e'.repeat(40) : 'f'.repeat(40); +const activeRuntimeId = `node-${process.versions.node}`; +const mismatchedRuntimeId = + activeRuntimeId === 'node-99.99.99' ? 'node-98.98.98' : 'node-99.99.99'; +const referenceHardwareId = `refhw-sha256-${'b'.repeat(64)}`; + +function sha256(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +function markdownInvocation( + root: string, + sourceCommitSha: string, + runtimeId: string, +): { result: ReturnType; output: string } { + const input = join(root, 'document.md'); + const modulePath = join(root, 'markdown.mjs'); + const output = join(root, 'markdown-samples.json'); + const moduleSource = + 'export function markdownToHtml(source) { return `

${source}

`; }\n'; + writeFileSync(input, '# Provenance fixture\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + markdownMeasurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + sourceCommitSha, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return { result, output }; +} + +function revisionInvocation( + root: string, + sourceCommitSha: string, + runtimeId: string, +): { result: ReturnType; output: string } { + const input = join(root, 'document-envelope.json'); + const modulePath = join(root, 'revision.mjs'); + const output = join(root, 'revision-samples.json'); + const moduleSource = `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'c'.repeat(64)}' } }; }\n`; + writeFileSync( + input, + '{"contractVersion":1,"mode":"markdown","document":"# Provenance fixture"}\n', + 'utf8', + ); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + revisionMeasurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + sourceCommitSha, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return { result, output }; +} + +describe('direct benchmark producer source/runtime provenance', () => { + it.each([ + ['Markdown', markdownInvocation], + ['revision', revisionInvocation], + ] as const)( + 'rejects a caller-supplied source SHA that is not the checked-out HEAD for %s evidence', + (_label, invoke) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-producer-source-')); + try { + const { result, output } = invoke( + root, + mismatchedSourceCommitSha, + activeRuntimeId, + ); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark measurement source commit does not match checked-out HEAD.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); + + it.each([ + ['Markdown', markdownInvocation], + ['revision', revisionInvocation], + ] as const)( + 'rejects a caller-supplied runtime ID that is not the active Node runtime for %s evidence', + (_label, invoke) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-producer-runtime-')); + try { + const { result, output } = invoke( + root, + currentSourceCommitSha, + mismatchedRuntimeId, + ); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark measurement runtime ID must match the active Node runtime.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); +}); From a83a4251e337197446034fb5834c36b4c059966e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 01:40:20 -0700 Subject: [PATCH 194/209] fix(perf): bind Markdown evidence to live source --- benchmarks/measure-markdown.mjs | 35 +++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 424e5a22..14904b70 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; import { performance } from 'node:perf_hooks'; import { closeSync, @@ -15,6 +16,8 @@ import { import { dirname, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; +const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(benchmarkDirectory, '..'); const MAX_INPUT_BYTES = 16 * 1024 * 1024; const MAX_MODULE_BYTES = 16 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; @@ -138,6 +141,37 @@ function resolveArguments(argv) { }); } +function assertMeasurementProvenance(sourceCommitSha, runtimeId) { + const result = spawnSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + const checkoutSha = result.stdout?.trim(); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !SHA1_PATTERN.test(checkoutSha ?? '') + ) { + throw new Error( + 'Benchmark measurement source commit could not be verified against the current checkout.', + ); + } + if (sourceCommitSha !== checkoutSha) { + throw new Error( + 'Benchmark measurement source commit does not match checked-out HEAD.', + ); + } + if (runtimeId !== `node-${process.versions.node}`) { + throw new Error( + 'Benchmark measurement runtime ID must match the active Node runtime.', + ); + } +} + function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversizedMessage) { let pathMetadata; try { @@ -380,6 +414,7 @@ async function main() { ); } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); const measuredModule = await loadMeasuredModule(modulePath); const contract = serializationContract(args.operation); From d4e7883c943e37867dca512efae284646d3dacae Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 01:41:24 -0700 Subject: [PATCH 195/209] fix(perf): bind revision evidence to live source --- benchmarks/measure-revision-evidence.mjs | 35 ++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index 5a69b2b6..782063cb 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; import { performance } from 'node:perf_hooks'; import { closeSync, @@ -15,6 +16,8 @@ import { import { dirname, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; +const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(benchmarkDirectory, '..'); const MAX_INPUT_BYTES = 16 * 1024 * 1024; const MAX_MODULE_BYTES = 16 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; @@ -103,6 +106,37 @@ function resolveArguments(argv) { }); } +function assertMeasurementProvenance(sourceCommitSha, runtimeId) { + const result = spawnSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + const checkoutSha = result.stdout?.trim(); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !SHA1_PATTERN.test(checkoutSha ?? '') + ) { + throw new Error( + 'Benchmark measurement source commit could not be verified against the current checkout.', + ); + } + if (sourceCommitSha !== checkoutSha) { + throw new Error( + 'Benchmark measurement source commit does not match checked-out HEAD.', + ); + } + if (runtimeId !== `node-${process.versions.node}`) { + throw new Error( + 'Benchmark measurement runtime ID must match the active Node runtime.', + ); + } +} + function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversizedMessage) { let pathMetadata; try { @@ -340,6 +374,7 @@ async function main() { ); } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); const measuredModule = await loadMeasuredModule(modulePath); if ( From 19c8c28abdcf6aedf658a7f047f9a55ca03c1b6e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 01:44:33 -0700 Subject: [PATCH 196/209] fix(test): preserve producer provenance typing --- ...ormanceMeasurementProducerSourceProvenanceContract.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/performanceMeasurementProducerSourceProvenanceContract.test.ts b/src/performanceMeasurementProducerSourceProvenanceContract.test.ts index 3d64d83a..865d993b 100644 --- a/src/performanceMeasurementProducerSourceProvenanceContract.test.ts +++ b/src/performanceMeasurementProducerSourceProvenanceContract.test.ts @@ -38,7 +38,7 @@ function markdownInvocation( root: string, sourceCommitSha: string, runtimeId: string, -): { result: ReturnType; output: string } { +) { const input = join(root, 'document.md'); const modulePath = join(root, 'markdown.mjs'); const output = join(root, 'markdown-samples.json'); @@ -83,7 +83,7 @@ function revisionInvocation( root: string, sourceCommitSha: string, runtimeId: string, -): { result: ReturnType; output: string } { +) { const input = join(root, 'document-envelope.json'); const modulePath = join(root, 'revision.mjs'); const output = join(root, 'revision-samples.json'); From fa91c06e13210c5a35368c143e7a2049f1df90a1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 01:49:48 -0700 Subject: [PATCH 197/209] fix(test): use live Markdown benchmark provenance --- ...ormanceMarkdownMeasurementContract.test.ts | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/performanceMarkdownMeasurementContract.test.ts b/src/performanceMarkdownMeasurementContract.test.ts index 87edd0e3..0ec5fdc2 100644 --- a/src/performanceMarkdownMeasurementContract.test.ts +++ b/src/performanceMarkdownMeasurementContract.test.ts @@ -25,14 +25,23 @@ interface BenchmarkSamples { readonly samples: number[]; } +const repositoryRoot = process.cwd(); const measurementScript = resolve( - process.cwd(), + repositoryRoot, 'benchmarks/measure-markdown.mjs', ); -const summaryScript = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); -const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const summaryScript = resolve(repositoryRoot, 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); const FALLBACK_ARTIFACT_SHA256 = 'b'.repeat(64); -const RUNTIME_ID = 'node-22.18.0'; +const RUNTIME_ID = `node-${process.versions.node}`; const HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; function fileSha256(path: string): string { @@ -89,7 +98,7 @@ describe('Markdown runtime measurement contract', () => { execFileSync( process.execPath, measurementArguments(input, modulePath, samplesPath, artifactSha256), - { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'pipe'] }, + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, ); const samples = JSON.parse( @@ -117,7 +126,7 @@ describe('Markdown runtime measurement contract', () => { execFileSync( process.execPath, [summaryScript, '--input', samplesPath, '--output', summaryDirectory], - { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'pipe'] }, + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, ); const summary = JSON.parse( readFileSync(join(summaryDirectory, 'summary.json'), 'utf8'), @@ -144,7 +153,7 @@ describe('Markdown runtime measurement contract', () => { const result = spawnSync( process.execPath, measurementArguments(input, modulePath, samplesPath), - { cwd: process.cwd(), encoding: 'utf8' }, + { cwd: repositoryRoot, encoding: 'utf8' }, ); expect(result.status).toBe(1); @@ -176,7 +185,7 @@ describe('Markdown runtime measurement contract', () => { const result = spawnSync( process.execPath, measurementArguments(input, modulePath, samplesPath), - { cwd: process.cwd(), encoding: 'utf8' }, + { cwd: repositoryRoot, encoding: 'utf8' }, ); expect(result.status).toBe(1); @@ -202,7 +211,7 @@ describe('Markdown runtime measurement contract', () => { const result = spawnSync( process.execPath, measurementArguments(input, modulePath, modulePath), - { cwd: process.cwd(), encoding: 'utf8' }, + { cwd: repositoryRoot, encoding: 'utf8' }, ); expect(result.status).toBe(1); @@ -229,7 +238,7 @@ describe('Markdown runtime measurement contract', () => { 'https://example.invalid/markdown.mjs', samplesPath, ), - { cwd: process.cwd(), encoding: 'utf8' }, + { cwd: repositoryRoot, encoding: 'utf8' }, ); expect(result.status).toBe(1); From d46dc2f38754b140841a4dd6b7f0c771dbcd65da Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 01:51:03 -0700 Subject: [PATCH 198/209] fix(test): use live revision benchmark provenance --- ...ormanceRevisionMeasurementContract.test.ts | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts index d7258f6a..8df70188 100644 --- a/src/performanceRevisionMeasurementContract.test.ts +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -23,13 +23,22 @@ interface BenchmarkSamples { readonly samples: number[]; } +const repositoryRoot = process.cwd(); const measurementScript = resolve( - process.cwd(), + repositoryRoot, 'benchmarks/measure-revision-evidence.mjs', ); -const summaryScript = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); -const SOURCE_COMMIT_SHA = 'a'.repeat(40); -const RUNTIME_ID = 'node-22.18.0'; +const summaryScript = resolve(repositoryRoot, 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const RUNTIME_ID = `node-${process.versions.node}`; const HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; function fileSha256(path: string): string { @@ -100,7 +109,7 @@ describe('revision-evidence runtime measurement contract', () => { ); execFileSync(process.execPath, argumentsFor(input, modulePath, samplesPath), { - cwd: process.cwd(), + cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'], }); @@ -130,7 +139,7 @@ describe('revision-evidence runtime measurement contract', () => { execFileSync( process.execPath, [summaryScript, '--input', samplesPath, '--output', summaryDirectory], - { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'pipe'] }, + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, ); expect( JSON.parse(readFileSync(join(summaryDirectory, 'summary.json'), 'utf8')), @@ -156,7 +165,7 @@ describe('revision-evidence runtime measurement contract', () => { const result = spawnSync( process.execPath, argumentsFor(input, modulePath, samplesPath), - { cwd: process.cwd(), encoding: 'utf8' }, + { cwd: repositoryRoot, encoding: 'utf8' }, ); expect(result.status).toBe(1); @@ -187,7 +196,7 @@ describe('revision-evidence runtime measurement contract', () => { const result = spawnSync( process.execPath, argumentsFor(input, modulePath, samplesPath), - { cwd: process.cwd(), encoding: 'utf8' }, + { cwd: repositoryRoot, encoding: 'utf8' }, ); expect(result.status).toBe(1); @@ -221,7 +230,7 @@ describe('revision-evidence runtime measurement contract', () => { const result = spawnSync( process.execPath, argumentsFor(input, modulePath, samplesPath), - { cwd: process.cwd(), encoding: 'utf8' }, + { cwd: repositoryRoot, encoding: 'utf8' }, ); expect(result.status).toBe(1); @@ -253,7 +262,7 @@ describe('revision-evidence runtime measurement contract', () => { const result = spawnSync( process.execPath, argumentsFor(input, modulePath, samplesPath), - { cwd: process.cwd(), encoding: 'utf8' }, + { cwd: repositoryRoot, encoding: 'utf8' }, ); expect(result.status).toBe(1); From c3d9fa6437ed11dccd77b051553aabebefc616c1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 01:52:10 -0700 Subject: [PATCH 199/209] fix(test): use active suite runtime provenance --- src/performanceSingleCommandSuiteContract.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/performanceSingleCommandSuiteContract.test.ts b/src/performanceSingleCommandSuiteContract.test.ts index 4c0b0dac..ca04e242 100644 --- a/src/performanceSingleCommandSuiteContract.test.ts +++ b/src/performanceSingleCommandSuiteContract.test.ts @@ -26,6 +26,7 @@ const currentSourceCommitSha = execFileSync( stdio: ['ignore', 'pipe', 'pipe'], }, ).trim(); +const currentRuntimeId = `node-${process.versions.node}`; afterEach(() => { for (const directory of temporaryDirectories.splice(0)) { @@ -64,7 +65,7 @@ function benchmarkArguments( '--revision-artifact-sha256', revisionArtifactSha256, '--runtime-id', - 'node-22.0.0', + currentRuntimeId, '--reference-hardware-id', `refhw-sha256-${'b'.repeat(64)}`, '--output', @@ -142,7 +143,7 @@ describe('single-command benchmark suite contract', () => { documentProfile: 'small', sampleCount: 2, sourceCommitSha: currentSourceCommitSha, - runtimeId: 'node-22.0.0', + runtimeId: currentRuntimeId, referenceHardwareId: `refhw-sha256-${'b'.repeat(64)}`, markdownSamples: 'markdown/samples.json', markdownSummaryJson: 'markdown/summary/summary.json', From 65cb3023dfd6548900ae6d56c492dc769c01dd6c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 02:07:23 -0700 Subject: [PATCH 200/209] fix(test): bind producer immutability fixtures to active provenance --- ...erformanceMeasurementProducerOutputHardlink.test.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/performanceMeasurementProducerOutputHardlink.test.ts b/src/performanceMeasurementProducerOutputHardlink.test.ts index 5aeea7a0..ffc4d283 100644 --- a/src/performanceMeasurementProducerOutputHardlink.test.ts +++ b/src/performanceMeasurementProducerOutputHardlink.test.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { spawnSync } from 'node:child_process'; +import { execFileSync, spawnSync } from 'node:child_process'; import { linkSync, mkdtempSync, @@ -16,8 +16,12 @@ const revisionScript = resolve( process.cwd(), 'benchmarks/measure-revision-evidence.mjs', ); -const sourceCommitSha = 'a'.repeat(40); -const runtimeId = 'node-22.18.0'; +const sourceCommitSha = execFileSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: process.cwd(), + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], +}).trim(); +const runtimeId = `node-${process.versions.node}`; const referenceHardwareId = 'github-actions-ubuntu-24.04-x64'; function sha256(content: string): string { From 8f1480adf16cfb41d6274dc57c94a3bd2e737ef5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 02:09:57 -0700 Subject: [PATCH 201/209] fix(test): bind Markdown privacy fixtures to active provenance --- ...MarkdownMeasurementErrorPrivacyContract.test.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts index c069b76f..3e172ebb 100644 --- a/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts +++ b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { spawnSync } from 'node:child_process'; +import { execFileSync, spawnSync } from 'node:child_process'; import { existsSync, mkdtempSync, @@ -11,8 +11,16 @@ import { join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; const script = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); -const SOURCE_COMMIT_SHA = 'a'.repeat(40); -const RUNTIME_ID = 'node-22.18.0'; +const SOURCE_COMMIT_SHA = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: process.cwd(), + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const RUNTIME_ID = `node-${process.versions.node}`; const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; function sha256(value: string) { From aba2d3be70830dfd53c31038573842dcfe36cdd3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 02:11:55 -0700 Subject: [PATCH 202/209] fix(test): bind HTML serialization fixtures to active provenance --- ...performanceHtmlSerializationMeasurement.test.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/performanceHtmlSerializationMeasurement.test.ts b/src/performanceHtmlSerializationMeasurement.test.ts index e7d016c2..7eaff65e 100644 --- a/src/performanceHtmlSerializationMeasurement.test.ts +++ b/src/performanceHtmlSerializationMeasurement.test.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { spawnSync } from 'node:child_process'; +import { execFileSync, spawnSync } from 'node:child_process'; import { mkdtempSync, readFileSync, @@ -14,8 +14,16 @@ const measurementScript = resolve( process.cwd(), 'benchmarks/measure-markdown.mjs', ); -const SOURCE_COMMIT_SHA = 'a'.repeat(40); -const RUNTIME_ID = 'node-22.18.0'; +const SOURCE_COMMIT_SHA = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: process.cwd(), + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const RUNTIME_ID = `node-${process.versions.node}`; const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; function sha256(path: string): string { From d334b959eeb132b17afc1d9a41c139e0ad66cf10 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 04:45:03 -0700 Subject: [PATCH 203/209] fix(test): bind HTML suite runtime to active provenance --- src/performanceHtmlSerializationSuiteContract.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/performanceHtmlSerializationSuiteContract.test.ts b/src/performanceHtmlSerializationSuiteContract.test.ts index e4a6eadb..98dd9389 100644 --- a/src/performanceHtmlSerializationSuiteContract.test.ts +++ b/src/performanceHtmlSerializationSuiteContract.test.ts @@ -21,6 +21,7 @@ const currentSourceCommitSha = execFileSync( stdio: ['ignore', 'pipe', 'pipe'], }, ).trim(); +const currentRuntimeId = `node-${process.versions.node}`; function sha256(source: string): string { return createHash('sha256').update(source).digest('hex'); @@ -78,7 +79,7 @@ describe('single-command HTML serialization benchmark contract', () => { '--revision-artifact-sha256', sha256(revisionModuleSource), '--runtime-id', - 'node-22.0.0', + currentRuntimeId, '--reference-hardware-id', `refhw-sha256-${'b'.repeat(64)}`, '--output', @@ -146,4 +147,4 @@ describe('single-command HTML serialization benchmark contract', () => { rmSync(directory, { recursive: true, force: true }); } }); -}); +}); \ No newline at end of file From fc54c8c30872354fc6ac6bbf0b4803445d17a40c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 11:09:55 -0700 Subject: [PATCH 204/209] test(perf): reject source movement during Office measurement --- .../test_performance_source_stability.py | 176 ++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 office/tests/test_performance_source_stability.py diff --git a/office/tests/test_performance_source_stability.py b/office/tests/test_performance_source_stability.py new file mode 100644 index 00000000..0c51cb6e --- /dev/null +++ b/office/tests/test_performance_source_stability.py @@ -0,0 +1,176 @@ +"""Source-stability contracts for Office benchmark provenance.""" + +from __future__ import annotations + +import json +import runpy +import subprocess +import sys +from pathlib import Path + +import pytest + + +MULTILINGUAL_PARAGRAPH = ( + "English: deterministic Office rendering fixture. 한국어: 합성 성능 문서입니다. " + "日本語: 合成性能文書です。 中文: 这是合成性能文档。 " + "Tiếng Việt: Đây là tài liệu hiệu năng tổng hợp." +) + + +def _docx_page(page_number: int) -> list[dict[str, object]]: + page = str(page_number).zfill(3) + return [ + {"type": "heading", "level": 1, "text": f"Synthetic page {page}"}, + { + "type": "paragraph", + "text": f"{MULTILINGUAL_PARAGRAPH} Page {page}.", + "alignment": "justify", + }, + { + "type": "rich_paragraph", + "runs": [ + {"text": f"Page {page} summary: ", "bold": True}, + {"text": "deterministic ", "italic": True}, + {"text": "Office rendering fixture.", "underline": True}, + ], + }, + { + "type": "bullet_list", + "ordered": False, + "items": [ + f"page {page} item A", + f"page {page} item B", + f"page {page} item C", + ], + }, + { + "type": "table", + "headers": ["Page", "Metric", "Value"], + "rows": [ + [page, "latency-sample", page_number], + [page, "memory-sample", page_number * 2], + [page, "revision-sample", page_number * 3], + [page, "render-sample", page_number * 4], + ], + }, + ] + + +def _canonical_docx_small_fixture_bytes() -> bytes: + blocks: list[dict[str, object]] = [] + for page_number in range(1, 3): + blocks.extend(_docx_page(page_number)) + if page_number < 2: + blocks.append({"type": "page_break"}) + request = { + "format": "docx", + "title": "Inkspan synthetic DOCX benchmark: small", + "author": "Inkspan synthetic benchmark", + "subject": "Deterministic synthetic performance fixture", + "blocks": blocks, + } + return (json.dumps(request, ensure_ascii=False, indent=2) + "\n").encode() + + +def _git(repository: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + cwd=repository, + check=True, + capture_output=True, + text=True, + ) + + +def _initialize_clean_repository(repository: Path) -> Path: + repository.mkdir() + _git(repository, "init", "-q") + tracked = repository / "tracked.txt" + tracked.write_text("before\n", encoding="utf-8") + _git(repository, "add", "tracked.txt") + subprocess.run( + [ + "git", + "-c", + "user.name=Inkspan benchmark test", + "-c", + "user.email=benchmark-test@example.invalid", + "commit", + "-qm", + "initial", + ], + cwd=repository, + check=True, + capture_output=True, + text=True, + ) + return tracked + + +def test_office_measurement_rejects_clean_source_revision_move_during_sampling( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Never emit benchmark evidence if a clean checkout advances while samples are acquired.""" + + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + namespace = runpy.run_path(str(script), run_name="inkspan_measure_render_source_stability") + main = namespace["_main"] + globals_ = main.__globals__ + + repository = tmp_path / "isolated-repository" + tracked = _initialize_clean_repository(repository) + globals_["REPOSITORY_ROOT"] = repository + + request_path = tmp_path / "synthetic-docx-small.json" + request_path.write_bytes(_canonical_docx_small_fixture_bytes()) + + def _move_revision( + _payload_bytes: bytes, + format_name: str, + _profile: str, + ) -> dict[str, object]: + tracked.write_text("after\n", encoding="utf-8") + _git(repository, "add", "tracked.txt") + subprocess.run( + [ + "git", + "-c", + "user.name=Inkspan benchmark test", + "-c", + "user.email=benchmark-test@example.invalid", + "commit", + "-qm", + "advance", + ], + cwd=repository, + check=True, + capture_output=True, + text=True, + ) + return {"format": format_name, "durationMs": 1.0, "peakRssBytes": 1024} + + globals_["_run_sample"] = _move_revision + + result = main( + [ + "--input", + str(request_path), + "--format", + "docx", + "--fixture-profile", + "small", + "--iterations", + "1", + "--reference-hardware", + "pytest-reference", + ] + ) + output = capsys.readouterr() + + assert result == 2 + assert output.out == "" + assert "benchmark source revision changed during measurement" in output.err + assert str(repository) not in output.err + assert str(request_path) not in output.err From 227418d76b51a664a003ea66bbd7320325eab52d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 11:11:50 -0700 Subject: [PATCH 205/209] fix(perf): bind Office evidence to stable source revision --- office/benchmarks/measure_render.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/office/benchmarks/measure_render.py b/office/benchmarks/measure_render.py index 4789d264..a2a62e76 100644 --- a/office/benchmarks/measure_render.py +++ b/office/benchmarks/measure_render.py @@ -286,6 +286,9 @@ def _main(argv: list[str] | None = None) -> int: Path(args.input), expected_bytes, expected_sha256 ) samples = [_run_sample(payload_bytes, args.format, profile) for _ in range(iterations)] + observed_source_sha = _source_sha() + if observed_source_sha != source_sha: + raise BenchmarkContractError("benchmark source revision changed during measurement") duration_values = [float(sample["durationMs"]) for sample in samples] rss_values = [float(sample["peakRssBytes"]) for sample in samples] evidence = { From 483c3fc54b8cecfd2ce18bb64a313d86e2230ebe Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 11:18:50 -0700 Subject: [PATCH 206/209] test(perf): reject source movement during direct measurement --- ...entProducerSourceStabilityContract.test.ts | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 src/performanceMeasurementProducerSourceStabilityContract.test.ts diff --git a/src/performanceMeasurementProducerSourceStabilityContract.test.ts b/src/performanceMeasurementProducerSourceStabilityContract.test.ts new file mode 100644 index 00000000..2c3f402b --- /dev/null +++ b/src/performanceMeasurementProducerSourceStabilityContract.test.ts @@ -0,0 +1,167 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + chmodSync, + existsSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { delimiter, join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const markdownMeasurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-markdown.mjs', +); +const revisionMeasurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-revision-evidence.mjs', +); +const firstSourceSha = 'a'.repeat(40); +const movedSourceSha = 'b'.repeat(40); +const runtimeId = `node-${process.versions.node}`; +const referenceHardwareId = `refhw-sha256-${'c'.repeat(64)}`; + +function sha256(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +function movingGitEnvironment(root: string): NodeJS.ProcessEnv { + const fakeBin = join(root, 'fake-bin'); + const statePath = join(root, 'git-invocations.txt'); + const fakeGit = join(fakeBin, 'git'); + const script = `#!/usr/bin/env node +const { existsSync, readFileSync, writeFileSync } = require('node:fs'); +const state = process.env.INKSPAN_FAKE_GIT_STATE; +const first = process.env.INKSPAN_FAKE_GIT_FIRST_SHA; +const moved = process.env.INKSPAN_FAKE_GIT_MOVED_SHA; +const count = existsSync(state) ? Number(readFileSync(state, 'utf8')) : 0; +writeFileSync(state, String(count + 1), 'utf8'); +process.stdout.write(\`${'${count === 0 ? first : moved}'}\\n\`); +`; + writeFileSync(fakeGit, script, { encoding: 'utf8', mode: 0o755 }); + chmodSync(fakeGit, 0o755); + return { + ...process.env, + PATH: `${fakeBin}${delimiter}${process.env.PATH ?? ''}`, + INKSPAN_FAKE_GIT_STATE: statePath, + INKSPAN_FAKE_GIT_FIRST_SHA: firstSourceSha, + INKSPAN_FAKE_GIT_MOVED_SHA: movedSourceSha, + }; +} + +function markdownInvocation(root: string) { + const input = join(root, 'document.md'); + const modulePath = join(root, 'markdown.mjs'); + const output = join(root, 'markdown-samples.json'); + const moduleSource = + 'export function markdownToHtml(source) { return `

${source}

`; }\n'; + writeFileSync(input, '# Source movement fixture\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + markdownMeasurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + firstSourceSha, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + env: movingGitEnvironment(root), + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return { output, result }; +} + +function revisionInvocation(root: string) { + const input = join(root, 'document-envelope.json'); + const modulePath = join(root, 'revision.mjs'); + const output = join(root, 'revision-samples.json'); + const moduleSource = `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'d'.repeat(64)}' } }; }\n`; + writeFileSync( + input, + '{"contractVersion":1,"mode":"markdown","document":"# Source movement fixture"}\n', + 'utf8', + ); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + revisionMeasurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + firstSourceSha, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + env: movingGitEnvironment(root), + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return { output, result }; +} + +describe.skipIf(process.platform === 'win32')( + 'direct benchmark producer source stability', + () => { + it.each([ + ['Markdown', markdownInvocation], + ['revision', revisionInvocation], + ] as const)( + 'rejects %s evidence when checked-out HEAD moves during sample acquisition', + (_label, invoke) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-producer-source-move-')); + try { + const { output, result } = invoke(root); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark measurement source commit does not match checked-out HEAD.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); + }, +); From 65e6327cb70b1828cce514fd25f84f1e131538cc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 11:22:13 -0700 Subject: [PATCH 207/209] test(perf): build isolated git shim for source-move RED --- ...erformanceMeasurementProducerSourceStabilityContract.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/performanceMeasurementProducerSourceStabilityContract.test.ts b/src/performanceMeasurementProducerSourceStabilityContract.test.ts index 2c3f402b..12cfdd9e 100644 --- a/src/performanceMeasurementProducerSourceStabilityContract.test.ts +++ b/src/performanceMeasurementProducerSourceStabilityContract.test.ts @@ -3,6 +3,7 @@ import { spawnSync } from 'node:child_process'; import { chmodSync, existsSync, + mkdirSync, mkdtempSync, rmSync, writeFileSync, @@ -42,6 +43,7 @@ const count = existsSync(state) ? Number(readFileSync(state, 'utf8')) : 0; writeFileSync(state, String(count + 1), 'utf8'); process.stdout.write(\`${'${count === 0 ? first : moved}'}\\n\`); `; + mkdirSync(fakeBin, { recursive: true }); writeFileSync(fakeGit, script, { encoding: 'utf8', mode: 0o755 }); chmodSync(fakeGit, 0o755); return { From 8a7e49c8e669a396b9b7587b2361dc9e33940bc8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 14:04:31 -0700 Subject: [PATCH 208/209] fix(perf): recheck source provenance after markdown measurement --- benchmarks/measure-markdown.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs index 14904b70..049b1df2 100644 --- a/benchmarks/measure-markdown.mjs +++ b/benchmarks/measure-markdown.mjs @@ -454,6 +454,7 @@ async function main() { } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); assertNoSymlinkOutputAncestors(args.outputPath); const outputMetadata = inspectOutputPath(args.outputPath); if (outputMetadata !== undefined && !outputMetadata.isFile()) { From a5123c7b0aa153d7e6eccd82ee07bc625af4ca7e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 14:07:01 -0700 Subject: [PATCH 209/209] fix(perf): recheck source provenance after revision measurement --- benchmarks/measure-revision-evidence.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs index 782063cb..1e3e438d 100644 --- a/benchmarks/measure-revision-evidence.mjs +++ b/benchmarks/measure-revision-evidence.mjs @@ -406,6 +406,7 @@ async function main() { } verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); assertNoSymlinkOutputAncestors(args.outputPath); const outputMetadata = inspectOutputPath(args.outputPath); if (outputMetadata !== undefined && !outputMetadata.isFile()) {