diff --git a/benchmarks/analyze-memory-settling.mjs b/benchmarks/analyze-memory-settling.mjs new file mode 100644 index 00000000..f15d3d83 --- /dev/null +++ b/benchmarks/analyze-memory-settling.mjs @@ -0,0 +1,309 @@ +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readSync, +} from 'node:fs'; +import { resolve } from 'node:path'; + +const MAX_INPUT_BYTES = 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000_000; +const READ_ONLY_NONBLOCKING = + constants.O_RDONLY | + (constants.O_NONBLOCK ?? 0) | + (constants.O_NOFOLLOW ?? 0); +const BENCHMARK_ID_PREFIX = 'editor-lifecycle-retained-memory-'; +const BENCHMARK_ID_PATTERN = + /^editor-lifecycle-retained-memory-(?:small|medium|large|stress)$/u; +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const EVIDENCE_KEYS = new Set([ + 'contractVersion', + 'benchmarkId', + 'unit', + 'sourceCommitSha', + 'artifactSha256', + 'documentProfile', + 'runtimeId', + 'referenceHardwareId', + 'warmupSamples', + 'samples', +]); +const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true }); + +function resolveArguments(argv) { + if ( + argv.length !== 6 || + argv[0] !== '--input' || + argv[1].length === 0 || + argv[2] !== '--window-size' || + argv[3].trim().length === 0 || + argv[4] !== '--max-growth-bytes' || + argv[5].trim().length === 0 + ) { + throw new Error( + 'Usage: node benchmarks/analyze-memory-settling.mjs --input --window-size --max-growth-bytes ', + ); + } + + const windowSize = Number(argv[3]); + if (!Number.isSafeInteger(windowSize) || windowSize <= 0) { + throw new Error('Memory settling window size must be a positive safe integer.'); + } + + const maxGrowthBytes = Number(argv[5]); + if (!Number.isFinite(maxGrowthBytes) || maxGrowthBytes < 0) { + throw new Error( + 'Memory settling max growth bytes must be a finite non-negative number.', + ); + } + + return Object.freeze({ + inputPath: resolve(argv[1]), + windowSize, + maxGrowthBytes, + }); +} + +function inspectEvidenceInputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Memory settling evidence input must be a regular file.'); + } +} + +function readBoundedJson(path) { + const pathMetadata = inspectEvidenceInputPath(path); + if (pathMetadata === undefined || pathMetadata.isSymbolicLink()) { + throw new Error( + 'Memory settling evidence input must be a regular non-symlink file.', + ); + } + if (!pathMetadata.isFile()) { + throw new Error('Memory settling evidence input must be a regular file.'); + } + + const descriptor = openSync(path, READ_ONLY_NONBLOCKING); + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error('Memory settling evidence input must be a regular file.'); + } + if (metadata.size > MAX_INPUT_BYTES) { + throw new Error('Memory settling evidence input exceeds the supported size.'); + } + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= MAX_INPUT_BYTES) { + const remainingBudget = MAX_INPUT_BYTES + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > MAX_INPUT_BYTES) { + throw new Error( + 'Memory settling evidence input exceeds the supported size.', + ); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + + let text; + try { + text = UTF8_DECODER.decode(Buffer.concat(chunks, totalBytes)); + } catch { + throw new Error('Memory settling evidence input must be valid UTF-8 JSON.'); + } + + try { + return JSON.parse(text); + } catch { + throw new Error('Memory settling evidence input must be valid JSON.'); + } + } finally { + closeSync(descriptor); + } +} + +function validateEvidence(value) { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Memory settling evidence input must be an object.'); + } + const keys = Object.keys(value); + if ( + keys.length !== EVIDENCE_KEYS.size || + keys.some((key) => !EVIDENCE_KEYS.has(key)) + ) { + throw new Error('Memory settling evidence input has an unsupported shape.'); + } + if (value.contractVersion !== 1) { + throw new Error('Memory settling evidence contractVersion must be 1.'); + } + if ( + typeof value.benchmarkId !== 'string' || + !BENCHMARK_ID_PATTERN.test(value.benchmarkId) + ) { + throw new Error('Memory settling evidence benchmarkId is invalid.'); + } + if (value.unit !== 'bytes') { + throw new Error('Memory settling evidence unit must be bytes.'); + } + if ( + typeof value.sourceCommitSha !== 'string' || + !SHA1_PATTERN.test(value.sourceCommitSha) + ) { + throw new Error('Memory settling evidence sourceCommitSha is invalid.'); + } + if ( + typeof value.artifactSha256 !== 'string' || + !SHA256_PATTERN.test(value.artifactSha256) + ) { + throw new Error('Memory settling evidence artifactSha256 is invalid.'); + } + if ( + typeof value.documentProfile !== 'string' || + !DOCUMENT_PROFILES.has(value.documentProfile) + ) { + throw new Error('Memory settling evidence documentProfile is invalid.'); + } + if ( + value.benchmarkId.slice(BENCHMARK_ID_PREFIX.length) !== value.documentProfile + ) { + throw new Error( + 'Memory settling evidence benchmark profile must match documentProfile.', + ); + } + if ( + typeof value.runtimeId !== 'string' || + !RUNTIME_ID_PATTERN.test(value.runtimeId) + ) { + throw new Error('Memory settling evidence runtimeId is invalid.'); + } + if ( + typeof value.referenceHardwareId !== 'string' || + !REFERENCE_HARDWARE_ID_PATTERN.test(value.referenceHardwareId) + ) { + throw new Error('Memory settling evidence referenceHardwareId is invalid.'); + } + if ( + !Number.isSafeInteger(value.warmupSamples) || + value.warmupSamples < 0 || + value.warmupSamples > MAX_SAMPLES + ) { + throw new Error('Memory settling evidence warmupSamples is invalid.'); + } + if ( + !Array.isArray(value.samples) || + value.samples.length === 0 || + value.samples.length > MAX_SAMPLES || + value.samples.some( + (sample) => + !Number.isSafeInteger(sample) || sample < 0, + ) + ) { + throw new Error( + 'Memory settling evidence samples must be bounded non-negative safe integers.', + ); + } + if (value.warmupSamples >= value.samples.length) { + throw new Error('Memory settling evidence warmupSamples is invalid.'); + } + + return Object.freeze({ + benchmarkId: value.benchmarkId, + unit: value.unit, + sourceCommitSha: value.sourceCommitSha, + artifactSha256: value.artifactSha256, + documentProfile: value.documentProfile, + runtimeId: value.runtimeId, + referenceHardwareId: value.referenceHardwareId, + warmupSamples: value.warmupSamples, + samples: Object.freeze([...value.samples]), + }); +} + +function median(values) { + const sorted = [...values].sort((left, right) => left - right); + const midpoint = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 1) return sorted[midpoint]; + + const left = sorted[midpoint - 1]; + const right = sorted[midpoint]; + const distance = right - left; + const wholeMidpoint = left + Math.floor(distance / 2); + if (distance % 2 === 1 && wholeMidpoint >= 2 ** 52) { + throw new Error( + 'Memory settling window median must be exactly representable.', + ); + } + return wholeMidpoint + (distance % 2) / 2; +} + +function analyze(evidence, windowSize, maxGrowthBytes) { + const settledSamples = evidence.samples.slice(evidence.warmupSamples); + if (settledSamples.length < windowSize * 2) { + throw new Error( + 'Memory settling evidence requires warmup plus two disjoint comparison windows.', + ); + } + + const firstWindowMedianBytes = median(settledSamples.slice(0, windowSize)); + const lastWindowMedianBytes = median(settledSamples.slice(-windowSize)); + const retainedGrowthBytes = lastWindowMedianBytes - firstWindowMedianBytes; + + return Object.freeze({ + contractVersion: 1, + benchmarkId: evidence.benchmarkId, + unit: evidence.unit, + sourceCommitSha: evidence.sourceCommitSha, + artifactSha256: evidence.artifactSha256, + documentProfile: evidence.documentProfile, + runtimeId: evidence.runtimeId, + referenceHardwareId: evidence.referenceHardwareId, + sampleCount: evidence.samples.length, + warmupSamples: evidence.warmupSamples, + windowSize, + firstWindowMedianBytes, + lastWindowMedianBytes, + retainedGrowthBytes, + maxGrowthBytes, + passed: retainedGrowthBytes <= maxGrowthBytes, + }); +} + +function main() { + const { inputPath, windowSize, maxGrowthBytes } = resolveArguments( + process.argv.slice(2), + ); + const evidence = validateEvidence(readBoundedJson(inputPath)); + const result = analyze(evidence, windowSize, maxGrowthBytes); + process.stdout.write(`${JSON.stringify(result)}\n`); + if (!result.passed) process.exitCode = 1; +} + +try { + main(); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Memory settling analysis failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} diff --git a/benchmarks/compare-summaries.mjs b/benchmarks/compare-summaries.mjs new file mode 100644 index 00000000..a5c252da --- /dev/null +++ b/benchmarks/compare-summaries.mjs @@ -0,0 +1,342 @@ +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readSync, +} from 'node:fs'; +import { resolve } from 'node:path'; + +const MAX_INPUT_BYTES = 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000_000; +const READ_ONLY_NONBLOCKING = + constants.O_RDONLY | + (constants.O_NONBLOCK ?? 0) | + (constants.O_NOFOLLOW ?? 0); +const BENCHMARK_ID_PATTERN = + /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; +const UNITS = new Set(['ms', 'bytes']); +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const METRICS = new Set(['p50', 'p75', 'p95', 'maximum']); +const SUMMARY_KEYS = new Set([ + 'contractVersion', + 'benchmarkId', + 'unit', + 'sourceCommitSha', + 'artifactSha256', + 'documentProfile', + 'runtimeId', + 'referenceHardwareId', + 'sampleCount', + 'percentileMethod', + 'minimum', + 'p50', + 'p75', + 'p95', + 'maximum', +]); +const COMPARABLE_FIELDS = [ + 'benchmarkId', + 'unit', + 'documentProfile', + 'runtimeId', + 'referenceHardwareId', + 'sampleCount', + 'percentileMethod', +]; +const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true }); + +function resolveArguments(argv) { + if ( + argv.length !== 8 || + argv[0] !== '--baseline' || + argv[1].length === 0 || + argv[2] !== '--current' || + argv[3].length === 0 || + argv[4] !== '--metric' || + !METRICS.has(argv[5]) || + argv[6] !== '--max-regression-percent' || + argv[7].trim().length === 0 + ) { + throw new Error( + 'Usage: node benchmarks/compare-summaries.mjs --baseline --current --metric --max-regression-percent ', + ); + } + + const maxRegressionPercent = Number(argv[7]); + if (!Number.isFinite(maxRegressionPercent) || maxRegressionPercent < 0) { + throw new Error( + 'Benchmark max regression percent must be a finite non-negative number.', + ); + } + + return Object.freeze({ + baselinePath: resolve(argv[1]), + currentPath: resolve(argv[3]), + metric: argv[5], + maxRegressionPercent, + }); +} + +function inspectSummaryInputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark summary input must be a regular file.'); + } +} + +function readBoundedJson(path) { + const pathMetadata = inspectSummaryInputPath(path); + if (pathMetadata === undefined || pathMetadata.isSymbolicLink()) { + throw new Error( + 'Benchmark summary input must be a regular non-symlink file.', + ); + } + if (!pathMetadata.isFile()) { + throw new Error('Benchmark summary input must be a regular file.'); + } + + const descriptor = openSync(path, READ_ONLY_NONBLOCKING); + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error('Benchmark summary input must be a regular file.'); + } + if (metadata.size > MAX_INPUT_BYTES) { + throw new Error('Benchmark summary input exceeds the supported size.'); + } + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= MAX_INPUT_BYTES) { + const remainingBudget = MAX_INPUT_BYTES + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > MAX_INPUT_BYTES) { + throw new Error('Benchmark summary input exceeds the supported size.'); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + + let text; + try { + text = UTF8_DECODER.decode(Buffer.concat(chunks, totalBytes)); + } catch { + throw new Error('Benchmark summary input must be valid UTF-8 JSON.'); + } + + try { + return JSON.parse(text); + } catch { + throw new Error('Benchmark summary input must be valid JSON.'); + } + } finally { + closeSync(descriptor); + } +} + +function validateSummary(value) { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Benchmark summary input must be an object.'); + } + const keys = Object.keys(value); + if ( + keys.length !== SUMMARY_KEYS.size || + keys.some((key) => !SUMMARY_KEYS.has(key)) + ) { + throw new Error('Benchmark summary input has an unsupported shape.'); + } + if (value.contractVersion !== 1) { + throw new Error('Benchmark summary contractVersion must be 1.'); + } + if ( + typeof value.benchmarkId !== 'string' || + !BENCHMARK_ID_PATTERN.test(value.benchmarkId) + ) { + throw new Error('Benchmark summary benchmarkId is invalid.'); + } + if (typeof value.unit !== 'string' || !UNITS.has(value.unit)) { + throw new Error('Benchmark summary unit is invalid.'); + } + if ( + typeof value.sourceCommitSha !== 'string' || + !SHA1_PATTERN.test(value.sourceCommitSha) + ) { + throw new Error('Benchmark summary sourceCommitSha is invalid.'); + } + if ( + typeof value.artifactSha256 !== 'string' || + !SHA256_PATTERN.test(value.artifactSha256) + ) { + throw new Error('Benchmark summary artifactSha256 is invalid.'); + } + if ( + typeof value.documentProfile !== 'string' || + !DOCUMENT_PROFILES.has(value.documentProfile) + ) { + throw new Error('Benchmark summary documentProfile is invalid.'); + } + if (!value.benchmarkId.endsWith(`-${value.documentProfile}`)) { + throw new Error('Benchmark summary profile must match documentProfile.'); + } + if ( + typeof value.runtimeId !== 'string' || + !RUNTIME_ID_PATTERN.test(value.runtimeId) + ) { + throw new Error('Benchmark summary runtimeId is invalid.'); + } + if ( + typeof value.referenceHardwareId !== 'string' || + !REFERENCE_HARDWARE_ID_PATTERN.test(value.referenceHardwareId) + ) { + throw new Error('Benchmark summary referenceHardwareId is invalid.'); + } + if ( + !Number.isSafeInteger(value.sampleCount) || + value.sampleCount <= 0 || + value.sampleCount > MAX_SAMPLES + ) { + throw new Error('Benchmark summary sampleCount is invalid.'); + } + if (value.percentileMethod !== 'nearest-rank') { + throw new Error('Benchmark summary percentileMethod is invalid.'); + } + + const measurements = [ + value.minimum, + value.p50, + value.p75, + value.p95, + value.maximum, + ]; + if ( + measurements.some( + (measurement) => + typeof measurement !== 'number' || + !Number.isFinite(measurement) || + measurement < 0, + ) + ) { + throw new Error( + 'Benchmark summary measurements must be finite non-negative numbers.', + ); + } + for (let index = 1; index < measurements.length; index += 1) { + if (measurements[index] < measurements[index - 1]) { + throw new Error('Benchmark summary percentile ordering is invalid.'); + } + } + + return Object.freeze({ + benchmarkId: value.benchmarkId, + unit: value.unit, + sourceCommitSha: value.sourceCommitSha, + artifactSha256: value.artifactSha256, + documentProfile: value.documentProfile, + runtimeId: value.runtimeId, + referenceHardwareId: value.referenceHardwareId, + sampleCount: value.sampleCount, + percentileMethod: value.percentileMethod, + minimum: value.minimum, + p50: value.p50, + p75: value.p75, + p95: value.p95, + maximum: value.maximum, + }); +} + +function assertComparable(baseline, current) { + for (const field of COMPARABLE_FIELDS) { + if (baseline[field] !== current[field]) { + throw new Error(`Benchmark summaries are not comparable: ${field} differs.`); + } + } + if (baseline.artifactSha256 === current.artifactSha256) { + throw new Error( + 'Benchmark summaries must identify distinct measured artifacts.', + ); + } +} + +function normalizePercent(value) { + const rounded = Number(value.toFixed(12)); + return Object.is(rounded, -0) ? 0 : rounded; +} + +function compare(baseline, current, metric, maxRegressionPercent) { + assertComparable(baseline, current); + const baselineValue = baseline[metric]; + const currentValue = current[metric]; + if (baselineValue === 0 && currentValue !== 0) { + throw new Error( + 'Benchmark regression percent is undefined for a zero non-matching baseline.', + ); + } + const regressionPercent = + baselineValue === 0 + ? 0 + : normalizePercent(((currentValue - baselineValue) / baselineValue) * 100); + if (!Number.isFinite(regressionPercent)) { + throw new Error( + 'Benchmark regression percent is not finite for the supplied measurements.', + ); + } + return Object.freeze({ + contractVersion: 1, + benchmarkId: baseline.benchmarkId, + unit: baseline.unit, + documentProfile: baseline.documentProfile, + runtimeId: baseline.runtimeId, + referenceHardwareId: baseline.referenceHardwareId, + sampleCount: baseline.sampleCount, + percentileMethod: baseline.percentileMethod, + metric, + baselineSourceCommitSha: baseline.sourceCommitSha, + baselineArtifactSha256: baseline.artifactSha256, + currentSourceCommitSha: current.sourceCommitSha, + currentArtifactSha256: current.artifactSha256, + baselineValue, + currentValue, + maxRegressionPercent, + regressionPercent, + passed: regressionPercent <= maxRegressionPercent, + }); +} + +function main() { + const { baselinePath, currentPath, metric, maxRegressionPercent } = + resolveArguments(process.argv.slice(2)); + const baseline = validateSummary(readBoundedJson(baselinePath)); + const current = validateSummary(readBoundedJson(currentPath)); + const result = compare(baseline, current, metric, maxRegressionPercent); + process.stdout.write(`${JSON.stringify(result)}\n`); + if (!result.passed) process.exitCode = 1; +} + +try { + main(); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Benchmark comparison failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} diff --git a/benchmarks/corpus.lock.json b/benchmarks/corpus.lock.json new file mode 100644 index 00000000..fa2fe87c --- /dev/null +++ b/benchmarks/corpus.lock.json @@ -0,0 +1,34 @@ +{ + "contractVersion": 1, + "synthetic": true, + "scripts": [ + "English", + "Korean", + "Japanese", + "Chinese", + "Vietnamese", + "mixed" + ], + "profiles": { + "small": { + "sections": 1, + "bytes": 2152, + "sha256": "420d18f2bb9e42d7e7e2cb5f74e67c90dfe15c3748b5d22875b4a6dc38ecbdea" + }, + "medium": { + "sections": 8, + "bytes": 15712, + "sha256": "921092809cc19be790c7a29a5457a7113e75aa7c76642d4ec09784b6096e045c" + }, + "large": { + "sections": 32, + "bytes": 62199, + "sha256": "6ea32c0c8d2b58bf958dd28424a0b6139954fcefe8850943966be9e67a13b392" + }, + "stress": { + "sections": 128, + "bytes": 248152, + "sha256": "5139848dc240863acb95ffdcf549fe7f151451a1002befc39c2d9c3395826928" + } + } +} diff --git a/benchmarks/generate-corpus.mjs b/benchmarks/generate-corpus.mjs new file mode 100644 index 00000000..af9d85e9 --- /dev/null +++ b/benchmarks/generate-corpus.mjs @@ -0,0 +1,155 @@ +import { createHash } from 'node:crypto'; +import { + lstatSync, + mkdirSync, + renameSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { resolve } from 'node:path'; + +const RASTER_FIXTURES = Object.freeze([ + Object.freeze({ + dimensions: '1x1', + base64: + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNQcEj4DwADBAHAX3ZiygAAAABJRU5ErkJggg==', + }), + Object.freeze({ + dimensions: '16x16', + base64: + 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAGUlEQVR42mNQcEj4TwlmGDVg1IBRA4aLAQDSpr8QG8NsyQAAAABJRU5ErkJggg==', + }), + Object.freeze({ + dimensions: '64x64', + base64: + 'iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAAZklEQVR42u3QQREAAAQAMFFEEUX/EuRw9liBRVbPZyFAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgQIECBAgAABAgTct/Bk8ZbOy3oMAAAAAElFTkSuQmCC', + }), +]); +const SCRIPT_PARAGRAPHS = [ + 'English: Deterministic authoring keeps document state explicit and reviewable.', + '한국어: 결정적 작성 흐름은 문서 상태와 변경 근거를 명확하게 유지합니다.', + '日本語: 決定的な編集フローは文書状態と変更根拠を明示的に保ちます。', + '中文: 确定性的编辑流程会明确保留文档状态与变更依据。', + 'Tiếng Việt: Luồng biên soạn xác định giữ trạng thái tài liệu và bằng chứng thay đổi rõ ràng.', + 'Mixed-script: Inkspan review 검증은 日本語と中文 그리고 Tiếng Việt를 한 문단에서 deterministic하게 다룹니다.', +]; +const PROFILE_SECTIONS = Object.freeze({ + small: 1, + medium: 8, + large: 32, + stress: 128, +}); +const SCRIPT_LABELS = Object.freeze([ + 'English', + 'Korean', + 'Japanese', + 'Chinese', + 'Vietnamese', + 'mixed', +]); + +let outputWriteCounter = 0; + +function writeRegularOutput(path, content) { + const existing = lstatSync(path, { throwIfNoEntry: false }); + if (existing !== undefined && !existing.isFile()) { + throw new Error('Benchmark corpus output must be a regular file.'); + } + + const temporaryPath = `${path}.tmp-${process.pid}-${outputWriteCounter}`; + outputWriteCounter += 1; + try { + writeFileSync(temporaryPath, content, { flag: 'wx' }); + renameSync(temporaryPath, path); + } finally { + rmSync(temporaryPath, { force: true }); + } +} + +function buildSection(index) { + const id = String(index).padStart(4, '0'); + const tableRows = Array.from({ length: 4 }, (_, rowIndex) => { + const row = String(rowIndex + 1).padStart(2, '0'); + return `| ${id}-r${row}c01 | ${id}-r${row}c02 | ${id}-r${row}c03 | ${id}-r${row}c04 | ${id}-r${row}c05 | ${id}-r${row}c06 |`; + }); + const rasterRows = RASTER_FIXTURES.map( + ({ dimensions, base64 }) => + `![synthetic raster ${dimensions} ${id}](data:image/png;base64,${base64})`, + ); + return [ + `# Synthetic section ${id}`, + '', + ...SCRIPT_PARAGRAPHS, + '', + `## Nested list ${id}`, + `- item ${id}-a`, + ` - item ${id}-a-1`, + ` - item ${id}-a-2`, + `- item ${id}-b`, + '', + `> Synthetic blockquote ${id}: benchmark text only; no production content.`, + '', + '```text', + `fixture=${id}; authority=none; network=none`, + '```', + '', + `[synthetic safe link ${id}](https://example.invalid/inkspan/${id})`, + '', + '| c01 | c02 | c03 | c04 | c05 | c06 |', + '| --- | --- | --- | --- | --- | --- |', + ...tableRows, + '', + ...rasterRows, + '', + '---', + '', + ].join('\n'); +} + +function buildProfile(profile, sectionCount) { + return [ + `# Inkspan deterministic benchmark fixture: ${profile}`, + '', + 'Synthetic fixture only. No customer, tenant, prompt, credential, or model data.', + 'Scripts: English, Korean, Japanese, Chinese, Vietnamese, and mixed-script structure.', + '', + ...Array.from({ length: sectionCount }, (_, index) => buildSection(index + 1)), + ].join('\n'); +} + +function sha256(bytes) { + return createHash('sha256').update(bytes).digest('hex'); +} + +function resolveOutputDirectory(argv) { + if (argv.length !== 2 || argv[0] !== '--output' || argv[1].length === 0) { + throw new Error('Usage: node benchmarks/generate-corpus.mjs --output '); + } + return resolve(argv[1]); +} + +const outputDirectory = resolveOutputDirectory(process.argv.slice(2)); +mkdirSync(outputDirectory, { recursive: true }); + +const profileManifest = {}; +for (const [profile, sections] of Object.entries(PROFILE_SECTIONS)) { + const body = buildProfile(profile, sections); + const bytes = Buffer.from(body, 'utf8'); + writeRegularOutput(resolve(outputDirectory, `${profile}.md`), bytes); + profileManifest[profile] = Object.freeze({ + sections, + bytes: bytes.byteLength, + sha256: sha256(bytes), + }); +} + +const manifest = Object.freeze({ + contractVersion: 1, + synthetic: true, + scripts: SCRIPT_LABELS, + profiles: profileManifest, +}); +writeRegularOutput( + resolve(outputDirectory, 'manifest.json'), + `${JSON.stringify(manifest, null, 2)}\n`, +); diff --git a/benchmarks/generate-office-fixtures.mjs b/benchmarks/generate-office-fixtures.mjs new file mode 100644 index 00000000..e3264cc4 --- /dev/null +++ b/benchmarks/generate-office-fixtures.mjs @@ -0,0 +1,302 @@ +import { createHash } from 'node:crypto'; +import { + closeSync, + constants, + fstatSync, + ftruncateSync, + lstatSync, + mkdirSync, + openSync, + writeFileSync, +} from 'node:fs'; +import { dirname, resolve } from 'node:path'; + +const DOCX_PROFILE_PAGES = Object.freeze({ + small: 2, + page120: 120, +}); + +const PPTX_PROFILE_SLIDES = Object.freeze({ + small: 2, + slide120: 120, +}); + +const EXCEL_MAX_COLUMNS = 16_384; +const MULTILINGUAL_PARAGRAPH = + 'English: deterministic Office rendering fixture. 한국어: 합성 성능 문서입니다. 日本語: 合成性能文書です。 中文: 这是合成性能文档。 Tiếng Việt: Đây là tài liệu hiệu năng tổng hợp.'; +const WRITE_NOFOLLOW = + constants.O_WRONLY | + constants.O_CREAT | + (constants.O_NONBLOCK ?? 0) | + (constants.O_NOFOLLOW ?? 0); +const OUTPUT_DIRECTORY_ERROR = + 'Office fixture output directory must be a non-symlink directory.'; + +function buildDocxPage(pageNumber) { + const page = String(pageNumber).padStart(3, '0'); + return [ + Object.freeze({ + type: 'heading', + level: 1, + text: `Synthetic page ${page}`, + }), + Object.freeze({ + type: 'paragraph', + text: `${MULTILINGUAL_PARAGRAPH} Page ${page}.`, + alignment: 'justify', + }), + Object.freeze({ + type: 'rich_paragraph', + runs: Object.freeze([ + Object.freeze({ text: `Page ${page} summary: `, bold: true }), + Object.freeze({ text: 'deterministic ', italic: true }), + Object.freeze({ text: 'Office rendering fixture.', underline: true }), + ]), + }), + Object.freeze({ + type: 'bullet_list', + ordered: false, + items: Object.freeze([ + `page ${page} item A`, + `page ${page} item B`, + `page ${page} item C`, + ]), + }), + Object.freeze({ + type: 'table', + headers: Object.freeze(['Page', 'Metric', 'Value']), + rows: Object.freeze([ + Object.freeze([page, 'latency-sample', pageNumber]), + Object.freeze([page, 'memory-sample', pageNumber * 2]), + Object.freeze([page, 'revision-sample', pageNumber * 3]), + Object.freeze([page, 'render-sample', pageNumber * 4]), + ]), + }), + ]; +} + +function buildDocxRequest(profile, pages) { + const blocks = []; + for (let page = 1; page <= pages; page += 1) { + blocks.push(...buildDocxPage(page)); + if (page < pages) { + blocks.push(Object.freeze({ type: 'page_break' })); + } + } + return Object.freeze({ + format: 'docx', + title: `Inkspan synthetic DOCX benchmark: ${profile}`, + author: 'Inkspan synthetic benchmark', + subject: 'Deterministic synthetic performance fixture', + blocks: Object.freeze(blocks), + }); +} + +function buildXlsxRequest(profile) { + if (profile === 'wide16384') { + const row = Array.from( + { length: EXCEL_MAX_COLUMNS }, + (_, index) => `C${String(index + 1).padStart(5, '0')}`, + ); + return Object.freeze({ + format: 'xlsx', + title: 'Inkspan synthetic XLSX benchmark: wide16384', + author: 'Inkspan synthetic benchmark', + sheets: Object.freeze([ + Object.freeze({ + name: 'Wide16384', + rows: Object.freeze([Object.freeze(row)]), + freeze_panes: 'XFD1048576', + }), + ]), + }); + } + + return Object.freeze({ + format: 'xlsx', + title: 'Inkspan synthetic XLSX benchmark: small', + author: 'Inkspan synthetic benchmark', + sheets: Object.freeze([ + Object.freeze({ + name: 'Synthetic', + header_row: true, + auto_filter: true, + freeze_panes: 'B2', + rows: Object.freeze([ + Object.freeze(['Language', 'Text', 'Latency', 'Memory']), + Object.freeze(['한국어', '합성 성능 문서', 1, 2]), + Object.freeze(['日本語', '合成性能文書', 3, 4]), + Object.freeze(['中文 / Tiếng Việt', '合成文档 / tài liệu tổng hợp', 5, 6]), + ]), + }), + ]), + }); +} + +function buildPptxSlide(slideNumber) { + const slide = String(slideNumber).padStart(3, '0'); + return Object.freeze({ + title: `한국어 합성 슬라이드 ${slide}`, + bullets: Object.freeze([ + `English deterministic slide ${slide}`, + Object.freeze({ text: `日本語 合成スライド ${slide}`, level: 0 }), + Object.freeze({ text: `中文 合成幻灯片 ${slide}`, level: 1 }), + Object.freeze({ text: `Tiếng Việt trang chiếu ${slide}`, level: 1 }), + ]), + }); +} + +function buildPptxRequest(profile, slides) { + return Object.freeze({ + format: 'pptx', + title: `Inkspan synthetic PPTX benchmark: ${profile}`, + author: 'Inkspan synthetic benchmark', + slides: Object.freeze( + Array.from({ length: slides }, (_, index) => buildPptxSlide(index + 1)), + ), + }); +} + +function sha256(bytes) { + return createHash('sha256').update(bytes).digest('hex'); +} + +function resolveOutputDirectory(argv) { + if (argv.length !== 2 || argv[0] !== '--output' || argv[1].length === 0) { + throw new Error( + 'Usage: node benchmarks/generate-office-fixtures.mjs --output ', + ); + } + return resolve(argv[1]); +} + +function inspectOutputDirectoryComponent(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Office fixture output directory could not be inspected.'); + } +} + +function assertNoSymlinkOutputDirectoryAncestors(path) { + let current = path; + while (true) { + const metadata = inspectOutputDirectoryComponent(current); + if (metadata?.isSymbolicLink()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + +function writeOutputFile(outputPath, bytes) { + let pathMetadata; + try { + pathMetadata = lstatSync(outputPath, { throwIfNoEntry: false }); + } catch { + throw new Error('Office fixture output path could not be inspected.'); + } + if (pathMetadata?.isSymbolicLink()) { + throw new Error('Office fixture output must be a regular non-symlink file.'); + } + if (pathMetadata !== undefined && !pathMetadata.isFile()) { + throw new Error('Office fixture output must be a regular file.'); + } + + let descriptor; + try { + descriptor = openSync(outputPath, WRITE_NOFOLLOW, 0o600); + const descriptorMetadata = fstatSync(descriptor); + if (!descriptorMetadata.isFile()) { + throw new Error('Office fixture output must be a regular file.'); + } + if (descriptorMetadata.nlink !== 1) { + throw new Error('Office fixture output must not be multiply linked.'); + } + ftruncateSync(descriptor, 0); + writeFileSync(descriptor, bytes); + } catch (error) { + if ( + error instanceof Error && + (error.message === 'Office fixture output must be a regular file.' || + error.message === 'Office fixture output must be a regular non-symlink file.' || + error.message === 'Office fixture output must not be multiply linked.') + ) { + throw error; + } + throw new Error('Office fixture output could not be written safely.'); + } finally { + if (descriptor !== undefined) closeSync(descriptor); + } +} + +function writeFixture(outputDirectory, fileName, request, units) { + const body = `${JSON.stringify(request, null, 2)}\n`; + const bytes = Buffer.from(body, 'utf8'); + writeOutputFile(resolve(outputDirectory, fileName), bytes); + return Object.freeze({ + units, + bytes: bytes.byteLength, + sha256: sha256(bytes), + }); +} + +const outputDirectory = resolveOutputDirectory(process.argv.slice(2)); +assertNoSymlinkOutputDirectoryAncestors(outputDirectory); +try { + mkdirSync(outputDirectory, { recursive: true }); +} catch { + throw new Error('Office fixture output directory could not be prepared.'); +} +assertNoSymlinkOutputDirectoryAncestors(outputDirectory); + +const docx = {}; +for (const [profile, pages] of Object.entries(DOCX_PROFILE_PAGES)) { + docx[profile] = writeFixture( + outputDirectory, + `docx-${profile}.json`, + buildDocxRequest(profile, pages), + pages, + ); +} + +const xlsx = Object.freeze({ + small: writeFixture( + outputDirectory, + 'xlsx-small.json', + buildXlsxRequest('small'), + 4, + ), + wide16384: writeFixture( + outputDirectory, + 'xlsx-wide16384.json', + buildXlsxRequest('wide16384'), + EXCEL_MAX_COLUMNS, + ), +}); + +const pptx = {}; +for (const [profile, slides] of Object.entries(PPTX_PROFILE_SLIDES)) { + pptx[profile] = writeFixture( + outputDirectory, + `pptx-${profile}.json`, + buildPptxRequest(profile, slides), + slides, + ); +} + +const manifest = Object.freeze({ + contractVersion: 1, + synthetic: true, + formats: Object.freeze({ + docx: Object.freeze(docx), + xlsx, + pptx: Object.freeze(pptx), + }), +}); +writeOutputFile( + resolve(outputDirectory, 'manifest.json'), + Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`, 'utf8'), +); \ No newline at end of file diff --git a/benchmarks/measure-markdown.mjs b/benchmarks/measure-markdown.mjs new file mode 100644 index 00000000..049b1df2 --- /dev/null +++ b/benchmarks/measure-markdown.mjs @@ -0,0 +1,498 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { performance } from 'node:perf_hooks'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdirSync, + openSync, + readSync, + realpathSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(benchmarkDirectory, '..'); +const MAX_INPUT_BYTES = 16 * 1024 * 1024; +const MAX_MODULE_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000; +const READ_ONLY_NOFOLLOW = + constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const SERIALIZATION_OPERATIONS = new Set([ + 'markdown-to-html', + 'html-to-markdown', +]); +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; +const OUTPUT_DIRECTORY_ERROR = + 'Markdown benchmark output directory must be a non-symlink directory.'; +const OUTPUT_EXISTS_ERROR = + 'Markdown benchmark output must not already exist.'; +const LEGACY_FLAGS = Object.freeze([ + '--input', + '--module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const OPERATION_FLAGS = Object.freeze([ + '--input', + '--module', + '--operation', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); + +function matchesArguments(argv, expectedFlags) { + return ( + argv.length === expectedFlags.length * 2 && + expectedFlags.every((flag, index) => argv[index * 2] === flag) && + expectedFlags.every((_, index) => argv[index * 2 + 1]?.length > 0) + ); +} + +function valuesForArguments(argv, expectedFlags) { + return Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); +} + +function resolveArguments(argv) { + let values; + let operation = 'markdown-to-html'; + if (matchesArguments(argv, OPERATION_FLAGS)) { + values = valuesForArguments(argv, OPERATION_FLAGS); + operation = values['--operation']; + if (!SERIALIZATION_OPERATIONS.has(operation)) { + throw new Error('Markdown benchmark serialization operation is invalid.'); + } + } else if (matchesArguments(argv, LEGACY_FLAGS)) { + values = valuesForArguments(argv, LEGACY_FLAGS); + } else { + throw new Error( + 'Usage: node benchmarks/measure-markdown.mjs --input --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', + ); + } + + const profile = values['--profile']; + if (!DOCUMENT_PROFILES.has(profile)) { + throw new Error('Markdown benchmark profile is invalid.'); + } + const sampleCount = Number(values['--samples']); + if ( + !Number.isSafeInteger(sampleCount) || + sampleCount < 1 || + sampleCount > MAX_SAMPLES + ) { + throw new Error('Markdown benchmark sample count must be an integer from 1 to 1000.'); + } + const sourceCommitSha = values['--source-commit-sha']; + if (!SHA1_PATTERN.test(sourceCommitSha)) { + throw new Error( + 'Markdown benchmark source commit must be a lowercase 40-character SHA.', + ); + } + const artifactSha256 = values['--artifact-sha256']; + if (!SHA256_PATTERN.test(artifactSha256)) { + throw new Error( + 'Markdown benchmark artifact digest must be a lowercase 64-character SHA-256.', + ); + } + const runtimeId = values['--runtime-id']; + if (!RUNTIME_ID_PATTERN.test(runtimeId)) { + throw new Error('Markdown benchmark runtime ID is invalid.'); + } + const referenceHardwareId = values['--reference-hardware-id']; + if (!REFERENCE_HARDWARE_ID_PATTERN.test(referenceHardwareId)) { + throw new Error('Markdown benchmark reference hardware ID is invalid.'); + } + + return Object.freeze({ + inputPath: resolve(values['--input']), + modulePath: values['--module'], + operation, + profile, + sampleCount, + sourceCommitSha, + artifactSha256, + runtimeId, + referenceHardwareId, + outputPath: resolve(values['--output']), + }); +} + +function assertMeasurementProvenance(sourceCommitSha, runtimeId) { + const result = spawnSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + const checkoutSha = result.stdout?.trim(); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !SHA1_PATTERN.test(checkoutSha ?? '') + ) { + throw new Error( + 'Benchmark measurement source commit could not be verified against the current checkout.', + ); + } + if (sourceCommitSha !== checkoutSha) { + throw new Error( + 'Benchmark measurement source commit does not match checked-out HEAD.', + ); + } + if (runtimeId !== `node-${process.versions.node}`) { + throw new Error( + 'Benchmark measurement runtime ID must match the active Node runtime.', + ); + } +} + +function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversizedMessage) { + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error(invalidFileMessage); + } + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error(invalidFileMessage); + } + + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error(invalidFileMessage); + } + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error(invalidFileMessage); + } + if (metadata.size > maximumBytes) { + throw new Error(oversizedMessage); + } + const chunks = []; + let totalBytes = 0; + while (totalBytes <= maximumBytes) { + const remainingBudget = maximumBytes + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > maximumBytes) { + throw new Error(oversizedMessage); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + return Buffer.concat(chunks, totalBytes); + } finally { + closeSync(descriptor); + } +} + +function readBoundedMarkdown(path) { + const bytes = readBoundedRegularFile( + path, + MAX_INPUT_BYTES, + 'Markdown benchmark input must be a regular non-symlink file.', + 'Markdown benchmark input exceeds the supported size.', + ); + try { + return new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch { + throw new Error('Markdown benchmark input must be valid UTF-8.'); + } +} + +function resolveLocalModule(pathOrUrl) { + if ( + pathOrUrl.startsWith('http:') || + pathOrUrl.startsWith('https:') || + pathOrUrl.startsWith('data:') || + pathOrUrl.startsWith('node:') + ) { + throw new Error('Measured Markdown module must be a local regular file.'); + } + let moduleUrl; + try { + moduleUrl = pathOrUrl.startsWith('file:') + ? new URL(pathOrUrl) + : pathToFileURL(resolve(pathOrUrl)); + } catch { + throw new Error('Measured Markdown module must be a local regular file.'); + } + if (moduleUrl.protocol !== 'file:') { + throw new Error('Measured Markdown module must be a local regular file.'); + } + let resolvedPath; + try { + resolvedPath = resolve(fileURLToPath(moduleUrl)); + } catch { + throw new Error('Measured Markdown module must be a local regular file.'); + } + let metadata; + try { + metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); + } catch { + throw new Error('Measured Markdown module must be a local regular file.'); + } + if ( + metadata === undefined || + metadata.isSymbolicLink() || + !metadata.isFile() + ) { + throw new Error('Measured Markdown module must be a local regular file.'); + } + try { + return realpathSync(resolvedPath); + } catch { + throw new Error('Measured Markdown module must be a local regular file.'); + } +} + +function measuredModuleSha256(modulePath) { + const bytes = readBoundedRegularFile( + modulePath, + MAX_MODULE_BYTES, + 'Measured Markdown module must be a local regular file.', + 'Measured Markdown module exceeds the supported size.', + ); + return createHash('sha256').update(bytes).digest('hex'); +} + +function verifyMeasuredModuleDigest(modulePath, expectedSha256) { + if (measuredModuleSha256(modulePath) !== expectedSha256) { + throw new Error( + 'Markdown benchmark artifact digest does not match the measured module.', + ); + } +} + +function inspectOutputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Markdown benchmark output path could not be inspected.'); + } +} + +function inspectOutputDirectoryComponent(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Markdown benchmark output directory could not be inspected.'); + } +} + +function assertNoSymlinkOutputAncestors(path) { + let current = dirname(path); + while (true) { + const metadata = inspectOutputDirectoryComponent(current); + if (metadata?.isSymbolicLink()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + +function refersToSameFile(leftPath, rightPath) { + const rightMetadata = inspectOutputPath(rightPath); + if (rightMetadata === undefined) return false; + if (!rightMetadata.isFile()) { + throw new Error('Markdown benchmark output must be a regular file.'); + } + const left = statSync(leftPath); + const right = statSync(rightPath); + return left.dev === right.dev && left.ino === right.ino; +} + +async function loadMeasuredModule(modulePath) { + try { + return await import(pathToFileURL(modulePath).href); + } catch { + throw new Error('Measured Markdown module could not be loaded.'); + } +} + +function serializationContract(operation) { + if (operation === 'html-to-markdown') { + return Object.freeze({ + exportName: 'htmlToMarkdown', + benchmarkPrefix: 'html-serialization', + executionFailure: 'Measured htmlToMarkdown() execution failed.', + returnFailure: 'Measured htmlToMarkdown() must return a string.', + }); + } + return Object.freeze({ + exportName: 'markdownToHtml', + benchmarkPrefix: 'markdown-serialization', + executionFailure: 'Measured markdownToHtml() execution failed.', + returnFailure: 'Measured markdownToHtml() must return a string.', + }); +} + +function runMeasuredSerialization(serializer, source, failureMessage) { + try { + return serializer(source); + } catch { + throw new Error(failureMessage); + } +} + +function writeMeasurementOutput(path, content) { + assertNoSymlinkOutputAncestors(path); + try { + mkdirSync(dirname(path), { recursive: true }); + } catch { + throw new Error('Markdown benchmark output could not be written.'); + } + assertNoSymlinkOutputAncestors(path); + try { + writeFileSync(path, content, { encoding: 'utf8', flag: 'wx' }); + } catch { + throw new Error('Markdown benchmark output could not be written.'); + } +} + +async function main() { + const args = resolveArguments(process.argv.slice(2)); + assertNoSymlinkOutputAncestors(args.outputPath); + const source = readBoundedMarkdown(args.inputPath); + if ( + args.inputPath === args.outputPath || + refersToSameFile(args.inputPath, args.outputPath) + ) { + throw new Error('Markdown benchmark output must not overwrite its input.'); + } + const modulePath = resolveLocalModule(args.modulePath); + if ( + modulePath === args.outputPath || + refersToSameFile(modulePath, args.outputPath) + ) { + throw new Error( + 'Markdown benchmark output must not overwrite the measured module.', + ); + } + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); + + const measuredModule = await loadMeasuredModule(modulePath); + const contract = serializationContract(args.operation); + const serializer = measuredModule[contract.exportName]; + if (typeof serializer !== 'function') { + throw new Error( + `Measured Markdown module must export ${contract.exportName}().`, + ); + } + + const warmup = runMeasuredSerialization( + serializer, + source, + contract.executionFailure, + ); + if (typeof warmup !== 'string') { + throw new Error(contract.returnFailure); + } + + const samples = []; + for (let index = 0; index < args.sampleCount; index += 1) { + const start = performance.now(); + const output = runMeasuredSerialization( + serializer, + source, + contract.executionFailure, + ); + const elapsed = performance.now() - start; + if ( + typeof output !== 'string' || + !Number.isFinite(elapsed) || + elapsed < 0 + ) { + throw new Error('Markdown measurement produced invalid runtime evidence.'); + } + samples.push(elapsed); + } + + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); + assertNoSymlinkOutputAncestors(args.outputPath); + const outputMetadata = inspectOutputPath(args.outputPath); + if (outputMetadata !== undefined && !outputMetadata.isFile()) { + throw new Error('Markdown benchmark output must be a regular file.'); + } + if (outputMetadata !== undefined && outputMetadata.nlink !== 1) { + throw new Error('Markdown benchmark output must not be multiply linked.'); + } + if (outputMetadata !== undefined) { + throw new Error(OUTPUT_EXISTS_ERROR); + } + writeMeasurementOutput( + args.outputPath, + `${JSON.stringify( + { + contractVersion: 1, + benchmarkId: `${contract.benchmarkPrefix}-${args.profile}`, + unit: 'ms', + sourceCommitSha: args.sourceCommitSha, + artifactSha256: args.artifactSha256, + documentProfile: args.profile, + runtimeId: args.runtimeId, + referenceHardwareId: args.referenceHardwareId, + samples, + }, + null, + 2, + )}\n`, + ); +} + +try { + await main(); +} catch (error) { + const message = + error instanceof Error + ? error.message + : 'Markdown benchmark measurement failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} diff --git a/benchmarks/measure-revision-evidence.mjs b/benchmarks/measure-revision-evidence.mjs new file mode 100644 index 00000000..1e3e438d --- /dev/null +++ b/benchmarks/measure-revision-evidence.mjs @@ -0,0 +1,450 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { performance } from 'node:perf_hooks'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdirSync, + openSync, + readSync, + realpathSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(benchmarkDirectory, '..'); +const MAX_INPUT_BYTES = 16 * 1024 * 1024; +const MAX_MODULE_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000; +const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; +const OUTPUT_DIRECTORY_ERROR = + 'Revision benchmark output directory must be a non-symlink directory.'; +const OUTPUT_EXISTS_ERROR = + 'Revision benchmark output must not already exist.'; + +function resolveArguments(argv) { + const expectedFlags = [ + '--input', + '--module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', + ]; + if ( + argv.length !== expectedFlags.length * 2 || + expectedFlags.some((flag, index) => argv[index * 2] !== flag) || + expectedFlags.some((_, index) => argv[index * 2 + 1]?.length === 0) + ) { + throw new Error( + 'Usage: node benchmarks/measure-revision-evidence.mjs --input --module --profile --samples --source-commit-sha --artifact-sha256 --runtime-id --reference-hardware-id --output ', + ); + } + + const values = Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); + const profile = values['--profile']; + if (!DOCUMENT_PROFILES.has(profile)) { + throw new Error('Revision benchmark profile is invalid.'); + } + const sampleCount = Number(values['--samples']); + if ( + !Number.isSafeInteger(sampleCount) || + sampleCount < 1 || + sampleCount > MAX_SAMPLES + ) { + throw new Error('Revision benchmark sample count must be an integer from 1 to 1000.'); + } + const sourceCommitSha = values['--source-commit-sha']; + if (!SHA1_PATTERN.test(sourceCommitSha)) { + throw new Error( + 'Revision benchmark source commit must be a lowercase 40-character SHA.', + ); + } + const artifactSha256 = values['--artifact-sha256']; + if (!SHA256_PATTERN.test(artifactSha256)) { + throw new Error( + 'Revision benchmark artifact digest must be a lowercase 64-character SHA-256.', + ); + } + const runtimeId = values['--runtime-id']; + if (!RUNTIME_ID_PATTERN.test(runtimeId)) { + throw new Error('Revision benchmark runtime ID is invalid.'); + } + const referenceHardwareId = values['--reference-hardware-id']; + if (!REFERENCE_HARDWARE_ID_PATTERN.test(referenceHardwareId)) { + throw new Error('Revision benchmark reference hardware ID is invalid.'); + } + + return Object.freeze({ + inputPath: resolve(values['--input']), + modulePath: values['--module'], + profile, + sampleCount, + sourceCommitSha, + artifactSha256, + runtimeId, + referenceHardwareId, + outputPath: resolve(values['--output']), + }); +} + +function assertMeasurementProvenance(sourceCommitSha, runtimeId) { + const result = spawnSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + const checkoutSha = result.stdout?.trim(); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !SHA1_PATTERN.test(checkoutSha ?? '') + ) { + throw new Error( + 'Benchmark measurement source commit could not be verified against the current checkout.', + ); + } + if (sourceCommitSha !== checkoutSha) { + throw new Error( + 'Benchmark measurement source commit does not match checked-out HEAD.', + ); + } + if (runtimeId !== `node-${process.versions.node}`) { + throw new Error( + 'Benchmark measurement runtime ID must match the active Node runtime.', + ); + } +} + +function readBoundedRegularFile(path, maximumBytes, invalidFileMessage, oversizedMessage) { + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error(invalidFileMessage); + } + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error(invalidFileMessage); + } + + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error(invalidFileMessage); + } + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error(invalidFileMessage); + } + if (metadata.size > maximumBytes) { + throw new Error(oversizedMessage); + } + const chunks = []; + let totalBytes = 0; + while (totalBytes <= maximumBytes) { + const remainingBudget = maximumBytes + 1 - totalBytes; + const chunk = Buffer.allocUnsafe(Math.min(READ_CHUNK_BYTES, remainingBudget)); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > maximumBytes) { + throw new Error(oversizedMessage); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + return Buffer.concat(chunks, totalBytes); + } finally { + closeSync(descriptor); + } +} + +function readBoundedEnvelopeBytes(path) { + return readBoundedRegularFile( + path, + MAX_INPUT_BYTES, + 'Revision benchmark input must be a regular non-symlink file.', + 'Revision benchmark input exceeds the supported size.', + ); +} + +function resolveLocalModule(pathOrUrl) { + if ( + pathOrUrl.startsWith('http:') || + pathOrUrl.startsWith('https:') || + pathOrUrl.startsWith('data:') || + pathOrUrl.startsWith('node:') + ) { + throw new Error('Measured revision module must be a local regular file.'); + } + let moduleUrl; + try { + moduleUrl = pathOrUrl.startsWith('file:') + ? new URL(pathOrUrl) + : pathToFileURL(resolve(pathOrUrl)); + } catch { + throw new Error('Measured revision module must be a local regular file.'); + } + if (moduleUrl.protocol !== 'file:') { + throw new Error('Measured revision module must be a local regular file.'); + } + let resolvedPath; + try { + resolvedPath = resolve(fileURLToPath(moduleUrl)); + } catch { + throw new Error('Measured revision module must be a local regular file.'); + } + let metadata; + try { + metadata = lstatSync(resolvedPath, { throwIfNoEntry: false }); + } catch { + throw new Error('Measured revision module must be a local regular file.'); + } + if ( + metadata === undefined || + metadata.isSymbolicLink() || + !metadata.isFile() + ) { + throw new Error('Measured revision module must be a local regular file.'); + } + try { + return realpathSync(resolvedPath); + } catch { + throw new Error('Measured revision module must be a local regular file.'); + } +} + +function measuredModuleSha256(modulePath) { + const bytes = readBoundedRegularFile( + modulePath, + MAX_MODULE_BYTES, + 'Measured revision module must be a local regular file.', + 'Measured revision module exceeds the supported size.', + ); + return createHash('sha256').update(bytes).digest('hex'); +} + +function verifyMeasuredModuleDigest(modulePath, expectedSha256) { + if (measuredModuleSha256(modulePath) !== expectedSha256) { + throw new Error( + 'Revision benchmark artifact digest does not match the measured module.', + ); + } +} + +function inspectOutputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Revision benchmark output path could not be inspected.'); + } +} + +function inspectOutputDirectoryComponent(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Revision benchmark output directory could not be inspected.'); + } +} + +function assertNoSymlinkOutputAncestors(path) { + let current = dirname(path); + while (true) { + const metadata = inspectOutputDirectoryComponent(current); + if (metadata?.isSymbolicLink()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + +function refersToSameFile(leftPath, rightPath) { + const rightMetadata = inspectOutputPath(rightPath); + if (rightMetadata === undefined) return false; + if (!rightMetadata.isFile()) { + throw new Error('Revision benchmark output must be a regular file.'); + } + const left = statSync(leftPath); + const right = statSync(rightPath); + return left.dev === right.dev && left.ino === right.ino; +} + +async function loadMeasuredModule(modulePath) { + try { + return await import(pathToFileURL(modulePath).href); + } catch { + throw new Error('Measured revision module could not be loaded.'); + } +} + +function writeMeasurementOutput(path, content) { + assertNoSymlinkOutputAncestors(path); + try { + mkdirSync(dirname(path), { recursive: true }); + } catch { + throw new Error('Revision benchmark output could not be written.'); + } + assertNoSymlinkOutputAncestors(path); + try { + writeFileSync(path, content, { encoding: 'utf8', flag: 'wx' }); + } catch { + throw new Error('Revision benchmark output could not be written.'); + } +} + +async function runMeasuredRevision(createRevisionEvidence, source) { + let evidence; + try { + evidence = await createRevisionEvidence(source); + } catch { + throw new Error('Measured revision-evidence execution failed.'); + } + + let digestHex; + try { + if (typeof evidence !== 'object' || evidence === null) { + throw new Error('invalid revision evidence'); + } + const revision = evidence.revision; + if (typeof revision !== 'object' || revision === null) { + throw new Error('invalid revision evidence'); + } + digestHex = revision.digestHex; + } catch { + throw new Error('Measured revision-evidence result is invalid.'); + } + + if (typeof digestHex !== 'string' || !SHA256_PATTERN.test(digestHex)) { + throw new Error('Measured revision-evidence result is invalid.'); + } +} + +async function main() { + const args = resolveArguments(process.argv.slice(2)); + assertNoSymlinkOutputAncestors(args.outputPath); + const source = readBoundedEnvelopeBytes(args.inputPath); + if ( + args.inputPath === args.outputPath || + refersToSameFile(args.inputPath, args.outputPath) + ) { + throw new Error('Revision benchmark output must not overwrite its input.'); + } + const modulePath = resolveLocalModule(args.modulePath); + if ( + modulePath === args.outputPath || + refersToSameFile(modulePath, args.outputPath) + ) { + throw new Error( + 'Revision benchmark output must not overwrite the measured module.', + ); + } + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); + + const measuredModule = await loadMeasuredModule(modulePath); + if ( + typeof measuredModule.createDocumentEnvelopeRevisionEvidenceBytes !== + 'function' + ) { + throw new Error( + 'Measured revision module must export createDocumentEnvelopeRevisionEvidenceBytes().', + ); + } + + await runMeasuredRevision( + measuredModule.createDocumentEnvelopeRevisionEvidenceBytes, + source, + ); + + const samples = []; + for (let index = 0; index < args.sampleCount; index += 1) { + const start = performance.now(); + await runMeasuredRevision( + measuredModule.createDocumentEnvelopeRevisionEvidenceBytes, + source, + ); + const elapsed = performance.now() - start; + if (!Number.isFinite(elapsed) || elapsed < 0) { + throw new Error('Revision measurement produced invalid runtime evidence.'); + } + samples.push(elapsed); + } + + verifyMeasuredModuleDigest(modulePath, args.artifactSha256); + assertMeasurementProvenance(args.sourceCommitSha, args.runtimeId); + assertNoSymlinkOutputAncestors(args.outputPath); + const outputMetadata = inspectOutputPath(args.outputPath); + if (outputMetadata !== undefined && !outputMetadata.isFile()) { + throw new Error('Revision benchmark output must be a regular file.'); + } + if (outputMetadata !== undefined && outputMetadata.nlink !== 1) { + throw new Error('Revision benchmark output must not be multiply linked.'); + } + if (outputMetadata !== undefined) { + throw new Error(OUTPUT_EXISTS_ERROR); + } + writeMeasurementOutput( + args.outputPath, + `${JSON.stringify( + { + contractVersion: 1, + benchmarkId: `revision-evidence-${args.profile}`, + unit: 'ms', + sourceCommitSha: args.sourceCommitSha, + artifactSha256: args.artifactSha256, + documentProfile: args.profile, + runtimeId: args.runtimeId, + referenceHardwareId: args.referenceHardwareId, + samples, + }, + null, + 2, + )}\n`, + ); +} + +try { + await main(); +} catch (error) { + const message = + error instanceof Error + ? error.message + : 'Revision benchmark measurement failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} diff --git a/benchmarks/office-fixtures.lock.json b/benchmarks/office-fixtures.lock.json new file mode 100644 index 00000000..01f9920e --- /dev/null +++ b/benchmarks/office-fixtures.lock.json @@ -0,0 +1,42 @@ +{ + "contractVersion": 1, + "synthetic": true, + "formats": { + "docx": { + "small": { + "units": 2, + "bytes": 2972, + "sha256": "c356496b106f5348e98b00d3c1e18185165653a2c2bda083acd7456c96b2eab3" + }, + "page120": { + "units": 120, + "bytes": 169737, + "sha256": "e5d90408c6061051ab1674d931b99d478fadb00a7ecbcb7025b4fa478cfbb507" + } + }, + "xlsx": { + "small": { + "units": 4, + "bytes": 723, + "sha256": "a267a6208de0e24559c200d047404d14e2263581e0a3af845f355817536883bf" + }, + "wide16384": { + "units": 16384, + "bytes": 327941, + "sha256": "0afdb216fda720cd506d7c24284c2740f326167d4859dc1a7b845c4a91b972ec" + } + }, + "pptx": { + "small": { + "units": 2, + "bytes": 970, + "sha256": "6c1a8ae1d2307a278ac97903eb3160a6a0de57b695b2b8772844af2c8cfa2ce1" + }, + "slide120": { + "units": 120, + "bytes": 50061, + "sha256": "f5994ff30752581fd3b3e5210ff59b281692ea268ce4f0b540e278d134dbd6ee" + } + } + } +} diff --git a/benchmarks/run-current-suite-core.mjs b/benchmarks/run-current-suite-core.mjs new file mode 100644 index 00000000..a18479e8 --- /dev/null +++ b/benchmarks/run-current-suite-core.mjs @@ -0,0 +1,619 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdirSync, + mkdtempSync, + openSync, + readSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(benchmarkDirectory, '..'); +const legacyFlags = Object.freeze([ + '--input', + '--module', + '--revision-input', + '--revision-module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--revision-artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const packedFlags = Object.freeze([ + '--input', + '--revision-input', + '--package-tarball', + '--package-sha256', + '--profile', + '--samples', + '--source-commit-sha', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const COMMIT_SHA_PATTERN = /^[0-9a-f]{40}$/u; +const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; +const MAX_PACKAGE_INDEX_BYTES = 1024 * 1024; +const MAX_PACKAGE_MANIFEST_BYTES = 1024 * 1024; +const MAX_MODULE_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const EXPECTED_PACKAGE_NAME = '@contextualwisdomlab/cwl-editor'; +const PACKAGE_MANIFEST_ENTRY = 'package/package.json'; +const MARKDOWN_MODULE_ENTRY = 'package/dist/cwl-markdown.js'; +const REVISION_MODULE_ENTRY = 'package/dist/cwl-revision-evidence.js'; +const OUTPUT_DIRECTORY_ERROR = + 'Benchmark suite output directory must be a non-symlink directory.'; +const OUTPUT_DIRECTORY_EXISTS_ERROR = + 'Benchmark suite output directory must not already exist.'; + +function matchesArguments(argv, expectedFlags) { + return ( + argv.length === expectedFlags.length * 2 && + expectedFlags.every((flag, index) => argv[index * 2] === flag) && + expectedFlags.every((_, index) => argv[index * 2 + 1]?.length > 0) + ); +} + +function valuesForArguments(argv, expectedFlags) { + return Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); +} + +function sharedArguments(values) { + return Object.freeze({ + documentProfile: values['--profile'], + sampleCount: values['--samples'], + sourceCommitSha: values['--source-commit-sha'], + runtimeId: values['--runtime-id'], + referenceHardwareId: values['--reference-hardware-id'], + markdownInputPath: values['--input'], + revisionInputPath: values['--revision-input'], + outputDirectory: resolve(values['--output']), + }); +} + +function measurementArguments({ + inputPath, + modulePath, + artifactSha256, + shared, +}) { + return Object.freeze([ + '--input', + inputPath, + '--module', + modulePath, + '--profile', + shared.documentProfile, + '--samples', + shared.sampleCount, + '--source-commit-sha', + shared.sourceCommitSha, + '--artifact-sha256', + artifactSha256, + '--runtime-id', + shared.runtimeId, + '--reference-hardware-id', + shared.referenceHardwareId, + ]); +} + +function currentCheckoutSha() { + const result = spawnSync('git', ['rev-parse', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + const checkoutSha = result.stdout?.trim(); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !COMMIT_SHA_PATTERN.test(checkoutSha ?? '') + ) { + throw new Error( + 'Benchmark suite source commit SHA could not be verified against the current checkout.', + ); + } + return checkoutSha; +} + +function resolveArguments(argv) { + if (matchesArguments(argv, packedFlags)) { + const values = valuesForArguments(argv, packedFlags); + const packageSha256 = values['--package-sha256']; + if (!SHA256_PATTERN.test(packageSha256)) { + throw new Error( + 'Benchmark suite package digest must be a lowercase 64-character SHA-256.', + ); + } + const activeRuntimeId = `node-${process.versions.node}`; + if (values['--runtime-id'] !== activeRuntimeId) { + throw new Error( + 'Benchmark suite runtime ID must match the active Node runtime.', + ); + } + if (values['--source-commit-sha'] !== currentCheckoutSha()) { + throw new Error( + 'Benchmark suite source commit SHA must match the current benchmark checkout.', + ); + } + return Object.freeze({ + mode: 'packed', + shared: sharedArguments(values), + packageTarballPath: resolve(values['--package-tarball']), + packageSha256, + }); + } + + if (matchesArguments(argv, legacyFlags)) { + const values = valuesForArguments(argv, legacyFlags); + const shared = sharedArguments(values); + return Object.freeze({ + mode: 'module', + shared, + markdownArguments: measurementArguments({ + inputPath: shared.markdownInputPath, + modulePath: values['--module'], + artifactSha256: values['--artifact-sha256'], + shared, + }), + revisionArguments: measurementArguments({ + inputPath: shared.revisionInputPath, + modulePath: values['--revision-module'], + artifactSha256: values['--revision-artifact-sha256'], + shared, + }), + packageEvidence: null, + }); + } + + throw new Error( + 'Usage: node benchmarks/run-current-suite.mjs --input --revision-input --package-tarball --package-sha256 --profile --samples --source-commit-sha --runtime-id --reference-hardware-id --output ', + ); +} + +function inspectOutputDirectory(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark suite output directory could not be inspected.'); + } +} + +function assertNoSymlinkDirectoryComponents(path) { + let current = path; + while (true) { + const metadata = inspectOutputDirectory(current); + if (metadata?.isSymbolicLink()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + +function prepareOutputDirectory(path) { + assertNoSymlinkDirectoryComponents(path); + const existing = inspectOutputDirectory(path); + if (existing !== undefined) { + if (!existing.isDirectory()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + throw new Error(OUTPUT_DIRECTORY_EXISTS_ERROR); + } + + try { + mkdirSync(path, { recursive: true }); + } catch { + throw new Error('Benchmark suite output directory could not be prepared.'); + } + + assertNoSymlinkDirectoryComponents(path); + const created = inspectOutputDirectory(path); + if (created === undefined || !created.isDirectory()) { + throw new Error(OUTPUT_DIRECTORY_ERROR); + } + return true; +} + +function removePartialOutputDirectory(path) { + try { + rmSync(path, { recursive: true, force: true }); + } catch { + throw new Error('Benchmark suite partial evidence could not be removed.'); + } +} + +function readBoundedRegularFile(path, maximumBytes, invalidMessage, oversizedMessage) { + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error(invalidMessage); + } + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error(invalidMessage); + } + + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error(invalidMessage); + } + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) throw new Error(invalidMessage); + if (metadata.size > maximumBytes) throw new Error(oversizedMessage); + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= maximumBytes) { + const remainingBudget = maximumBytes + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > maximumBytes) throw new Error(oversizedMessage); + chunks.push(chunk.subarray(0, bytesRead)); + } + return Buffer.concat(chunks, totalBytes); + } finally { + closeSync(descriptor); + } +} + +function packageTarballBytes(path) { + return readBoundedRegularFile( + path, + MAX_PACKAGE_BYTES, + 'Benchmark suite package tarball must be a regular non-symlink file.', + 'Benchmark suite package tarball exceeds the supported size.', + ); +} + +function verifyPackageDigest(path, expectedSha256) { + const actualSha256 = createHash('sha256') + .update(packageTarballBytes(path)) + .digest('hex'); + if (actualSha256 !== expectedSha256) { + throw new Error('Benchmark suite package digest does not match the packed artifact.'); + } +} + +function runTar(argumentsList, maximumBytes, failureMessage) { + const result = spawnSync('tar', argumentsList, { + cwd: repositoryRoot, + maxBuffer: maximumBytes, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 30_000, + }); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !Buffer.isBuffer(result.stdout) || + result.stdout.byteLength > maximumBytes + ) { + throw new Error(failureMessage); + } + return result.stdout; +} + +function decodeUtf8(bytes, failureMessage) { + try { + return new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch { + throw new Error(failureMessage); + } +} + +function listPackedEntries(tarballPath) { + const listing = decodeUtf8( + runTar( + ['-tzf', tarballPath], + MAX_PACKAGE_INDEX_BYTES, + 'Benchmark suite package index could not be read.', + ), + 'Benchmark suite package index must be valid UTF-8.', + ); + return listing.split('\n').filter((entry) => entry.length > 0); +} + +function assertUniquePackedEntry(entries, expectedEntry) { + if (entries.filter((entry) => entry === expectedEntry).length !== 1) { + throw new Error('Benchmark suite package is missing a unique required artifact.'); + } +} + +function readPackedEntry(tarballPath, entry, maximumBytes) { + return runTar( + ['-xOzf', tarballPath, entry], + maximumBytes, + 'Benchmark suite package artifact could not be read.', + ); +} + +function parsePackageManifest(bytes) { + let manifest; + try { + manifest = JSON.parse( + decodeUtf8(bytes, 'Benchmark suite package manifest must be valid UTF-8.'), + ); + } catch (error) { + if (error instanceof Error && error.message.includes('valid UTF-8')) throw error; + throw new Error('Benchmark suite package manifest must be valid JSON.'); + } + if ( + manifest === null || + typeof manifest !== 'object' || + Array.isArray(manifest) || + manifest.name !== EXPECTED_PACKAGE_NAME || + typeof manifest.version !== 'string' || + manifest.version.length === 0 || + manifest.version.length > 128 + ) { + throw new Error('Benchmark suite package identity is invalid.'); + } + return Object.freeze({ name: manifest.name, version: manifest.version }); +} + +function moduleSha256(bytes) { + return createHash('sha256').update(bytes).digest('hex'); +} + +function preparePackedBenchmarkModules(args) { + verifyPackageDigest(args.packageTarballPath, args.packageSha256); + const entries = listPackedEntries(args.packageTarballPath); + for (const entry of [ + PACKAGE_MANIFEST_ENTRY, + MARKDOWN_MODULE_ENTRY, + REVISION_MODULE_ENTRY, + ]) { + assertUniquePackedEntry(entries, entry); + } + + const manifestBytes = readPackedEntry( + args.packageTarballPath, + PACKAGE_MANIFEST_ENTRY, + MAX_PACKAGE_MANIFEST_BYTES, + ); + const manifest = parsePackageManifest(manifestBytes); + const markdownModuleBytes = readPackedEntry( + args.packageTarballPath, + MARKDOWN_MODULE_ENTRY, + MAX_MODULE_BYTES, + ); + const revisionModuleBytes = readPackedEntry( + args.packageTarballPath, + REVISION_MODULE_ENTRY, + MAX_MODULE_BYTES, + ); + verifyPackageDigest(args.packageTarballPath, args.packageSha256); + + const temporaryDirectory = mkdtempSync( + join(tmpdir(), 'inkspan-packed-benchmark-'), + ); + const markdownModulePath = join(temporaryDirectory, 'cwl-markdown.mjs'); + const revisionModulePath = join( + temporaryDirectory, + 'cwl-revision-evidence.mjs', + ); + try { + writeFileSync(markdownModulePath, markdownModuleBytes); + writeFileSync(revisionModulePath, revisionModuleBytes); + } catch { + rmSync(temporaryDirectory, { recursive: true, force: true }); + throw new Error('Benchmark suite package modules could not be prepared.'); + } + + return Object.freeze({ + temporaryDirectory, + markdownModulePath, + markdownArtifactSha256: moduleSha256(markdownModuleBytes), + revisionModulePath, + revisionArtifactSha256: moduleSha256(revisionModuleBytes), + packageEvidence: Object.freeze({ + packageName: manifest.name, + packageVersion: manifest.version, + packageSha256: args.packageSha256, + }), + }); +} + +function runBoundedNodeScript(scriptName, args, failureMessage) { + const result = spawnSync( + process.execPath, + [resolve(benchmarkDirectory, scriptName), ...args], + { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 4 * 1024 * 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 120_000, + }, + ); + + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 + ) { + throw new Error(failureMessage); + } +} + +function runMeasurementAndSummary({ + measurementScript, + measurementArguments: argumentsList, + samplesPath, + summaryDirectory, + measurementFailure, + summaryFailure, +}) { + runBoundedNodeScript( + measurementScript, + [...argumentsList, '--output', samplesPath], + measurementFailure, + ); + runBoundedNodeScript( + 'summarize-samples.mjs', + ['--input', samplesPath, '--output', summaryDirectory], + summaryFailure, + ); +} + +function runSuite(args, markdownArguments, revisionArguments) { + const markdownSamplesPath = resolve( + args.outputDirectory, + 'markdown', + 'samples.json', + ); + const markdownSummaryDirectory = resolve( + args.outputDirectory, + 'markdown', + 'summary', + ); + const revisionSamplesPath = resolve( + args.outputDirectory, + 'revision', + 'samples.json', + ); + const revisionSummaryDirectory = resolve( + args.outputDirectory, + 'revision', + 'summary', + ); + + runMeasurementAndSummary({ + measurementScript: 'measure-markdown.mjs', + measurementArguments: markdownArguments, + samplesPath: markdownSamplesPath, + summaryDirectory: markdownSummaryDirectory, + measurementFailure: 'Benchmark suite Markdown measurement failed.', + summaryFailure: 'Benchmark suite Markdown summary failed.', + }); + runMeasurementAndSummary({ + measurementScript: 'measure-revision-evidence.mjs', + measurementArguments: revisionArguments, + samplesPath: revisionSamplesPath, + summaryDirectory: revisionSummaryDirectory, + measurementFailure: 'Benchmark suite revision measurement failed.', + summaryFailure: 'Benchmark suite revision summary failed.', + }); +} + +function suiteManifest(args, packageEvidence) { + return Object.freeze({ + contractVersion: 1, + documentProfile: args.documentProfile, + sampleCount: Number(args.sampleCount), + sourceCommitSha: args.sourceCommitSha, + runtimeId: args.runtimeId, + referenceHardwareId: args.referenceHardwareId, + ...(packageEvidence ?? {}), + markdownSamples: 'markdown/samples.json', + markdownSummaryJson: 'markdown/summary/summary.json', + markdownSummaryText: 'markdown/summary/summary.txt', + revisionSamples: 'revision/samples.json', + revisionSummaryJson: 'revision/summary/summary.json', + revisionSummaryText: 'revision/summary/summary.txt', + status: 'completed', + }); +} + +function main(argv) { + const resolved = resolveArguments(argv); + const shared = resolved.shared; + let preparedPackage; + if (resolved.mode === 'packed') { + preparedPackage = preparePackedBenchmarkModules(resolved); + } + + const markdownArguments = + resolved.mode === 'packed' + ? measurementArguments({ + inputPath: shared.markdownInputPath, + modulePath: preparedPackage.markdownModulePath, + artifactSha256: preparedPackage.markdownArtifactSha256, + shared, + }) + : resolved.markdownArguments; + const revisionArguments = + resolved.mode === 'packed' + ? measurementArguments({ + inputPath: shared.revisionInputPath, + modulePath: preparedPackage.revisionModulePath, + artifactSha256: preparedPackage.revisionArtifactSha256, + shared, + }) + : resolved.revisionArguments; + const packageEvidence = + resolved.mode === 'packed' ? preparedPackage.packageEvidence : null; + + let createdOutputDirectory = false; + try { + createdOutputDirectory = prepareOutputDirectory(shared.outputDirectory); + runSuite(shared, markdownArguments, revisionArguments); + if (resolved.mode === 'packed') { + verifyPackageDigest(resolved.packageTarballPath, resolved.packageSha256); + } + } catch (error) { + if (createdOutputDirectory) { + removePartialOutputDirectory(shared.outputDirectory); + } + throw error; + } finally { + if (preparedPackage !== undefined) { + rmSync(preparedPackage.temporaryDirectory, { + recursive: true, + force: true, + }); + } + } + + process.stdout.write( + `${JSON.stringify(suiteManifest(shared, packageEvidence))}\n`, + ); +} + +try { + main(process.argv.slice(2)); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Benchmark suite failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} diff --git a/benchmarks/run-current-suite.mjs b/benchmarks/run-current-suite.mjs new file mode 100644 index 00000000..21ca46e5 --- /dev/null +++ b/benchmarks/run-current-suite.mjs @@ -0,0 +1,571 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + closeSync, + constants, + fstatSync, + lstatSync, + mkdtempSync, + openSync, + readSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { assertCleanSourceCheckout } from './source-checkout-provenance.mjs'; + +const benchmarkDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(benchmarkDirectory, '..'); +const coreRunnerPath = resolve(benchmarkDirectory, 'run-current-suite-core.mjs'); +const markdownMeasurementPath = resolve( + benchmarkDirectory, + 'measure-markdown.mjs', +); +const sampleSummaryPath = resolve(benchmarkDirectory, 'summarize-samples.mjs'); +const MAX_PACKAGE_BYTES = 64 * 1024 * 1024; +const MAX_MODULE_BYTES = 16 * 1024 * 1024; +const MAX_CHILD_OUTPUT_BYTES = 4 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const READ_ONLY_NOFOLLOW = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0); +const PACKED_MARKDOWN_MODULE_ENTRY = 'package/dist/cwl-markdown.js'; +const legacyFlags = Object.freeze([ + '--input', + '--module', + '--revision-input', + '--revision-module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--revision-artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const htmlLegacyFlags = Object.freeze([ + '--input', + '--html-input', + '--module', + '--revision-input', + '--revision-module', + '--profile', + '--samples', + '--source-commit-sha', + '--artifact-sha256', + '--revision-artifact-sha256', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const packedFlags = Object.freeze([ + '--input', + '--revision-input', + '--package-tarball', + '--package-sha256', + '--profile', + '--samples', + '--source-commit-sha', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); +const packedHtmlFlags = Object.freeze([ + '--input', + '--html-input', + '--revision-input', + '--package-tarball', + '--package-sha256', + '--profile', + '--samples', + '--source-commit-sha', + '--runtime-id', + '--reference-hardware-id', + '--output', +]); + +function matchesArguments(argv, expectedFlags) { + return ( + argv.length === expectedFlags.length * 2 && + expectedFlags.every((flag, index) => argv[index * 2] === flag) && + expectedFlags.every((_, index) => argv[index * 2 + 1]?.length > 0) + ); +} + +function valuesForArguments(argv, expectedFlags) { + return Object.fromEntries( + expectedFlags.map((flag, index) => [flag, argv[index * 2 + 1]]), + ); +} + +function argumentsForFlags(values, flags) { + return flags.flatMap((flag) => [flag, values[flag]]); +} + +function matchingFlags(argv) { + if (matchesArguments(argv, htmlLegacyFlags)) return htmlLegacyFlags; + if (matchesArguments(argv, packedHtmlFlags)) return packedHtmlFlags; + if (matchesArguments(argv, packedFlags)) return packedFlags; + if (matchesArguments(argv, legacyFlags)) return legacyFlags; + return null; +} + +function inspectOutputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark suite output directory could not be inspected.'); + } +} + +function assertFreshOutputDirectory(argv) { + const flags = matchingFlags(argv); + if (flags === null) return; + const outputValueIndex = flags.indexOf('--output') * 2 + 1; + const outputDirectory = resolve(repositoryRoot, argv[outputValueIndex]); + + let current = outputDirectory; + while (true) { + const metadata = inspectOutputPath(current); + if (metadata?.isSymbolicLink()) { + throw new Error( + 'Benchmark suite output directory must be a non-symlink directory.', + ); + } + if (current === outputDirectory && metadata !== undefined) { + if (!metadata.isDirectory()) { + throw new Error( + 'Benchmark suite output directory must be a non-symlink directory.', + ); + } + throw new Error('Benchmark suite output directory must not already exist.'); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + +function claimedLegacySourceCommitSha(argv) { + if (matchesArguments(argv, htmlLegacyFlags)) { + return valuesForArguments(argv, htmlLegacyFlags)['--source-commit-sha']; + } + if (matchesArguments(argv, legacyFlags)) { + return valuesForArguments(argv, legacyFlags)['--source-commit-sha']; + } + return undefined; +} + +function readPackedTarballSnapshot(path) { + let pathMetadata; + try { + pathMetadata = lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error( + 'Benchmark suite package tarball must be a regular non-symlink file.', + ); + } + if ( + pathMetadata === undefined || + pathMetadata.isSymbolicLink() || + !pathMetadata.isFile() + ) { + throw new Error( + 'Benchmark suite package tarball must be a regular non-symlink file.', + ); + } + + let descriptor; + try { + descriptor = openSync(path, READ_ONLY_NOFOLLOW); + } catch { + throw new Error( + 'Benchmark suite package tarball must be a regular non-symlink file.', + ); + } + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error( + 'Benchmark suite package tarball must be a regular non-symlink file.', + ); + } + if (metadata.size > MAX_PACKAGE_BYTES) { + throw new Error('Benchmark suite package tarball exceeds the supported size.'); + } + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= MAX_PACKAGE_BYTES) { + const remainingBudget = MAX_PACKAGE_BYTES + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > MAX_PACKAGE_BYTES) { + throw new Error('Benchmark suite package tarball exceeds the supported size.'); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + return Buffer.concat(chunks, totalBytes); + } finally { + closeSync(descriptor); + } +} + +function snapshotPackedArguments(argv) { + const flags = matchesArguments(argv, packedHtmlFlags) + ? packedHtmlFlags + : matchesArguments(argv, packedFlags) + ? packedFlags + : null; + if (flags === null) { + return Object.freeze({ argv, temporaryDirectory: null }); + } + + const packageTarballValueIndex = + flags.indexOf('--package-tarball') * 2 + 1; + const packageTarballPath = resolve(argv[packageTarballValueIndex]); + const packageBytes = readPackedTarballSnapshot(packageTarballPath); + const temporaryDirectory = mkdtempSync( + join(tmpdir(), 'inkspan-packed-suite-snapshot-'), + ); + const snapshotPath = join(temporaryDirectory, 'package.tgz'); + try { + writeFileSync(snapshotPath, packageBytes, { mode: 0o600 }); + } catch { + rmSync(temporaryDirectory, { recursive: true, force: true }); + throw new Error('Benchmark suite package tarball snapshot could not be prepared.'); + } + + const snapshottedArguments = [...argv]; + snapshottedArguments[packageTarballValueIndex] = snapshotPath; + return Object.freeze({ + argv: snapshottedArguments, + temporaryDirectory, + }); +} + +function runBoundedNode(scriptPath, args, failureMessage) { + const result = spawnSync(process.execPath, [scriptPath, ...args], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: MAX_CHILD_OUTPUT_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 600_000, + }); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 + ) { + throw new Error(failureMessage); + } + return result.stdout; +} + +function runCoreNodePreservingError(args) { + const result = spawnSync(process.execPath, [coreRunnerPath, ...args], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: MAX_CHILD_OUTPUT_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 600_000, + }); + if (result.error !== undefined || result.signal !== null) { + throw new Error('Benchmark suite internal runner could not complete.'); + } + if (result.status !== 0) { + const message = result.stderr.trimEnd(); + throw new Error( + message.length > 0 ? message : 'Benchmark suite internal runner failed.', + ); + } + return result.stdout; +} + +function parseCoreManifest(stdout) { + let manifest; + try { + manifest = JSON.parse(stdout.trim()); + } catch { + throw new Error('Benchmark suite internal runner returned invalid evidence.'); + } + if ( + manifest === null || + typeof manifest !== 'object' || + Array.isArray(manifest) || + manifest.status !== 'completed' + ) { + throw new Error('Benchmark suite internal runner returned invalid evidence.'); + } + return manifest; +} + +function readPackedMarkdownModule(tarballPath) { + const result = spawnSync( + 'tar', + ['-xOzf', tarballPath, PACKED_MARKDOWN_MODULE_ENTRY], + { + cwd: repositoryRoot, + maxBuffer: MAX_MODULE_BYTES + 1, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 30_000, + }, + ); + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + !Buffer.isBuffer(result.stdout) || + result.stdout.byteLength > MAX_MODULE_BYTES + ) { + throw new Error('Benchmark suite packed Markdown module could not be read.'); + } + return result.stdout; +} + +function assertPackedSnapshotDigest(tarballPath, expectedSha256) { + const actualSha256 = createHash('sha256') + .update(readPackedTarballSnapshot(tarballPath)) + .digest('hex'); + if (actualSha256 !== expectedSha256) { + throw new Error('Benchmark suite package digest does not match the packed artifact.'); + } +} + +function htmlSerializationEvidenceArguments(values, modulePath, artifactSha256) { + return [ + '--input', + values['--html-input'], + '--module', + modulePath, + '--operation', + 'html-to-markdown', + '--profile', + values['--profile'], + '--samples', + values['--samples'], + '--source-commit-sha', + values['--source-commit-sha'], + '--artifact-sha256', + artifactSha256, + '--runtime-id', + values['--runtime-id'], + '--reference-hardware-id', + values['--reference-hardware-id'], + ]; +} + +function runHtmlSerializationSuite(argv) { + const values = valuesForArguments(argv, htmlLegacyFlags); + const outputDirectory = resolve(repositoryRoot, values['--output']); + const legacyArguments = argumentsForFlags(values, legacyFlags); + let coreCompleted = false; + + try { + const coreStdout = runBoundedNode( + coreRunnerPath, + legacyArguments, + 'Benchmark suite internal runner failed.', + ); + coreCompleted = true; + const manifest = parseCoreManifest(coreStdout); + const samplesPath = resolve( + outputDirectory, + 'html-serialization', + 'samples.json', + ); + const summaryDirectory = resolve( + outputDirectory, + 'html-serialization', + 'summary', + ); + + runBoundedNode( + markdownMeasurementPath, + [ + ...htmlSerializationEvidenceArguments( + values, + values['--module'], + values['--artifact-sha256'], + ), + '--output', + samplesPath, + ], + 'Benchmark suite HTML serialization measurement failed.', + ); + runBoundedNode( + sampleSummaryPath, + ['--input', samplesPath, '--output', summaryDirectory], + 'Benchmark suite HTML serialization summary failed.', + ); + + process.stdout.write( + `${JSON.stringify({ + ...manifest, + htmlSerializationSamples: 'html-serialization/samples.json', + htmlSerializationSummaryJson: + 'html-serialization/summary/summary.json', + htmlSerializationSummaryText: + 'html-serialization/summary/summary.txt', + })}\n`, + ); + } catch (error) { + if (coreCompleted) { + rmSync(outputDirectory, { recursive: true, force: true }); + } + throw error; + } +} + +function runPackedHtmlSerializationSuite(argv) { + const values = valuesForArguments(argv, packedHtmlFlags); + const outputDirectory = resolve(repositoryRoot, values['--output']); + const coreArguments = argumentsForFlags(values, packedFlags); + const snapshotted = snapshotPackedArguments(coreArguments); + let coreCompleted = false; + + try { + const coreStdout = runCoreNodePreservingError(snapshotted.argv); + coreCompleted = true; + const manifest = parseCoreManifest(coreStdout); + const snapshotValues = valuesForArguments(snapshotted.argv, packedFlags); + const snapshotTarballPath = snapshotValues['--package-tarball']; + const markdownModuleBytes = readPackedMarkdownModule(snapshotTarballPath); + const markdownModulePath = join( + snapshotted.temporaryDirectory, + 'cwl-markdown.mjs', + ); + try { + writeFileSync(markdownModulePath, markdownModuleBytes, { mode: 0o600 }); + } catch { + throw new Error('Benchmark suite packed Markdown module could not be prepared.'); + } + const markdownArtifactSha256 = createHash('sha256') + .update(markdownModuleBytes) + .digest('hex'); + const samplesPath = resolve( + outputDirectory, + 'html-serialization', + 'samples.json', + ); + const summaryDirectory = resolve( + outputDirectory, + 'html-serialization', + 'summary', + ); + + runBoundedNode( + markdownMeasurementPath, + [ + ...htmlSerializationEvidenceArguments( + values, + markdownModulePath, + markdownArtifactSha256, + ), + '--output', + samplesPath, + ], + 'Benchmark suite HTML serialization measurement failed.', + ); + runBoundedNode( + sampleSummaryPath, + ['--input', samplesPath, '--output', summaryDirectory], + 'Benchmark suite HTML serialization summary failed.', + ); + assertPackedSnapshotDigest( + snapshotTarballPath, + values['--package-sha256'], + ); + + process.stdout.write( + `${JSON.stringify({ + ...manifest, + htmlSerializationSamples: 'html-serialization/samples.json', + htmlSerializationSummaryJson: + 'html-serialization/summary/summary.json', + htmlSerializationSummaryText: + 'html-serialization/summary/summary.txt', + })}\n`, + ); + } catch (error) { + if (coreCompleted) { + rmSync(outputDirectory, { recursive: true, force: true }); + } + throw error; + } finally { + if (snapshotted.temporaryDirectory !== null) { + rmSync(snapshotted.temporaryDirectory, { + recursive: true, + force: true, + }); + } + } +} + +function runExistingSuite(argv) { + const snapshotted = snapshotPackedArguments(argv); + + try { + const result = spawnSync( + process.execPath, + [coreRunnerPath, ...snapshotted.argv], + { + cwd: repositoryRoot, + stdio: 'inherit', + timeout: 600_000, + }, + ); + + if (result.error !== undefined || result.signal !== null) { + throw new Error('Benchmark suite internal runner could not complete.'); + } + + process.exitCode = result.status ?? 1; + } finally { + if (snapshotted.temporaryDirectory !== null) { + rmSync(snapshotted.temporaryDirectory, { + recursive: true, + force: true, + }); + } + } +} + +function main(argv) { + assertCleanSourceCheckout(repositoryRoot); + assertFreshOutputDirectory(argv); + const expectedLegacySourceCommitSha = claimedLegacySourceCommitSha(argv); + if (expectedLegacySourceCommitSha !== undefined) { + assertCleanSourceCheckout(repositoryRoot, expectedLegacySourceCommitSha); + } + if (matchesArguments(argv, htmlLegacyFlags)) { + runHtmlSerializationSuite(argv); + return; + } + if (matchesArguments(argv, packedHtmlFlags)) { + runPackedHtmlSerializationSuite(argv); + return; + } + runExistingSuite(argv); +} + +try { + main(process.argv.slice(2)); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Benchmark suite failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} diff --git a/benchmarks/source-checkout-provenance.mjs b/benchmarks/source-checkout-provenance.mjs new file mode 100644 index 00000000..c1cfdf79 --- /dev/null +++ b/benchmarks/source-checkout-provenance.mjs @@ -0,0 +1,66 @@ +import { spawnSync } from 'node:child_process'; + +const MAX_STATUS_BYTES = 1024 * 1024; + +function checkedOutHeadSha(repositoryRoot) { + const result = spawnSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: MAX_STATUS_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }); + + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + typeof result.stdout !== 'string' + ) { + throw new Error( + 'Benchmark suite source checkout identity could not be verified.', + ); + } + + return result.stdout.trim(); +} + +export function assertCleanSourceCheckout(repositoryRoot, expectedSourceCommitSha) { + const result = spawnSync( + 'git', + ['status', '--porcelain=v1', '--untracked-files=all'], + { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: MAX_STATUS_BYTES, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + if ( + result.error !== undefined || + result.signal !== null || + result.status !== 0 || + typeof result.stdout !== 'string' + ) { + throw new Error( + 'Benchmark suite source checkout cleanliness could not be verified.', + ); + } + + if (result.stdout.length !== 0) { + throw new Error( + 'Benchmark suite source checkout must be clean before acquisition evidence is recorded.', + ); + } + + if ( + expectedSourceCommitSha !== undefined && + checkedOutHeadSha(repositoryRoot) !== expectedSourceCommitSha + ) { + throw new Error( + 'Benchmark suite source commit does not match checked-out HEAD.', + ); + } +} diff --git a/benchmarks/summarize-samples.mjs b/benchmarks/summarize-samples.mjs new file mode 100644 index 00000000..b98d2b87 --- /dev/null +++ b/benchmarks/summarize-samples.mjs @@ -0,0 +1,372 @@ +import { + closeSync, + constants, + existsSync, + fstatSync, + lstatSync, + mkdirSync, + openSync, + readSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { dirname, resolve } from 'node:path'; + +const MAX_INPUT_BYTES = 16 * 1024 * 1024; +const READ_CHUNK_BYTES = 64 * 1024; +const MAX_SAMPLES = 1_000_000; +const READ_ONLY_NONBLOCKING_NOFOLLOW = + constants.O_RDONLY | + (constants.O_NONBLOCK ?? 0) | + (constants.O_NOFOLLOW ?? 0); +const BENCHMARK_ID_PATTERN = + /^(?:ssr-shell-render|client-hydration|editor-mount|first-editable-paint|editor-input|keyboard-input|ime-composition|toolbar-action|undo-redo|table-edit|paste|image-insertion|markdown-serialization|html-serialization|envelope-parse|envelope-canonicalization|revision-evidence|transition-evidence|autosave-enqueue|autosave-coalescing|autosave-commit|yjs-update|print-media|office-parse|office-render|office-publication)-(?:small|medium|large|stress)$/u; +const UNITS = new Set(['ms', 'bytes']); +const SHA1_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const RUNTIME_ID_PATTERN = + /^(?:node|python|chromium|firefox|webkit|playwright)-[0-9]+(?:\.[0-9]+){1,3}$/u; +const REFERENCE_HARDWARE_ID_PATTERN = + /^(?:github-actions-(?:ubuntu|windows|macos)-[0-9]+(?:\.[0-9]+){0,2}-(?:x64|arm64)|refhw-sha256-[0-9a-f]{64})$/u; +const DOCUMENT_PROFILES = new Set(['small', 'medium', 'large', 'stress']); +const BENCHMARK_INPUT_KEYS = new Set([ + 'contractVersion', + 'benchmarkId', + 'unit', + 'sourceCommitSha', + 'artifactSha256', + 'documentProfile', + 'runtimeId', + 'referenceHardwareId', + 'samples', +]); +const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true }); + +function resolveArguments(argv) { + if ( + argv.length !== 4 || + argv[0] !== '--input' || + argv[1].length === 0 || + argv[2] !== '--output' || + argv[3].length === 0 + ) { + throw new Error( + 'Usage: node benchmarks/summarize-samples.mjs --input --output ', + ); + } + return Object.freeze({ + inputPath: resolve(argv[1]), + outputDirectory: resolve(argv[3]), + }); +} + +function inspectSampleInputPath(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark sample input must be a regular file.'); + } +} + +function readBoundedJson(path) { + const pathMetadata = inspectSampleInputPath(path); + if (pathMetadata?.isSymbolicLink()) { + throw new Error( + 'Benchmark sample input must be a regular non-symlink file.', + ); + } + if (pathMetadata === undefined || !pathMetadata.isFile()) { + throw new Error('Benchmark sample input must be a regular file.'); + } + + const descriptor = openSync(path, READ_ONLY_NONBLOCKING_NOFOLLOW); + try { + const metadata = fstatSync(descriptor); + if (!metadata.isFile()) { + throw new Error('Benchmark sample input must be a regular file.'); + } + if (metadata.size > MAX_INPUT_BYTES) { + throw new Error('Benchmark sample input exceeds the supported size.'); + } + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= MAX_INPUT_BYTES) { + const remainingBudget = MAX_INPUT_BYTES + 1 - totalBytes; + const chunk = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, remainingBudget), + ); + const bytesRead = readSync( + descriptor, + chunk, + 0, + chunk.byteLength, + null, + ); + if (bytesRead === 0) break; + totalBytes += bytesRead; + if (totalBytes > MAX_INPUT_BYTES) { + throw new Error('Benchmark sample input exceeds the supported size.'); + } + chunks.push(chunk.subarray(0, bytesRead)); + } + + const bytes = Buffer.concat(chunks, totalBytes); + let text; + try { + text = UTF8_DECODER.decode(bytes); + } catch { + throw new Error('Benchmark sample input must be valid UTF-8 JSON.'); + } + let parsed; + try { + parsed = JSON.parse(text); + } catch { + throw new Error('Benchmark sample input must be valid JSON.'); + } + return parsed; + } finally { + closeSync(descriptor); + } +} + +function validateInput(value) { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Benchmark sample input must be an object.'); + } + if (Object.keys(value).some((key) => !BENCHMARK_INPUT_KEYS.has(key))) { + throw new Error('Benchmark sample input contains unsupported fields.'); + } + if (value.contractVersion !== 1) { + throw new Error('Benchmark sample contractVersion must be 1.'); + } + if ( + typeof value.benchmarkId !== 'string' || + !BENCHMARK_ID_PATTERN.test(value.benchmarkId) + ) { + throw new Error('Benchmark benchmarkId is invalid.'); + } + if (typeof value.unit !== 'string' || !UNITS.has(value.unit)) { + throw new Error('Benchmark unit is invalid.'); + } + if ( + typeof value.sourceCommitSha !== 'string' || + !SHA1_PATTERN.test(value.sourceCommitSha) + ) { + throw new Error( + 'Benchmark sourceCommitSha must be a lowercase 40-character commit SHA.', + ); + } + if ( + typeof value.artifactSha256 !== 'string' || + !SHA256_PATTERN.test(value.artifactSha256) + ) { + throw new Error( + 'Benchmark artifactSha256 must be a lowercase 64-character SHA-256 digest.', + ); + } + if ( + typeof value.documentProfile !== 'string' || + !DOCUMENT_PROFILES.has(value.documentProfile) + ) { + throw new Error('Benchmark documentProfile is invalid.'); + } + if (!value.benchmarkId.endsWith(`-${value.documentProfile}`)) { + throw new Error('Benchmark sample profile must match documentProfile.'); + } + if ( + typeof value.runtimeId !== 'string' || + !RUNTIME_ID_PATTERN.test(value.runtimeId) + ) { + throw new Error('Benchmark runtimeId is invalid.'); + } + if ( + typeof value.referenceHardwareId !== 'string' || + !REFERENCE_HARDWARE_ID_PATTERN.test(value.referenceHardwareId) + ) { + throw new Error('Benchmark referenceHardwareId is invalid.'); + } + if ( + !Array.isArray(value.samples) || + value.samples.length === 0 || + value.samples.length > MAX_SAMPLES + ) { + throw new Error('Benchmark samples must be a non-empty bounded array.'); + } + if ( + value.samples.some( + (sample) => + typeof sample !== 'number' || !Number.isFinite(sample) || sample < 0, + ) + ) { + throw new Error('Benchmark samples must be finite non-negative numbers.'); + } + return Object.freeze({ + benchmarkId: value.benchmarkId, + unit: value.unit, + sourceCommitSha: value.sourceCommitSha, + artifactSha256: value.artifactSha256, + documentProfile: value.documentProfile, + runtimeId: value.runtimeId, + referenceHardwareId: value.referenceHardwareId, + samples: Object.freeze([...value.samples]), + }); +} + +function nearestRank(sorted, percentile) { + const index = Math.ceil(percentile * sorted.length) - 1; + return sorted[index]; +} + +function summarize(input) { + const sorted = [...input.samples].sort((left, right) => left - right); + return Object.freeze({ + contractVersion: 1, + benchmarkId: input.benchmarkId, + unit: input.unit, + sourceCommitSha: input.sourceCommitSha, + artifactSha256: input.artifactSha256, + documentProfile: input.documentProfile, + runtimeId: input.runtimeId, + referenceHardwareId: input.referenceHardwareId, + sampleCount: sorted.length, + percentileMethod: 'nearest-rank', + minimum: sorted[0], + p50: nearestRank(sorted, 0.5), + p75: nearestRank(sorted, 0.75), + p95: nearestRank(sorted, 0.95), + maximum: sorted.at(-1), + }); +} + +function formatSummary(summary) { + return [ + `benchmark=${summary.benchmarkId}`, + `unit=${summary.unit}`, + `source_commit_sha=${summary.sourceCommitSha}`, + `artifact_sha256=${summary.artifactSha256}`, + `document_profile=${summary.documentProfile}`, + `runtime_id=${summary.runtimeId}`, + `reference_hardware_id=${summary.referenceHardwareId}`, + `samples=${summary.sampleCount}`, + `percentile_method=${summary.percentileMethod}`, + `minimum=${summary.minimum}`, + `p50=${summary.p50}`, + `p75=${summary.p75}`, + `p95=${summary.p95}`, + `maximum=${summary.maximum}`, + '', + ].join('\n'); +} + +function refersToSameFile(leftPath, rightPath) { + if (!existsSync(leftPath) || !existsSync(rightPath)) return false; + const left = statSync(leftPath); + const right = statSync(rightPath); + return left.dev === right.dev && left.ino === right.ino; +} + +function assertRegularOutputDestination(path) { + const metadata = lstatSync(path, { throwIfNoEntry: false }); + if (metadata !== undefined && !metadata.isFile()) { + throw new Error('Benchmark summary output paths must be regular files.'); + } +} + +function assertSingleLinkOutputDestination(path) { + const metadata = lstatSync(path, { throwIfNoEntry: false }); + if (metadata !== undefined && metadata.nlink !== 1) { + throw new Error( + 'Benchmark summary output paths must not be multiply linked.', + ); + } +} + +function inspectOutputDirectoryComponent(path) { + try { + return lstatSync(path, { throwIfNoEntry: false }); + } catch { + throw new Error('Benchmark summary output directory could not be prepared.'); + } +} + +function assertNoSymlinkDirectoryComponents(path) { + let current = path; + while (true) { + const metadata = inspectOutputDirectoryComponent(current); + if (metadata?.isSymbolicLink()) { + throw new Error( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + } + const parent = dirname(current); + if (parent === current) return; + current = parent; + } +} + +function prepareOutputDirectory(path) { + assertNoSymlinkDirectoryComponents(path); + const current = inspectOutputDirectoryComponent(path); + if (current !== undefined) { + if (!current.isDirectory()) { + throw new Error( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + } + return; + } + + try { + mkdirSync(path, { recursive: true }); + } catch { + throw new Error('Benchmark summary output directory could not be prepared.'); + } + + assertNoSymlinkDirectoryComponents(path); + const created = inspectOutputDirectoryComponent(path); + if (created === undefined || !created.isDirectory()) { + throw new Error( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + } +} + +function main() { + const { inputPath, outputDirectory } = resolveArguments(process.argv.slice(2)); + const summaryJsonPath = resolve(outputDirectory, 'summary.json'); + const summaryTextPath = resolve(outputDirectory, 'summary.txt'); + prepareOutputDirectory(outputDirectory); + if ( + inputPath === summaryJsonPath || + inputPath === summaryTextPath || + refersToSameFile(inputPath, summaryJsonPath) || + refersToSameFile(inputPath, summaryTextPath) + ) { + throw new Error('Benchmark output must not overwrite the sample input.'); + } + assertRegularOutputDestination(summaryJsonPath); + assertRegularOutputDestination(summaryTextPath); + if (refersToSameFile(summaryJsonPath, summaryTextPath)) { + throw new Error('Benchmark summary outputs must be distinct files.'); + } + assertSingleLinkOutputDestination(summaryJsonPath); + assertSingleLinkOutputDestination(summaryTextPath); + const input = validateInput(readBoundedJson(inputPath)); + const summary = summarize(input); + writeFileSync( + summaryJsonPath, + `${JSON.stringify(summary, null, 2)}\n`, + 'utf8', + ); + writeFileSync(summaryTextPath, formatSummary(summary), 'utf8'); +} + +try { + main(); +} catch (error) { + const message = + error instanceof Error ? error.message : 'Benchmark summary failed.'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +} diff --git a/office/benchmarks/measure_render.py b/office/benchmarks/measure_render.py new file mode 100644 index 00000000..a2a62e76 --- /dev/null +++ b/office/benchmarks/measure_render.py @@ -0,0 +1,337 @@ +"""Produce privacy-safe timing and peak-RSS evidence for canonical Office fixtures.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import platform +import re +import stat +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +OFFICE_ROOT = Path(__file__).resolve().parents[1] +REPOSITORY_ROOT = OFFICE_ROOT.parent +LOCK_PATH = REPOSITORY_ROOT / "benchmarks" / "office-fixtures.lock.json" +SOURCE_ROOT = OFFICE_ROOT / "src" +if str(SOURCE_ROOT) not in sys.path: + sys.path.insert(0, str(SOURCE_ROOT)) + +from inkspan_office.safe_renderer import write_office_document # noqa: E402 + +MAX_ITERATIONS = 100 +MAX_TOKEN_CODE_UNITS = 128 +SAMPLE_TIMEOUT_SECONDS = 120 +TOKEN_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:-]*\Z") +SHA256_PATTERN = re.compile(r"[0-9a-f]{64}\Z") +GIT_SHA_PATTERN = re.compile(r"[0-9a-f]{40}\Z") +SUPPORTED_FORMATS = {"docx", "xlsx", "pptx"} + + +class BenchmarkContractError(Exception): + """Raised when benchmark evidence cannot be produced safely and truthfully.""" + + +def _metadata_token(value: str, label: str) -> str: + if ( + not isinstance(value, str) + or len(value) > MAX_TOKEN_CODE_UNITS + or not TOKEN_PATTERN.fullmatch(value) + ): + raise BenchmarkContractError(f"{label} must be a bounded metadata token") + return value + + +def _positive_iterations(value: int) -> int: + if not isinstance(value, int) or isinstance(value, bool) or not 1 <= value <= MAX_ITERATIONS: + raise BenchmarkContractError("iterations must be between 1 and 100") + return value + + +def _load_fixture_contract(format_name: str, profile: str) -> tuple[int, str]: + try: + lock = json.loads(LOCK_PATH.read_text(encoding="utf-8")) + if lock.get("contractVersion") != 1 or lock.get("synthetic") is not True: + raise BenchmarkContractError("canonical Office fixture lock is invalid") + record = lock["formats"][format_name][profile] + expected_bytes = record["bytes"] + expected_sha256 = record["sha256"] + except (OSError, KeyError, TypeError, json.JSONDecodeError) as exc: + raise BenchmarkContractError("canonical Office fixture lock is invalid") from exc + if ( + not isinstance(expected_bytes, int) + or isinstance(expected_bytes, bool) + or expected_bytes <= 0 + or not isinstance(expected_sha256, str) + or not SHA256_PATTERN.fullmatch(expected_sha256) + ): + raise BenchmarkContractError("canonical Office fixture lock is invalid") + return expected_bytes, expected_sha256 + + +def _read_exact_regular_fixture(path: Path, expected_bytes: int, expected_sha256: str) -> bytes: + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise BenchmarkContractError("Office benchmark input could not be opened safely") from exc + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size != expected_bytes: + raise BenchmarkContractError( + "input does not match the canonical synthetic Office fixture" + ) + chunks: list[bytes] = [] + remaining = expected_bytes + while remaining: + chunk = os.read(descriptor, min(remaining, 1024 * 1024)) + if not chunk: + raise BenchmarkContractError( + "input does not match the canonical synthetic Office fixture" + ) + chunks.append(chunk) + remaining -= len(chunk) + if os.read(descriptor, 1): + raise BenchmarkContractError( + "input does not match the canonical synthetic Office fixture" + ) + payload = b"".join(chunks) + except OSError as exc: + raise BenchmarkContractError("Office benchmark input could not be read safely") from exc + finally: + os.close(descriptor) + if hashlib.sha256(payload).hexdigest() != expected_sha256: + raise BenchmarkContractError("input does not match the canonical synthetic Office fixture") + return payload + + +def _read_exact_stdin_fixture(expected_bytes: int, expected_sha256: str) -> bytes: + payload = sys.stdin.buffer.read(expected_bytes + 1) + if len(payload) != expected_bytes or hashlib.sha256(payload).hexdigest() != expected_sha256: + raise BenchmarkContractError("input does not match the canonical synthetic Office fixture") + return payload + + +def _source_sha() -> str: + status = subprocess.run( + ["git", "-C", str(REPOSITORY_ROOT), "status", "--porcelain", "--untracked-files=all"], + check=False, + capture_output=True, + text=True, + ) + if status.returncode != 0 or status.stdout: + raise BenchmarkContractError("benchmark checkout must be clean") + revision = subprocess.run( + ["git", "-C", str(REPOSITORY_ROOT), "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + ) + sha = revision.stdout.strip() + if revision.returncode != 0 or not GIT_SHA_PATTERN.fullmatch(sha): + raise BenchmarkContractError("benchmark source revision could not be verified") + return sha + + +def _peak_rss_bytes() -> int: + try: + import resource + except ImportError as exc: # pragma: no cover - benchmark CI is POSIX + raise BenchmarkContractError("peak RSS measurement is unavailable on this runtime") from exc + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + if peak <= 0: + raise BenchmarkContractError("peak RSS measurement is unavailable on this runtime") + if sys.platform == "darwin": + return int(peak) + return int(peak) * 1024 + + +def _child_measure(format_name: str, profile: str) -> int: + try: + profile = _metadata_token(profile, "fixture profile") + expected_bytes, expected_sha256 = _load_fixture_contract(format_name, profile) + payload_bytes = _read_exact_stdin_fixture(expected_bytes, expected_sha256) + payload = json.loads(payload_bytes.decode("utf-8")) + if not isinstance(payload, dict): + raise BenchmarkContractError("canonical Office fixture must contain an object") + if payload.get("format") != format_name: + raise BenchmarkContractError("canonical Office fixture format is inconsistent") + with tempfile.TemporaryDirectory(prefix="inkspan-office-benchmark-") as directory: + output_path = Path(directory) / f"render.{format_name}" + started = time.perf_counter_ns() + write_office_document(payload, output_path) + duration_ms = (time.perf_counter_ns() - started) / 1_000_000 + peak_rss_bytes = _peak_rss_bytes() + print( + json.dumps( + { + "format": format_name, + "durationMs": round(duration_ms, 6), + "peakRssBytes": peak_rss_bytes, + }, + separators=(",", ":"), + sort_keys=True, + ) + ) + return 0 + except BenchmarkContractError as exc: + print(str(exc), file=sys.stderr) + return 2 + except Exception: + print("Office benchmark render sample failed.", file=sys.stderr) + return 2 + + +def _run_sample(payload_bytes: bytes, format_name: str, profile: str) -> dict[str, Any]: + try: + completed = subprocess.run( + [ + sys.executable, + str(Path(__file__).resolve()), + "--child", + "--format", + format_name, + "--fixture-profile", + profile, + ], + input=payload_bytes, + check=False, + cwd=REPOSITORY_ROOT, + capture_output=True, + timeout=SAMPLE_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired: + raise BenchmarkContractError("Office benchmark render sample timed out") from None + if completed.returncode != 0: + raise BenchmarkContractError("Office benchmark render sample failed") + try: + sample = json.loads(completed.stdout.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise BenchmarkContractError("Office benchmark render sample was invalid") from exc + if not isinstance(sample, dict): + raise BenchmarkContractError("Office benchmark render sample was invalid") + duration = sample.get("durationMs") + peak_rss = sample.get("peakRssBytes") + observed_format = sample.get("format") + if ( + observed_format != format_name + or not isinstance(duration, (int, float)) + or isinstance(duration, bool) + or not math.isfinite(duration) + or duration < 0 + or not isinstance(peak_rss, int) + or isinstance(peak_rss, bool) + or peak_rss <= 0 + ): + raise BenchmarkContractError("Office benchmark render sample was invalid") + return {"format": observed_format, "durationMs": duration, "peakRssBytes": peak_rss} + + +def _percentile(values: list[float], quantile: float) -> float: + ordered = sorted(values) + if len(ordered) == 1: + return ordered[0] + position = (len(ordered) - 1) * quantile + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + weight = position - lower + return ordered[lower] * (1 - weight) + ordered[upper] * weight + + +def _summary(values: list[float], *, integral: bool) -> dict[str, int | float]: + result: dict[str, int | float] = {} + for name, quantile in (("p50", 0.50), ("p75", 0.75), ("p95", 0.95)): + value = _percentile(values, quantile) + result[name] = int(round(value)) if integral else round(value, 6) + maximum = max(values) + result["max"] = int(maximum) if integral else round(maximum, 6) + return result + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Measure a canonical synthetic Inkspan Office fixture") + parser.add_argument("--input", required=True, help="canonical generated Office fixture path") + parser.add_argument("--format", required=True, choices=sorted(SUPPORTED_FORMATS)) + parser.add_argument("--fixture-profile", required=True) + parser.add_argument("--iterations", type=int, default=5) + parser.add_argument("--reference-hardware", required=True) + return parser + + +def _child_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Render one verified synthetic Inkspan Office fixture") + parser.add_argument("--format", required=True, choices=sorted(SUPPORTED_FORMATS)) + parser.add_argument("--fixture-profile", required=True) + return parser + + +def _main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + iterations = _positive_iterations(args.iterations) + profile = _metadata_token(args.fixture_profile, "fixture profile") + reference_hardware = _metadata_token(args.reference_hardware, "reference hardware") + expected_bytes, expected_sha256 = _load_fixture_contract(args.format, profile) + source_sha = _source_sha() + payload_bytes = _read_exact_regular_fixture( + Path(args.input), expected_bytes, expected_sha256 + ) + samples = [_run_sample(payload_bytes, args.format, profile) for _ in range(iterations)] + observed_source_sha = _source_sha() + if observed_source_sha != source_sha: + raise BenchmarkContractError("benchmark source revision changed during measurement") + duration_values = [float(sample["durationMs"]) for sample in samples] + rss_values = [float(sample["peakRssBytes"]) for sample in samples] + evidence = { + "contractVersion": 1, + "synthetic": True, + "operation": "office_render", + "fixtureId": f"{args.format}.{profile}", + "format": args.format, + "profile": profile, + "iterations": iterations, + "referenceHardware": reference_hardware, + "fixtureBytes": expected_bytes, + "fixtureSha256": expected_sha256, + "sourceSha": source_sha, + "runtime": { + "implementation": platform.python_implementation(), + "python": platform.python_version(), + "platform": sys.platform, + }, + "samples": samples, + "summary": { + "durationMs": _summary(duration_values, integral=False), + "peakRssBytes": _summary(rss_values, integral=True), + }, + } + print(json.dumps(evidence, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + except BenchmarkContractError as exc: + print(str(exc), file=sys.stderr) + return 2 + except Exception: + print("Office benchmark measurement failed.", file=sys.stderr) + return 2 + + +def main() -> int: + """Run the benchmark command or its isolated one-render child process.""" + + if sys.argv[1:2] == ["--child"]: + args = _child_parser().parse_args(sys.argv[2:]) + return _child_measure(args.format, args.fixture_profile) + return _main() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/office/tests/test_performance_measurement.py b/office/tests/test_performance_measurement.py new file mode 100644 index 00000000..b9e6c88f --- /dev/null +++ b/office/tests/test_performance_measurement.py @@ -0,0 +1,249 @@ +"""Contract tests for privacy-safe Office render performance evidence.""" + +from __future__ import annotations + +import json +import runpy +import subprocess +import sys +from pathlib import Path + +import pytest + + +MULTILINGUAL_PARAGRAPH = ( + "English: deterministic Office rendering fixture. 한국어: 합성 성능 문서입니다. " + "日本語: 合成性能文書です。 中文: 这是合成性能文档。 " + "Tiếng Việt: Đây là tài liệu hiệu năng tổng hợp." +) + + +def _docx_page(page_number: int) -> list[dict[str, object]]: + page = str(page_number).zfill(3) + return [ + {"type": "heading", "level": 1, "text": f"Synthetic page {page}"}, + { + "type": "paragraph", + "text": f"{MULTILINGUAL_PARAGRAPH} Page {page}.", + "alignment": "justify", + }, + { + "type": "rich_paragraph", + "runs": [ + {"text": f"Page {page} summary: ", "bold": True}, + {"text": "deterministic ", "italic": True}, + {"text": "Office rendering fixture.", "underline": True}, + ], + }, + { + "type": "bullet_list", + "ordered": False, + "items": [ + f"page {page} item A", + f"page {page} item B", + f"page {page} item C", + ], + }, + { + "type": "table", + "headers": ["Page", "Metric", "Value"], + "rows": [ + [page, "latency-sample", page_number], + [page, "memory-sample", page_number * 2], + [page, "revision-sample", page_number * 3], + [page, "render-sample", page_number * 4], + ], + }, + ] + + +def _canonical_docx_small_fixture_bytes() -> bytes: + blocks: list[dict[str, object]] = [] + for page_number in range(1, 3): + blocks.extend(_docx_page(page_number)) + if page_number < 2: + blocks.append({"type": "page_break"}) + request = { + "format": "docx", + "title": "Inkspan synthetic DOCX benchmark: small", + "author": "Inkspan synthetic benchmark", + "subject": "Deterministic synthetic performance fixture", + "blocks": blocks, + } + return (json.dumps(request, ensure_ascii=False, indent=2) + "\n").encode() + + +def _measure_command(script: Path, request_path: Path, iterations: str) -> list[str]: + return [ + sys.executable, + str(script), + "--input", + str(request_path), + "--format", + "docx", + "--fixture-profile", + "small", + "--iterations", + iterations, + "--reference-hardware", + "pytest-reference", + ] + + +def test_measure_office_render_records_duration_peak_rss_and_provenance(tmp_path: Path) -> None: + """Measure a lock-verified synthetic fixture without copying its document body into evidence.""" + + request_path = tmp_path / "synthetic-docx-small.json" + request_bytes = _canonical_docx_small_fixture_bytes() + request_path.write_bytes(request_bytes) + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + completed = subprocess.run( + _measure_command(script, request_path, "2"), + check=False, + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + text=True, + ) + + assert completed.returncode == 0, completed.stderr + evidence = json.loads(completed.stdout) + assert evidence["contractVersion"] == 1 + assert evidence["synthetic"] is True + assert evidence["operation"] == "office_render" + assert evidence["fixtureId"] == "docx.small" + assert evidence["profile"] == "small" + assert evidence["format"] == "docx" + assert evidence["iterations"] == 2 + assert evidence["referenceHardware"] == "pytest-reference" + assert evidence["fixtureBytes"] == len(request_bytes) + assert len(evidence["fixtureSha256"]) == 64 + assert len(evidence["sourceSha"]) == 40 + assert evidence["runtime"]["implementation"] + assert evidence["runtime"]["python"] + assert evidence["runtime"]["platform"] + assert len(evidence["samples"]) == 2 + for sample in evidence["samples"]: + assert isinstance(sample["durationMs"], (int, float)) + assert sample["durationMs"] >= 0 + assert isinstance(sample["peakRssBytes"], int) + assert sample["peakRssBytes"] > 0 + for percentile in ("p50", "p75", "p95", "max"): + assert evidence["summary"]["durationMs"][percentile] >= 0 + assert evidence["summary"]["peakRssBytes"][percentile] > 0 + + combined_output = completed.stdout + completed.stderr + assert MULTILINGUAL_PARAGRAPH not in combined_output + assert str(request_path) not in combined_output + + +def test_measure_office_render_rejects_noncanonical_content_without_leaking_it( + tmp_path: Path, +) -> None: + """Do not label arbitrary document content as canonical synthetic benchmark evidence.""" + + sentinel = "PRIVATE-NONCANONICAL-DOCUMENT-SENTINEL" + request_path = tmp_path / "private-customer-name.json" + request_path.write_text( + json.dumps( + { + "format": "docx", + "title": "private customer document", + "blocks": [{"type": "paragraph", "text": sentinel}], + } + ), + encoding="utf-8", + ) + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + completed = subprocess.run( + _measure_command(script, request_path, "1"), + check=False, + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + text=True, + ) + + assert completed.returncode != 0 + assert "input does not match the canonical synthetic Office fixture" in completed.stderr + assert sentinel not in completed.stderr + assert str(request_path) not in completed.stderr + + +def test_measure_office_render_rejects_unbounded_iteration_counts_without_reading_input( + tmp_path: Path, +) -> None: + """Reject impossible benchmark work before inspecting the caller-selected request path.""" + + request_path = tmp_path / "must-not-be-read.json" + request_path.write_text("PRIVATE-ITERATION-SENTINEL", encoding="utf-8") + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + completed = subprocess.run( + _measure_command(script, request_path, "1001"), + check=False, + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + text=True, + ) + + assert completed.returncode != 0 + assert "iterations must be between 1 and 100" in completed.stderr + assert str(request_path) not in completed.stderr + assert "PRIVATE-ITERATION-SENTINEL" not in completed.stderr + + +def test_office_render_sample_times_out_without_leaking_payload( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Bound a hung renderer child and normalize timeout failure without exposing document data.""" + + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + namespace = runpy.run_path(str(script), run_name="inkspan_measure_render_test") + benchmark_error = namespace["BenchmarkContractError"] + run_sample = namespace["_run_sample"] + observed_timeout: list[float | None] = [] + sentinel = b'PRIVATE-HUNG-RENDER-SENTINEL' + + def _timeout(*args: object, **kwargs: object) -> subprocess.CompletedProcess[bytes]: + timeout = kwargs.get("timeout") + observed_timeout.append(timeout if isinstance(timeout, (int, float)) else None) + raise subprocess.TimeoutExpired(args[0] if args else "renderer", timeout or 0) + + monkeypatch.setattr(subprocess, "run", _timeout) + + with pytest.raises(benchmark_error, match="Office benchmark render sample timed out") as exc_info: + run_sample(sentinel, "docx", "small") + + assert observed_timeout == [120] + assert sentinel.decode() not in str(exc_info.value) + + +def test_office_render_child_rejects_unverified_private_payload() -> None: + """Require the isolated child to re-verify canonical fixture identity before rendering.""" + + sentinel = "PRIVATE-DIRECT-CHILD-DOCUMENT-SENTINEL" + payload = json.dumps( + { + "format": "docx", + "title": "private direct child document", + "blocks": [{"type": "paragraph", "text": sentinel}], + } + ).encode() + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + completed = subprocess.run( + [ + sys.executable, + str(script), + "--child", + "--format", + "docx", + "--fixture-profile", + "small", + ], + input=payload, + check=False, + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + ) + + assert completed.returncode != 0 + assert b"input does not match the canonical synthetic Office fixture" in completed.stderr + assert sentinel.encode() not in completed.stderr diff --git a/office/tests/test_performance_source_stability.py b/office/tests/test_performance_source_stability.py new file mode 100644 index 00000000..0c51cb6e --- /dev/null +++ b/office/tests/test_performance_source_stability.py @@ -0,0 +1,176 @@ +"""Source-stability contracts for Office benchmark provenance.""" + +from __future__ import annotations + +import json +import runpy +import subprocess +import sys +from pathlib import Path + +import pytest + + +MULTILINGUAL_PARAGRAPH = ( + "English: deterministic Office rendering fixture. 한국어: 합성 성능 문서입니다. " + "日本語: 合成性能文書です。 中文: 这是合成性能文档。 " + "Tiếng Việt: Đây là tài liệu hiệu năng tổng hợp." +) + + +def _docx_page(page_number: int) -> list[dict[str, object]]: + page = str(page_number).zfill(3) + return [ + {"type": "heading", "level": 1, "text": f"Synthetic page {page}"}, + { + "type": "paragraph", + "text": f"{MULTILINGUAL_PARAGRAPH} Page {page}.", + "alignment": "justify", + }, + { + "type": "rich_paragraph", + "runs": [ + {"text": f"Page {page} summary: ", "bold": True}, + {"text": "deterministic ", "italic": True}, + {"text": "Office rendering fixture.", "underline": True}, + ], + }, + { + "type": "bullet_list", + "ordered": False, + "items": [ + f"page {page} item A", + f"page {page} item B", + f"page {page} item C", + ], + }, + { + "type": "table", + "headers": ["Page", "Metric", "Value"], + "rows": [ + [page, "latency-sample", page_number], + [page, "memory-sample", page_number * 2], + [page, "revision-sample", page_number * 3], + [page, "render-sample", page_number * 4], + ], + }, + ] + + +def _canonical_docx_small_fixture_bytes() -> bytes: + blocks: list[dict[str, object]] = [] + for page_number in range(1, 3): + blocks.extend(_docx_page(page_number)) + if page_number < 2: + blocks.append({"type": "page_break"}) + request = { + "format": "docx", + "title": "Inkspan synthetic DOCX benchmark: small", + "author": "Inkspan synthetic benchmark", + "subject": "Deterministic synthetic performance fixture", + "blocks": blocks, + } + return (json.dumps(request, ensure_ascii=False, indent=2) + "\n").encode() + + +def _git(repository: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + cwd=repository, + check=True, + capture_output=True, + text=True, + ) + + +def _initialize_clean_repository(repository: Path) -> Path: + repository.mkdir() + _git(repository, "init", "-q") + tracked = repository / "tracked.txt" + tracked.write_text("before\n", encoding="utf-8") + _git(repository, "add", "tracked.txt") + subprocess.run( + [ + "git", + "-c", + "user.name=Inkspan benchmark test", + "-c", + "user.email=benchmark-test@example.invalid", + "commit", + "-qm", + "initial", + ], + cwd=repository, + check=True, + capture_output=True, + text=True, + ) + return tracked + + +def test_office_measurement_rejects_clean_source_revision_move_during_sampling( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Never emit benchmark evidence if a clean checkout advances while samples are acquired.""" + + script = Path(__file__).resolve().parents[1] / "benchmarks" / "measure_render.py" + namespace = runpy.run_path(str(script), run_name="inkspan_measure_render_source_stability") + main = namespace["_main"] + globals_ = main.__globals__ + + repository = tmp_path / "isolated-repository" + tracked = _initialize_clean_repository(repository) + globals_["REPOSITORY_ROOT"] = repository + + request_path = tmp_path / "synthetic-docx-small.json" + request_path.write_bytes(_canonical_docx_small_fixture_bytes()) + + def _move_revision( + _payload_bytes: bytes, + format_name: str, + _profile: str, + ) -> dict[str, object]: + tracked.write_text("after\n", encoding="utf-8") + _git(repository, "add", "tracked.txt") + subprocess.run( + [ + "git", + "-c", + "user.name=Inkspan benchmark test", + "-c", + "user.email=benchmark-test@example.invalid", + "commit", + "-qm", + "advance", + ], + cwd=repository, + check=True, + capture_output=True, + text=True, + ) + return {"format": format_name, "durationMs": 1.0, "peakRssBytes": 1024} + + globals_["_run_sample"] = _move_revision + + result = main( + [ + "--input", + str(request_path), + "--format", + "docx", + "--fixture-profile", + "small", + "--iterations", + "1", + "--reference-hardware", + "pytest-reference", + ] + ) + output = capsys.readouterr() + + assert result == 2 + assert output.out == "" + assert "benchmark source revision changed during measurement" in output.err + assert str(repository) not in output.err + assert str(request_path) not in output.err diff --git a/src/demoBundleChunking.test.ts b/src/demoBundleChunking.test.ts new file mode 100644 index 00000000..d6097e45 --- /dev/null +++ b/src/demoBundleChunking.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest'; +import { demoVendorChunk } from '../vite.demo.chunking'; + +describe('demo bundle chunking contract', () => { + it('keeps major editor dependency families in deterministic vendor chunks', () => { + const pnpmPrefix = '/repo/node_modules/.pnpm/example/node_modules/'; + + expect(demoVendorChunk(`${pnpmPrefix}react-dom/client.js`)).toBe( + 'react-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}@tiptap/pm/state/index.js`)).toBe( + 'prosemirror-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}prosemirror-state/dist/index.js`)).toBe( + 'prosemirror-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}@tiptap/core/dist/index.js`)).toBe( + 'tiptap-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}marked/lib/marked.esm.js`)).toBe( + 'serialization-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}turndown/lib/turndown.es.js`)).toBe( + 'serialization-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}yjs/dist/yjs.mjs`)).toBe( + 'collaboration-vendor', + ); + expect(demoVendorChunk(`${pnpmPrefix}lodash-es/lodash.js`)).toBe('vendor'); + }); + + it('leaves application modules to Rollup and normalizes Windows paths', () => { + expect(demoVendorChunk('/repo/demo/App.tsx')).toBeUndefined(); + expect(demoVendorChunk(String.raw`C:\repo\node_modules\react\index.js`)).toBe( + 'react-vendor', + ); + }); +}); diff --git a/src/performanceBenchmarkExistingOutputPathPrivacyContract.test.ts b/src/performanceBenchmarkExistingOutputPathPrivacyContract.test.ts new file mode 100644 index 00000000..0555b3db --- /dev/null +++ b/src/performanceBenchmarkExistingOutputPathPrivacyContract.test.ts @@ -0,0 +1,69 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +const cases = [ + { + name: 'Markdown', + script: 'benchmarks/measure-markdown.mjs', + expectedError: 'Markdown benchmark input must be a regular non-symlink file.', + }, + { + name: 'revision', + script: 'benchmarks/measure-revision-evidence.mjs', + expectedError: 'Revision benchmark input must be a regular non-symlink file.', + }, +] as const; + +describe('benchmark existing-output path privacy contract', () => { + for (const testCase of cases) { + it(`redacts ${testCase.name} input paths before existing-output alias checks`, () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-alias-path-')); + const privateSentinel = `tenant-private-${testCase.name.toLowerCase()}-input-parent`; + const blockedParent = join(root, privateSentinel); + const input = join(blockedParent, 'document-input'); + const output = join(root, 'samples.json'); + + try { + writeFileSync(blockedParent, 'not a directory', 'utf8'); + writeFileSync(output, '{}\n', 'utf8'); + + const result = spawnSync( + process.execPath, + [ + resolve(process.cwd(), testCase.script), + '--input', + input, + '--module', + join(root, 'unused-module.mjs'), + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'b'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe(testCase.expectedError); + expect(result.stderr).not.toContain(privateSentinel); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + } +}); diff --git a/src/performanceCorpusContract.test.ts b/src/performanceCorpusContract.test.ts new file mode 100644 index 00000000..02bc5c02 --- /dev/null +++ b/src/performanceCorpusContract.test.ts @@ -0,0 +1,111 @@ +import { execFileSync } from 'node:child_process'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +interface BenchmarkProfileLock { + readonly sections: number; + readonly bytes: number; + readonly sha256: string; +} + +interface BenchmarkCorpusLock { + readonly contractVersion: 1; + readonly synthetic: true; + readonly scripts: readonly [ + 'English', + 'Korean', + 'Japanese', + 'Chinese', + 'Vietnamese', + 'mixed', + ]; + readonly profiles: Readonly< + Record<'small' | 'medium' | 'large' | 'stress', BenchmarkProfileLock> + >; +} + +function runGenerator(outputDirectory: string): BenchmarkCorpusLock { + const script = resolve(process.cwd(), 'benchmarks/generate-corpus.mjs'); + execFileSync(process.execPath, [script, '--output', outputDirectory], { + cwd: process.cwd(), + stdio: ['ignore', 'pipe', 'pipe'], + }); + return JSON.parse( + readFileSync(join(outputDirectory, 'manifest.json'), 'utf8'), + ) as BenchmarkCorpusLock; +} + +describe('deterministic synthetic performance corpus', () => { + it('reproduces the committed corpus lock exactly across independent runs', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-corpus-')); + const first = join(root, 'first'); + const second = join(root, 'second'); + try { + const expected = JSON.parse( + readFileSync( + resolve(process.cwd(), 'benchmarks/corpus.lock.json'), + 'utf8', + ), + ) as BenchmarkCorpusLock; + const firstManifest = runGenerator(first); + const secondManifest = runGenerator(second); + + expect(firstManifest).toEqual(expected); + expect(secondManifest).toEqual(expected); + expect(firstManifest.synthetic).toBe(true); + expect(firstManifest.scripts).toEqual([ + 'English', + 'Korean', + 'Japanese', + 'Chinese', + 'Vietnamese', + 'mixed', + ]); + + for (const profile of ['small', 'medium', 'large', 'stress'] as const) { + const firstBytes = readFileSync(join(first, `${profile}.md`)); + const secondBytes = readFileSync(join(second, `${profile}.md`)); + expect(firstBytes.equals(secondBytes)).toBe(true); + expect(firstBytes.byteLength).toBe(expected.profiles[profile].bytes); + } + + const smallBody = readFileSync(join(first, 'small.md'), 'utf8'); + expect(smallBody).toContain( + 'Mixed-script: Inkspan review 검증은 日本語と中文 그리고 Tiếng Việt를 한 문단에서 deterministic하게 다룹니다.', + ); + for (const dimensions of ['1x1', '16x16', '64x64']) { + expect(smallBody).toContain(`![synthetic raster ${dimensions} 0001](`); + } + expect( + new Set(smallBody.match(/data:image\/png;base64,[A-Za-z0-9+/=]+/g)).size, + ).toBe(3); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed instead of following a corpus output symlink', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-corpus-symlink-')); + const outputDirectory = join(root, 'output'); + const victimPath = join(root, 'victim.md'); + try { + mkdirSync(outputDirectory, { recursive: true }); + writeFileSync(victimPath, 'buyer-owned evidence\n', 'utf8'); + symlinkSync(victimPath, join(outputDirectory, 'small.md')); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readFileSync(victimPath, 'utf8')).toBe('buyer-owned evidence\n'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceHtmlSerializationMeasurement.test.ts b/src/performanceHtmlSerializationMeasurement.test.ts new file mode 100644 index 00000000..7eaff65e --- /dev/null +++ b/src/performanceHtmlSerializationMeasurement.test.ts @@ -0,0 +1,103 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-markdown.mjs', +); +const SOURCE_COMMIT_SHA = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: process.cwd(), + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const RUNTIME_ID = `node-${process.versions.node}`; +const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function sha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +describe('HTML serialization performance measurement', () => { + it('measures packed htmlToMarkdown without falling back to markdownToHtml', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-html-serialization-')); + const input = join(root, 'small.html'); + const modulePath = join(root, 'packed-markdown.mjs'); + const output = join(root, 'samples.json'); + + try { + writeFileSync(input, '

Synthetic benchmark fixture

\n', 'utf8'); + writeFileSync( + modulePath, + [ + "export function markdownToHtml() { throw new Error('wrong serialization direction'); }", + "export function htmlToMarkdown(source) { return source.replace(/<[^>]+>/gu, '').trim(); }", + '', + ].join('\n'), + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--operation', + 'html-to-markdown', + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + sha256(modulePath), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + REFERENCE_HARDWARE_ID, + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + + const evidence = JSON.parse(readFileSync(output, 'utf8')) as { + benchmarkId: string; + unit: string; + documentProfile: string; + samples: unknown[]; + }; + expect(evidence.benchmarkId).toBe('html-serialization-small'); + expect(evidence.unit).toBe('ms'); + expect(evidence.documentProfile).toBe('small'); + expect(evidence.samples).toHaveLength(2); + expect( + evidence.samples.every( + (sample) => + typeof sample === 'number' && Number.isFinite(sample) && sample >= 0, + ), + ).toBe(true); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceHtmlSerializationSuiteContract.test.ts b/src/performanceHtmlSerializationSuiteContract.test.ts new file mode 100644 index 00000000..98dd9389 --- /dev/null +++ b/src/performanceHtmlSerializationSuiteContract.test.ts @@ -0,0 +1,150 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const currentSourceCommitSha = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const currentRuntimeId = `node-${process.versions.node}`; + +function sha256(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +describe('single-command HTML serialization benchmark contract', () => { + it('measures HTML-to-Markdown serialization alongside Markdown and revision evidence', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-html-suite-')); + const markdownInput = join(directory, 'document.md'); + const htmlInput = join(directory, 'document.html'); + const markdownModule = join(directory, 'markdown.mjs'); + const revisionInput = join(directory, 'document-envelope.json'); + const revisionModule = join(directory, 'revision.mjs'); + const outputDirectory = join(directory, 'evidence'); + const markdownModuleSource = [ + "export function markdownToHtml(source) { return `

${source}

`; }", + "export function htmlToMarkdown(source) { return source.replace(/<[^>]+>/gu, '').trim(); }", + '', + ].join('\n'); + const revisionModuleSource = `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'c'.repeat(64)}' } }; }\n`; + + try { + writeFileSync(markdownInput, '# Buyer benchmark\n', 'utf8'); + writeFileSync(htmlInput, '

Buyer benchmark

\n', 'utf8'); + writeFileSync(markdownModule, markdownModuleSource, 'utf8'); + writeFileSync( + revisionInput, + '{"contractVersion":1,"mode":"markdown","document":"# Buyer benchmark"}\n', + 'utf8', + ); + writeFileSync(revisionModule, revisionModuleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + suitePath, + '--input', + markdownInput, + '--html-input', + htmlInput, + '--module', + markdownModule, + '--revision-input', + revisionInput, + '--revision-module', + revisionModule, + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + currentSourceCommitSha, + '--artifact-sha256', + sha256(markdownModuleSource), + '--revision-artifact-sha256', + sha256(revisionModuleSource), + '--runtime-id', + currentRuntimeId, + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + outputDirectory, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + + const manifest = JSON.parse(result.stdout.trim()) as Record; + expect(manifest).toMatchObject({ + htmlSerializationSamples: 'html-serialization/samples.json', + htmlSerializationSummaryJson: + 'html-serialization/summary/summary.json', + htmlSerializationSummaryText: + 'html-serialization/summary/summary.txt', + status: 'completed', + }); + + const samples = JSON.parse( + readFileSync( + join(outputDirectory, 'html-serialization', 'samples.json'), + 'utf8', + ), + ) as { + benchmarkId?: unknown; + documentProfile?: unknown; + samples?: unknown[]; + }; + expect(samples.benchmarkId).toBe('html-serialization-small'); + expect(samples.documentProfile).toBe('small'); + expect(samples.samples).toHaveLength(2); + + const summary = JSON.parse( + readFileSync( + join( + outputDirectory, + 'html-serialization', + 'summary', + 'summary.json', + ), + 'utf8', + ), + ) as { benchmarkId?: unknown }; + expect(summary.benchmarkId).toBe('html-serialization-small'); + expect( + readFileSync( + join( + outputDirectory, + 'html-serialization', + 'summary', + 'summary.txt', + ), + 'utf8', + ), + ).toContain('html-serialization-small'); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); +}); \ No newline at end of file diff --git a/src/performanceMarkdownMeasurementContract.test.ts b/src/performanceMarkdownMeasurementContract.test.ts new file mode 100644 index 00000000..0ec5fdc2 --- /dev/null +++ b/src/performanceMarkdownMeasurementContract.test.ts @@ -0,0 +1,255 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +interface BenchmarkSamples { + readonly contractVersion: 1; + readonly benchmarkId: string; + readonly unit: 'ms'; + readonly sourceCommitSha: string; + readonly artifactSha256: string; + readonly documentProfile: 'small' | 'medium' | 'large' | 'stress'; + readonly runtimeId: string; + readonly referenceHardwareId: string; + readonly samples: number[]; +} + +const repositoryRoot = process.cwd(); +const measurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-markdown.mjs', +); +const summaryScript = resolve(repositoryRoot, 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const FALLBACK_ARTIFACT_SHA256 = 'b'.repeat(64); +const RUNTIME_ID = `node-${process.versions.node}`; +const HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function fileSha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +function measurementArguments( + input: string, + modulePath: string, + output: string, + artifactSha256 = existsSync(modulePath) + ? fileSha256(modulePath) + : FALLBACK_ARTIFACT_SHA256, +): string[] { + return [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '3', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + artifactSha256, + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + HARDWARE_ID, + '--output', + output, + ]; +} + +describe('Markdown runtime measurement contract', () => { + it('writes bounded privacy-safe samples consumable by the canonical summarizer', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-')); + const input = join(root, 'large.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const samplesPath = join(root, 'samples.json'); + const summaryDirectory = join(root, 'summary'); + try { + writeFileSync(input, '# Buyer benchmark fixture\n\nSynthetic content only.\n', 'utf8'); + writeFileSync( + modulePath, + "export function markdownToHtml(source) { return `

${source.length}

`; }\n", + 'utf8', + ); + const artifactSha256 = fileSha256(modulePath); + + execFileSync( + process.execPath, + measurementArguments(input, modulePath, samplesPath, artifactSha256), + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, + ); + + const samples = JSON.parse( + readFileSync(samplesPath, 'utf8'), + ) as BenchmarkSamples; + expect(samples).toMatchObject({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256, + documentProfile: 'large', + runtimeId: RUNTIME_ID, + referenceHardwareId: HARDWARE_ID, + }); + expect(samples.samples).toHaveLength(3); + expect( + samples.samples.every( + (sample) => Number.isFinite(sample) && sample >= 0, + ), + ).toBe(true); + expect(readFileSync(samplesPath, 'utf8')).not.toContain('Buyer benchmark fixture'); + expect(readFileSync(samplesPath, 'utf8')).not.toContain('Synthetic content only'); + + execFileSync( + process.execPath, + [summaryScript, '--input', samplesPath, '--output', summaryDirectory], + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, + ); + const summary = JSON.parse( + readFileSync(join(summaryDirectory, 'summary.json'), 'utf8'), + ) as { sampleCount: number; benchmarkId: string; unit: string }; + expect(summary).toMatchObject({ + sampleCount: 3, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before output when the measured module lacks the public serializer', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-export-')); + const input = join(root, 'small.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic\n', 'utf8'); + writeFileSync(modulePath, 'export const other = true;\n', 'utf8'); + + const result = spawnSync( + process.execPath, + measurementArguments(input, modulePath, samplesPath), + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module must export markdownToHtml().', + ); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects symlinked document inputs before reading benchmark content', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-symlink-')); + const realInput = join(root, 'real.md'); + const input = join(root, 'linked.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeFileSync(realInput, '# Synthetic\n', 'utf8'); + symlinkSync(realInput, input); + writeFileSync( + modulePath, + 'export function markdownToHtml(source) { return source; }\n', + 'utf8', + ); + + const result = spawnSync( + process.execPath, + measurementArguments(input, modulePath, samplesPath), + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark input must be a regular non-symlink file.', + ); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects an output path that aliases the measured module', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-output-')); + const input = join(root, 'small.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const moduleSource = + 'export function markdownToHtml(source) { return `

${source}

`; }\n'; + try { + writeFileSync(input, '# Synthetic\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + const result = spawnSync( + process.execPath, + measurementArguments(input, modulePath, modulePath), + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark output must not overwrite the measured module.', + ); + expect(readFileSync(modulePath, 'utf8')).toBe(moduleSource); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('requires a file-backed module URL rather than network or package-name resolution', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-measurement-module-')); + const input = join(root, 'small.md'); + const samplesPath = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic\n', 'utf8'); + const result = spawnSync( + process.execPath, + measurementArguments( + input, + 'https://example.invalid/markdown.mjs', + samplesPath, + ), + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module must be a local regular file.', + ); + expect(existsSync(samplesPath)).toBe(false); + expect(() => pathToFileURL(input)).not.toThrow(); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts new file mode 100644 index 00000000..3e172ebb --- /dev/null +++ b/src/performanceMarkdownMeasurementErrorPrivacyContract.test.ts @@ -0,0 +1,151 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + existsSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); +const SOURCE_COMMIT_SHA = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: process.cwd(), + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const RUNTIME_ID = `node-${process.versions.node}`; +const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function sha256(value: string) { + return createHash('sha256').update(value, 'utf8').digest('hex'); +} + +function runMeasurement( + moduleSource: string, + outputForRoot: (root: string) => string = (root) => join(root, 'samples.json'), +) { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-error-privacy-')); + const input = join(root, 'document.md'); + const modulePath = join(root, 'measured.mjs'); + const output = outputForRoot(root); + writeFileSync(input, '# Public benchmark fixture\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + REFERENCE_HARDWARE_ID, + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + return { root, output, result }; +} + +describe('Markdown measurement error privacy contract', () => { + it('redacts exceptions raised while loading the measured module', () => { + const privateSentinel = 'private-import-sentinel-must-not-leak'; + const moduleSource = `throw new Error('${privateSentinel}');\nexport function markdownToHtml(value) { return value; }\n`; + const { root, output, result } = runMeasurement(moduleSource); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module could not be loaded.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts exceptions raised by the measured serializer', () => { + const privateSentinel = 'private-serializer-sentinel-must-not-leak'; + const moduleSource = `export function markdownToHtml() { throw new Error('${privateSentinel}'); }\n`; + const { root, output, result } = runMeasurement(moduleSource); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured markdownToHtml() execution failed.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts filesystem details when output path traversal fails', () => { + const privateSentinel = 'private-output-sentinel-must-not-leak'; + const moduleSource = + 'export function markdownToHtml(value) { return value; }\n'; + const { root, output, result } = runMeasurement(moduleSource, (testRoot) => { + const blockedParent = join(testRoot, privateSentinel); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + return join(blockedParent, 'samples.json'); + }); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark output path could not be inspected.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts filesystem details when output publication cannot create its directory', () => { + const privateSentinel = `private-markdown-publication-${process.pid}`; + const moduleSource = + 'export function markdownToHtml(value) { return value; }\n'; + const blockedOutput = join('/sys', privateSentinel, 'samples.json'); + const { root, output, result } = runMeasurement( + moduleSource, + () => blockedOutput, + ); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark output could not be written.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMarkdownMeasurementInputPrivacyContract.test.ts b/src/performanceMarkdownMeasurementInputPrivacyContract.test.ts new file mode 100644 index 00000000..748e94a1 --- /dev/null +++ b/src/performanceMarkdownMeasurementInputPrivacyContract.test.ts @@ -0,0 +1,72 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const RUNTIME_ID = 'node-22.18.0'; +const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function sha256(value: string) { + return createHash('sha256').update(value, 'utf8').digest('hex'); +} + +describe('Markdown measurement input error privacy contract', () => { + it('redacts private filesystem details when input traversal fails', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-input-privacy-')); + const privateSentinel = 'private-input-sentinel-must-not-leak'; + const blockedParent = join(root, privateSentinel); + const input = join(blockedParent, 'document.md'); + const modulePath = join(root, 'measured.mjs'); + const output = join(root, 'samples.json'); + const moduleSource = + 'export function markdownToHtml(value) { return value; }\n'; + + writeFileSync(blockedParent, 'not a directory', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + REFERENCE_HARDWARE_ID, + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark input must be a regular non-symlink file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMarkdownMeasurementModulePathPrivacyContract.test.ts b/src/performanceMarkdownMeasurementModulePathPrivacyContract.test.ts new file mode 100644 index 00000000..352e7f4d --- /dev/null +++ b/src/performanceMarkdownMeasurementModulePathPrivacyContract.test.ts @@ -0,0 +1,65 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const RUNTIME_ID = 'node-22.18.0'; +const REFERENCE_HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +describe('Markdown measurement module path error privacy contract', () => { + it('redacts private filesystem details when module path traversal fails', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-module-privacy-')); + const privateSentinel = 'private-module-sentinel-must-not-leak'; + const input = join(root, 'document.md'); + const blockedParent = join(root, privateSentinel); + const modulePath = join(blockedParent, 'measured.mjs'); + const output = join(root, 'samples.json'); + + writeFileSync(input, '# bounded\n', 'utf8'); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + '0'.repeat(64), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + REFERENCE_HARDWARE_ID, + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module must be a local regular file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMarkdownMeasurementOutputAncestorSymlink.test.ts b/src/performanceMarkdownMeasurementOutputAncestorSymlink.test.ts new file mode 100644 index 00000000..cc60c3cb --- /dev/null +++ b/src/performanceMarkdownMeasurementOutputAncestorSymlink.test.ts @@ -0,0 +1,76 @@ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); + +function sha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +describe('Markdown benchmark output path ancestry', () => { + it('fails closed before writing beneath a symlinked output ancestor', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-output-ancestor-')); + const input = join(root, 'document.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const outside = join(root, 'outside-target'); + const alias = join(root, 'aliased-parent'); + const output = join(alias, 'nested-output', 'samples.json'); + try { + writeFileSync(input, '# Synthetic\n', 'utf8'); + writeFileSync( + modulePath, + 'export function markdownToHtml(source) { return `

${source.length}

`; }\n', + 'utf8', + ); + mkdirSync(outside); + symlinkSync(outside, alias, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + sha256(modulePath), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark output directory must be a non-symlink directory.', + ); + expect(existsSync(join(outside, 'nested-output', 'samples.json'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMarkdownMeasurementProvenanceContract.test.ts b/src/performanceMarkdownMeasurementProvenanceContract.test.ts new file mode 100644 index 00000000..dea6bb44 --- /dev/null +++ b/src/performanceMarkdownMeasurementProvenanceContract.test.ts @@ -0,0 +1,62 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-markdown.mjs', +); + +describe('Markdown measurement artifact provenance', () => { + it('rejects caller metadata that does not match the exact measured module bytes', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-provenance-')); + const input = join(root, 'large.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const output = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic benchmark fixture\n', 'utf8'); + writeFileSync( + modulePath, + 'export function markdownToHtml(source) { return `

${source}

`; }\n', + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'f'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Markdown benchmark artifact digest does not match the measured module.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMarkdownModuleUrlContract.test.ts b/src/performanceMarkdownModuleUrlContract.test.ts new file mode 100644 index 00000000..af022d47 --- /dev/null +++ b/src/performanceMarkdownModuleUrlContract.test.ts @@ -0,0 +1,69 @@ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-markdown.mjs', +); + +describe('Markdown measurement module URL authority', () => { + it('rejects file URLs with a non-local host before loading the measured module', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-module-url-')); + const input = join(root, 'large.md'); + const modulePath = join(root, 'packed-markdown.mjs'); + const output = join(root, 'samples.json'); + try { + writeFileSync(input, '# Synthetic benchmark fixture\n', 'utf8'); + writeFileSync( + modulePath, + 'export function markdownToHtml(source) { return `

${source}

`; }\n', + 'utf8', + ); + const artifactSha256 = createHash('sha256') + .update(readFileSync(modulePath)) + .digest('hex'); + const nonLocalFileUrl = pathToFileURL(modulePath); + nonLocalFileUrl.hostname = 'example.invalid'; + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + nonLocalFileUrl.href, + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + artifactSha256, + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured Markdown module must be a local regular file.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMeasurementOutputContract.test.ts b/src/performanceMeasurementOutputContract.test.ts new file mode 100644 index 00000000..895732ad --- /dev/null +++ b/src/performanceMeasurementOutputContract.test.ts @@ -0,0 +1,199 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + linkSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +function writeValidInput(path: string): void { + writeFileSync( + path, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [1, 2, 3], + })}\n`, + 'utf8', + ); +} + +describe('benchmark summary output contract', () => { + it('rejects an invalid second destination before publishing the first summary', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-output-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const summaryJson = join(output, 'summary.json'); + const summaryText = join(output, 'summary.txt'); + try { + writeValidInput(input); + mkdirSync(summaryText, { recursive: true }); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output paths must be regular files.', + ); + expect(existsSync(summaryJson)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects a dangling summary symlink before it can create the symlink target', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-output-symlink-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const summaryJson = join(output, 'summary.json'); + const escapedTarget = join(root, 'escaped-summary.json'); + try { + writeValidInput(input); + mkdirSync(output, { recursive: true }); + symlinkSync(escapedTarget, summaryJson); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output paths must be regular files.', + ); + expect(existsSync(escapedTarget)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects two output paths that alias the same regular file before publication', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-output-alias-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const summaryJson = join(output, 'summary.json'); + const summaryText = join(output, 'summary.txt'); + try { + writeValidInput(input); + mkdirSync(output, { recursive: true }); + writeFileSync(summaryJson, 'sentinel', 'utf8'); + linkSync(summaryJson, summaryText); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary outputs must be distinct files.', + ); + expect(readFileSync(summaryJson, 'utf8')).toBe('sentinel'); + expect(readFileSync(summaryText, 'utf8')).toBe('sentinel'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed on a named-pipe sample input instead of blocking before regular-file validation', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-input-fifo-')); + const input = join(root, 'samples.pipe'); + const output = join(root, 'output'); + try { + const mkfifo = spawnSync('mkfifo', [input], { encoding: 'utf8' }); + expect(mkfifo.status).toBe(0); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + timeout: 1000, + }, + ); + + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input must be a regular file.', + ); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts filesystem details when the output directory cannot be prepared', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-output-privacy-')); + const input = join(root, 'samples.json'); + const privateSentinel = 'private-summary-output-sentinel-must-not-leak'; + const blockedParent = join(root, privateSentinel); + const output = join(blockedParent, 'output'); + try { + writeValidInput(input); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output directory could not be prepared.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMeasurementPrivacyContract.test.ts b/src/performanceMeasurementPrivacyContract.test.ts new file mode 100644 index 00000000..9f994c5e --- /dev/null +++ b/src/performanceMeasurementPrivacyContract.test.ts @@ -0,0 +1,96 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +const validInput = { + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [10, 20, 30], +} as const; + +function runSummary(inputValue: object) { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-privacy-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + writeFileSync(input, `${JSON.stringify(inputValue)}\n`, 'utf8'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + return { root, output, result }; +} + +describe('benchmark evidence privacy contract', () => { + it('rejects unsupported metadata instead of accepting arbitrary evidence payloads', () => { + const { root, output, result } = runSummary({ + ...validInput, + prompt: 'must-not-enter-benchmark-evidence', + }); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input contains unsupported fields.', + ); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it.each([ + [ + 'benchmarkId', + 'tenant-acme-large', + 'Benchmark benchmarkId is invalid.', + ], + ['unit', 'tenant-acme', 'Benchmark unit is invalid.'], + ['runtimeId', 'tenant-acme', 'Benchmark runtimeId is invalid.'], + [ + 'referenceHardwareId', + 'tenant-acme', + 'Benchmark referenceHardwareId is invalid.', + ], + ])( + 'rejects caller-controlled %s values that could launder private identifiers into evidence', + (field, value, expectedError) => { + const { root, output, result } = runSummary({ + ...validInput, + [field]: value, + }); + try { + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe(expectedError); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/src/performanceMeasurementProducerOutputHardlink.test.ts b/src/performanceMeasurementProducerOutputHardlink.test.ts new file mode 100644 index 00000000..ffc4d283 --- /dev/null +++ b/src/performanceMeasurementProducerOutputHardlink.test.ts @@ -0,0 +1,183 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + linkSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const markdownScript = resolve(process.cwd(), 'benchmarks/measure-markdown.mjs'); +const revisionScript = resolve( + process.cwd(), + 'benchmarks/measure-revision-evidence.mjs', +); +const sourceCommitSha = execFileSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: process.cwd(), + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], +}).trim(); +const runtimeId = `node-${process.versions.node}`; +const referenceHardwareId = 'github-actions-ubuntu-24.04-x64'; + +function sha256(content: string): string { + return createHash('sha256').update(content).digest('hex'); +} + +function commonArguments( + input: string, + module: string, + moduleSha256: string, + output: string, +): string[] { + return [ + '--input', + input, + '--module', + module, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + sourceCommitSha, + '--artifact-sha256', + moduleSha256, + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ]; +} + +function expectOutputPreservedFailure( + script: string, + args: string[], + sentinel: string, + expectedMessage: string, +): void { + const originalSentinel = readFileSync(sentinel, 'utf8'); + const result = spawnSync(process.execPath, [script, ...args], { + cwd: process.cwd(), + encoding: 'utf8', + }); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe(expectedMessage); + expect(readFileSync(sentinel, 'utf8')).toBe(originalSentinel); +} + +describe('benchmark producer output immutability', () => { + it('fails closed before Markdown measurement overwrites an unrelated hard link', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-hardlink-')); + const input = join(root, 'document.md'); + const module = join(root, 'markdown-module.mjs'); + const output = join(root, 'samples.json'); + const sentinel = join(root, 'buyer-owned.txt'); + const moduleSource = 'export const markdownToHtml = (source) => `

${source}

`;\n'; + + try { + writeFileSync(input, '# Hello\n', 'utf8'); + writeFileSync(module, moduleSource, 'utf8'); + writeFileSync(sentinel, 'buyer-owned-content\n', 'utf8'); + linkSync(sentinel, output); + + expectOutputPreservedFailure( + markdownScript, + commonArguments(input, module, sha256(moduleSource), output), + sentinel, + 'Markdown benchmark output must not be multiply linked.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before revision measurement overwrites an unrelated hard link', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-hardlink-')); + const input = join(root, 'document-envelope.json'); + const module = join(root, 'revision-module.mjs'); + const output = join(root, 'samples.json'); + const sentinel = join(root, 'buyer-owned.txt'); + const moduleSource = [ + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) {', + " return { revision: { digestHex: String(source.byteLength).padStart(64, '0') } };", + '}', + '', + ].join('\n'); + + try { + writeFileSync(input, '{"contractVersion":1}\n', 'utf8'); + writeFileSync(module, moduleSource, 'utf8'); + writeFileSync(sentinel, 'buyer-owned-content\n', 'utf8'); + linkSync(sentinel, output); + + expectOutputPreservedFailure( + revisionScript, + commonArguments(input, module, sha256(moduleSource), output), + sentinel, + 'Revision benchmark output must not be multiply linked.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before Markdown measurement overwrites existing evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-markdown-existing-output-')); + const input = join(root, 'document.md'); + const module = join(root, 'markdown-module.mjs'); + const output = join(root, 'samples.json'); + const moduleSource = 'export const markdownToHtml = (source) => `

${source}

`;\n'; + + try { + writeFileSync(input, '# Hello\n', 'utf8'); + writeFileSync(module, moduleSource, 'utf8'); + writeFileSync(output, '{"status":"accepted"}\n', 'utf8'); + + expectOutputPreservedFailure( + markdownScript, + commonArguments(input, module, sha256(moduleSource), output), + output, + 'Markdown benchmark output must not already exist.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before revision measurement overwrites existing evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-existing-output-')); + const input = join(root, 'document-envelope.json'); + const module = join(root, 'revision-module.mjs'); + const output = join(root, 'samples.json'); + const moduleSource = [ + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) {', + " return { revision: { digestHex: String(source.byteLength).padStart(64, '0') } };", + '}', + '', + ].join('\n'); + + try { + writeFileSync(input, '{"contractVersion":1}\n', 'utf8'); + writeFileSync(module, moduleSource, 'utf8'); + writeFileSync(output, '{"status":"accepted"}\n', 'utf8'); + + expectOutputPreservedFailure( + revisionScript, + commonArguments(input, module, sha256(moduleSource), output), + output, + 'Revision benchmark output must not already exist.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMeasurementProducerSourceProvenanceContract.test.ts b/src/performanceMeasurementProducerSourceProvenanceContract.test.ts new file mode 100644 index 00000000..865d993b --- /dev/null +++ b/src/performanceMeasurementProducerSourceProvenanceContract.test.ts @@ -0,0 +1,180 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const markdownMeasurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-markdown.mjs', +); +const revisionMeasurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-revision-evidence.mjs', +); +const currentSourceCommitSha = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const mismatchedSourceCommitSha = + currentSourceCommitSha === 'f'.repeat(40) ? 'e'.repeat(40) : 'f'.repeat(40); +const activeRuntimeId = `node-${process.versions.node}`; +const mismatchedRuntimeId = + activeRuntimeId === 'node-99.99.99' ? 'node-98.98.98' : 'node-99.99.99'; +const referenceHardwareId = `refhw-sha256-${'b'.repeat(64)}`; + +function sha256(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +function markdownInvocation( + root: string, + sourceCommitSha: string, + runtimeId: string, +) { + const input = join(root, 'document.md'); + const modulePath = join(root, 'markdown.mjs'); + const output = join(root, 'markdown-samples.json'); + const moduleSource = + 'export function markdownToHtml(source) { return `

${source}

`; }\n'; + writeFileSync(input, '# Provenance fixture\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + markdownMeasurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + sourceCommitSha, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return { result, output }; +} + +function revisionInvocation( + root: string, + sourceCommitSha: string, + runtimeId: string, +) { + const input = join(root, 'document-envelope.json'); + const modulePath = join(root, 'revision.mjs'); + const output = join(root, 'revision-samples.json'); + const moduleSource = `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'c'.repeat(64)}' } }; }\n`; + writeFileSync( + input, + '{"contractVersion":1,"mode":"markdown","document":"# Provenance fixture"}\n', + 'utf8', + ); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + revisionMeasurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + sourceCommitSha, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return { result, output }; +} + +describe('direct benchmark producer source/runtime provenance', () => { + it.each([ + ['Markdown', markdownInvocation], + ['revision', revisionInvocation], + ] as const)( + 'rejects a caller-supplied source SHA that is not the checked-out HEAD for %s evidence', + (_label, invoke) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-producer-source-')); + try { + const { result, output } = invoke( + root, + mismatchedSourceCommitSha, + activeRuntimeId, + ); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark measurement source commit does not match checked-out HEAD.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); + + it.each([ + ['Markdown', markdownInvocation], + ['revision', revisionInvocation], + ] as const)( + 'rejects a caller-supplied runtime ID that is not the active Node runtime for %s evidence', + (_label, invoke) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-producer-runtime-')); + try { + const { result, output } = invoke( + root, + currentSourceCommitSha, + mismatchedRuntimeId, + ); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark measurement runtime ID must match the active Node runtime.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/src/performanceMeasurementProducerSourceStabilityContract.test.ts b/src/performanceMeasurementProducerSourceStabilityContract.test.ts new file mode 100644 index 00000000..12cfdd9e --- /dev/null +++ b/src/performanceMeasurementProducerSourceStabilityContract.test.ts @@ -0,0 +1,169 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { delimiter, join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const markdownMeasurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-markdown.mjs', +); +const revisionMeasurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-revision-evidence.mjs', +); +const firstSourceSha = 'a'.repeat(40); +const movedSourceSha = 'b'.repeat(40); +const runtimeId = `node-${process.versions.node}`; +const referenceHardwareId = `refhw-sha256-${'c'.repeat(64)}`; + +function sha256(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +function movingGitEnvironment(root: string): NodeJS.ProcessEnv { + const fakeBin = join(root, 'fake-bin'); + const statePath = join(root, 'git-invocations.txt'); + const fakeGit = join(fakeBin, 'git'); + const script = `#!/usr/bin/env node +const { existsSync, readFileSync, writeFileSync } = require('node:fs'); +const state = process.env.INKSPAN_FAKE_GIT_STATE; +const first = process.env.INKSPAN_FAKE_GIT_FIRST_SHA; +const moved = process.env.INKSPAN_FAKE_GIT_MOVED_SHA; +const count = existsSync(state) ? Number(readFileSync(state, 'utf8')) : 0; +writeFileSync(state, String(count + 1), 'utf8'); +process.stdout.write(\`${'${count === 0 ? first : moved}'}\\n\`); +`; + mkdirSync(fakeBin, { recursive: true }); + writeFileSync(fakeGit, script, { encoding: 'utf8', mode: 0o755 }); + chmodSync(fakeGit, 0o755); + return { + ...process.env, + PATH: `${fakeBin}${delimiter}${process.env.PATH ?? ''}`, + INKSPAN_FAKE_GIT_STATE: statePath, + INKSPAN_FAKE_GIT_FIRST_SHA: firstSourceSha, + INKSPAN_FAKE_GIT_MOVED_SHA: movedSourceSha, + }; +} + +function markdownInvocation(root: string) { + const input = join(root, 'document.md'); + const modulePath = join(root, 'markdown.mjs'); + const output = join(root, 'markdown-samples.json'); + const moduleSource = + 'export function markdownToHtml(source) { return `

${source}

`; }\n'; + writeFileSync(input, '# Source movement fixture\n', 'utf8'); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + markdownMeasurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + firstSourceSha, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + env: movingGitEnvironment(root), + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return { output, result }; +} + +function revisionInvocation(root: string) { + const input = join(root, 'document-envelope.json'); + const modulePath = join(root, 'revision.mjs'); + const output = join(root, 'revision-samples.json'); + const moduleSource = `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'d'.repeat(64)}' } }; }\n`; + writeFileSync( + input, + '{"contractVersion":1,"mode":"markdown","document":"# Source movement fixture"}\n', + 'utf8', + ); + writeFileSync(modulePath, moduleSource, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + revisionMeasurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + firstSourceSha, + '--artifact-sha256', + sha256(moduleSource), + '--runtime-id', + runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + output, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + env: movingGitEnvironment(root), + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return { output, result }; +} + +describe.skipIf(process.platform === 'win32')( + 'direct benchmark producer source stability', + () => { + it.each([ + ['Markdown', markdownInvocation], + ['revision', revisionInvocation], + ] as const)( + 'rejects %s evidence when checked-out HEAD moves during sample acquisition', + (_label, invoke) => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-producer-source-move-')); + try { + const { output, result } = invoke(root); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark measurement source commit does not match checked-out HEAD.', + ); + expect(existsSync(output)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, + ); + }, +); diff --git a/src/performanceMeasurementStatisticsAncestorSymlink.test.ts b/src/performanceMeasurementStatisticsAncestorSymlink.test.ts new file mode 100644 index 00000000..aae380be --- /dev/null +++ b/src/performanceMeasurementStatisticsAncestorSymlink.test.ts @@ -0,0 +1,66 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +function writeInput(path: string): void { + writeFileSync( + path, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256: 'b'.repeat(64), + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [10, 20, 30], + })}\n`, + 'utf8', + ); +} + +describe('benchmark summary output path ancestry', () => { + it('fails closed before writing through a symlinked output ancestor', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-ancestor-')); + const input = join(root, 'samples.json'); + const outside = join(root, 'outside-target'); + const alias = join(root, 'aliased-parent'); + const output = join(alias, 'nested-output'); + try { + writeInput(input); + mkdirSync(outside); + symlinkSync(outside, alias, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + expect(existsSync(join(outside, 'nested-output', 'summary.json'))).toBe(false); + expect(existsSync(join(outside, 'nested-output', 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMeasurementStatisticsContract.test.ts b/src/performanceMeasurementStatisticsContract.test.ts new file mode 100644 index 00000000..7f7627b3 --- /dev/null +++ b/src/performanceMeasurementStatisticsContract.test.ts @@ -0,0 +1,325 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { + existsSync, + linkSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + truncateSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +interface BenchmarkSummary { + readonly contractVersion: 1; + readonly benchmarkId: string; + readonly unit: string; + readonly sourceCommitSha: string; + readonly artifactSha256: string; + readonly documentProfile: 'small' | 'medium' | 'large' | 'stress'; + readonly runtimeId: string; + readonly referenceHardwareId: string; + readonly sampleCount: number; + readonly percentileMethod: 'nearest-rank'; + readonly minimum: number; + readonly p50: number; + readonly p75: number; + readonly p95: number; + readonly maximum: number; +} + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +function writeInput(path: string, samples: readonly number[]): void { + writeFileSync( + path, + `${JSON.stringify( + { + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples, + }, + null, + 2, + )}\n`, + 'utf8', + ); +} + +function runSummary(inputPath: string, outputDirectory: string): BenchmarkSummary { + execFileSync( + process.execPath, + [script, '--input', inputPath, '--output', outputDirectory], + { + cwd: process.cwd(), + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + return JSON.parse( + readFileSync(join(outputDirectory, 'summary.json'), 'utf8'), + ) as BenchmarkSummary; +} + +describe('deterministic benchmark sample statistics', () => { + it('writes reproducible nearest-rank JSON and human-readable summaries with provenance metadata', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-')); + const input = join(root, 'samples.json'); + const first = join(root, 'first'); + const second = join(root, 'second'); + try { + writeInput(input, [20, 10, 40, 30, 50]); + + const firstSummary = runSummary(input, first); + const secondSummary = runSummary(input, second); + expect(firstSummary).toEqual(secondSummary); + expect(firstSummary).toEqual({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 5, + percentileMethod: 'nearest-rank', + minimum: 10, + p50: 30, + p75: 40, + p95: 50, + maximum: 50, + }); + + const expectedText = [ + 'benchmark=markdown-serialization-large', + 'unit=ms', + `source_commit_sha=${SOURCE_COMMIT_SHA}`, + `artifact_sha256=${ARTIFACT_SHA256}`, + 'document_profile=large', + 'runtime_id=node-22.18.0', + 'reference_hardware_id=github-actions-ubuntu-24.04-x64', + 'samples=5', + 'percentile_method=nearest-rank', + 'minimum=10', + 'p50=30', + 'p75=40', + 'p95=50', + 'maximum=50', + '', + ].join('\n'); + expect(readFileSync(join(first, 'summary.txt'), 'utf8')).toBe(expectedText); + expect(readFileSync(join(second, 'summary.txt'), 'utf8')).toBe(expectedText); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed when immutable provenance metadata is missing', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-metadata-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + writeFileSync( + input, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + samples: [1, 2, 3], + })}\n`, + 'utf8', + ); + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sourceCommitSha must be a lowercase 40-character commit SHA.', + ); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed on invalid measurement samples without coercion', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-invalid-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + writeInput(input, [1, -1, 3]); + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark samples must be finite non-negative numbers.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects obviously oversized sample input before whole-file reads', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-size-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const preload = join(root, 'reject-whole-file-read.mjs'); + try { + writeFileSync(input, '', 'utf8'); + truncateSync(input, 16 * 1024 * 1024 + 1); + writeFileSync( + preload, + `import fs from 'node:fs';\nimport { syncBuiltinESMExports } from 'node:module';\nfs.readFileSync = () => { throw new Error('benchmark whole-file read sentinel'); };\nsyncBuiltinESMExports();\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + '--import', + pathToFileURL(preload).href, + script, + '--input', + input, + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input exceeds the supported size.', + ); + expect(result.stderr).not.toContain('benchmark whole-file read sentinel'); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('refuses to overwrite the measurement input with generated evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-alias-')); + const input = join(root, 'summary.json'); + const output = root; + try { + writeInput(input, [10, 20, 30]); + const originalInput = readFileSync(input, 'utf8'); + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark output must not overwrite the sample input.', + ); + expect(readFileSync(input, 'utf8')).toBe(originalInput); + expect(existsSync(join(root, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('refuses a hard-linked output alias without mutating source evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-hardlink-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const summaryJson = join(output, 'summary.json'); + try { + writeInput(input, [10, 20, 30]); + mkdirSync(output, { recursive: true }); + linkSync(input, summaryJson); + const originalInput = readFileSync(input, 'utf8'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark output must not overwrite the sample input.', + ); + expect(readFileSync(input, 'utf8')).toBe(originalInput); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before writing summaries through a symlink output directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-summary-symlink-')); + const input = join(root, 'samples.json'); + const target = join(root, 'outside-target'); + const output = join(root, 'output-link'); + try { + writeInput(input, [10, 20, 30]); + mkdirSync(target); + symlinkSync(target, output, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output directory must be a non-symlink directory.', + ); + expect(existsSync(join(target, 'summary.json'))).toBe(false); + expect(existsSync(join(target, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMeasurementStatisticsInputSymlink.test.ts b/src/performanceMeasurementStatisticsInputSymlink.test.ts new file mode 100644 index 00000000..0c481d53 --- /dev/null +++ b/src/performanceMeasurementStatisticsInputSymlink.test.ts @@ -0,0 +1,53 @@ +import { spawnSync } from 'node:child_process'; +import { + mkdtempSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +function validSamples(): string { + return `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256: 'b'.repeat(64), + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [10, 20, 30], + })}\n`; +} + +describe('benchmark summary sample input file authority', () => { + it('fails closed instead of following a symlinked sample input', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-summary-input-symlink-')); + const target = join(root, 'private-samples.json'); + const alias = join(root, 'samples.json'); + const output = join(root, 'summary'); + try { + writeFileSync(target, validSamples(), 'utf8'); + symlinkSync(target, alias, process.platform === 'win32' ? 'file' : undefined); + + const result = spawnSync( + process.execPath, + [script, '--input', alias, '--output', output], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input must be a regular non-symlink file.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMeasurementStatisticsOutputHardlink.test.ts b/src/performanceMeasurementStatisticsOutputHardlink.test.ts new file mode 100644 index 00000000..75e18c00 --- /dev/null +++ b/src/performanceMeasurementStatisticsOutputHardlink.test.ts @@ -0,0 +1,67 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + linkSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +function writeInput(path: string): void { + writeFileSync( + path, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'markdown-serialization-small', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256: 'b'.repeat(64), + documentProfile: 'small', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [1, 2, 3], + })}\n`, + 'utf8', + ); +} + +describe('benchmark summary output hard-link safety', () => { + it('fails closed before truncating an unrelated hard-linked output target', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-summary-output-hardlink-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + const sentinel = join(root, 'buyer-owned.txt'); + const summaryJson = join(output, 'summary.json'); + + try { + writeInput(input); + mkdirSync(output); + writeFileSync(sentinel, 'buyer-owned-content\n', 'utf8'); + linkSync(sentinel, summaryJson); + const originalSentinel = readFileSync(sentinel, 'utf8'); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary output paths must not be multiply linked.', + ); + expect(readFileSync(sentinel, 'utf8')).toBe(originalSentinel); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMeasurementSuiteAncestorSymlink.test.ts b/src/performanceMeasurementSuiteAncestorSymlink.test.ts new file mode 100644 index 00000000..5ea36b6d --- /dev/null +++ b/src/performanceMeasurementSuiteAncestorSymlink.test.ts @@ -0,0 +1,70 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + symlinkSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/run-current-suite.mjs'); + +describe('benchmark suite output path ancestry', () => { + it('fails closed before preparing an output beneath a symlinked ancestor', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-ancestor-')); + const outside = join(root, 'outside-target'); + const alias = join(root, 'aliased-parent'); + const output = join(alias, 'nested-output'); + try { + mkdirSync(outside); + symlinkSync(outside, alias, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + join(root, 'unused.md'), + '--module', + join(root, 'unused.mjs'), + '--revision-input', + join(root, 'unused-envelope.json'), + '--revision-module', + join(root, 'unused-revision.mjs'), + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'b'.repeat(64), + '--revision-artifact-sha256', + 'c'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark suite output directory must be a non-symlink directory.', + ); + expect(existsSync(join(outside, 'nested-output'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMeasurementUtf8Contract.test.ts b/src/performanceMeasurementUtf8Contract.test.ts new file mode 100644 index 00000000..45ceec9f --- /dev/null +++ b/src/performanceMeasurementUtf8Contract.test.ts @@ -0,0 +1,63 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +describe('benchmark evidence UTF-8 contract', () => { + it('rejects malformed UTF-8 before parsing or generating summary evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-utf8-')); + const input = join(root, 'samples.json'); + const output = join(root, 'output'); + try { + const prefix = [ + '{"contractVersion":1,', + '"benchmarkId":"markdown-serialization-large",', + '"unit":"ms",', + `"sourceCommitSha":"${SOURCE_COMMIT_SHA}",`, + `"artifactSha256":"${ARTIFACT_SHA256}",`, + '"documentProfile":"large",', + '"runtimeId":"node-22.18.0",', + '"referenceHardwareId":"github-actions-ubuntu-24.04-x64",', + '"samples":[1,2,3],', + '"untrustedNote":"', + ].join(''); + writeFileSync( + input, + Buffer.concat([ + Buffer.from(prefix, 'utf8'), + Buffer.from([0x80]), + Buffer.from('"}\n', 'utf8'), + ]), + ); + + const result = spawnSync( + process.execPath, + [script, '--input', input, '--output', output], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input must be valid UTF-8 JSON.', + ); + expect(existsSync(join(output, 'summary.json'))).toBe(false); + expect(existsSync(join(output, 'summary.txt'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceMemorySettlingContract.test.ts b/src/performanceMemorySettlingContract.test.ts new file mode 100644 index 00000000..63f26dce --- /dev/null +++ b/src/performanceMemorySettlingContract.test.ts @@ -0,0 +1,213 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/analyze-memory-settling.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); + +type MemoryEvidenceOverrides = Partial<{ + benchmarkId: string; + sourceCommitSha: string; + artifactSha256: string; + documentProfile: string; + runtimeId: string; + referenceHardwareId: string; + warmupSamples: number; + samples: number[]; +}>; + +function evidence(overrides: MemoryEvidenceOverrides = {}) { + return { + contractVersion: 1, + benchmarkId: 'editor-lifecycle-retained-memory-large', + unit: 'bytes', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-24.0.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + warmupSamples: 2, + samples: [900, 920, 1000, 1005, 995, 1010, 1015, 1008, 1012, 1010], + ...overrides, + }; +} + +function runAnalysis( + root: string, + input: ReturnType, + maxGrowthBytes: string, + windowSize = '3', +) { + const inputPath = join(root, 'memory-evidence.json'); + writeFileSync(inputPath, `${JSON.stringify(input)}\n`, 'utf8'); + return spawnSync( + process.execPath, + [ + script, + '--input', + inputPath, + '--window-size', + windowSize, + '--max-growth-bytes', + maxGrowthBytes, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); +} + +describe('retained-memory settling evidence contract', () => { + it('passes bounded settled growth using explicit warmup and comparison windows', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-pass-')); + try { + const result = runAnalysis(root, evidence(), '16'); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toEqual({ + contractVersion: 1, + benchmarkId: 'editor-lifecycle-retained-memory-large', + unit: 'bytes', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'node-24.0.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 10, + warmupSamples: 2, + windowSize: 3, + firstWindowMedianBytes: 1000, + lastWindowMedianBytes: 1010, + retainedGrowthBytes: 10, + maxGrowthBytes: 16, + passed: true, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails a retained-memory growth breach while preserving the public receipt', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-fail-')); + try { + const result = runAnalysis( + root, + evidence({ + samples: [900, 920, 1000, 1005, 995, 1100, 1120, 1110, 1130, 1140], + }), + '50', + ); + + expect(result.status).toBe(1); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toMatchObject({ + firstWindowMedianBytes: 1000, + lastWindowMedianBytes: 1130, + retainedGrowthBytes: 130, + maxGrowthBytes: 50, + passed: false, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects evidence that cannot supply two disjoint settled windows', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-short-')); + try { + const result = runAnalysis( + root, + evidence({ warmupSamples: 2, samples: [900, 920, 1000, 1005, 1010] }), + '50', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Memory settling evidence requires warmup plus two disjoint comparison windows.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts an input path when a parent component is not a directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-path-privacy-')); + try { + const privateMarker = 'tenant-private-memory-evidence-parent'; + const parentPath = join(root, privateMarker); + writeFileSync(parentPath, 'not-a-directory', 'utf8'); + const inputPath = join(parentPath, 'memory-evidence.json'); + const result = spawnSync( + process.execPath, + [ + script, + '--input', + inputPath, + '--window-size', + '3', + '--max-growth-bytes', + '50', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Memory settling evidence input must be a regular file.', + ); + expect(result.stderr).not.toContain(privateMarker); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects evidence whose benchmark profile disagrees with documentProfile', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-profile-')); + try { + const result = runAnalysis( + root, + evidence({ + benchmarkId: 'editor-lifecycle-retained-memory-small', + documentProfile: 'large', + }), + '50', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Memory settling evidence benchmark profile must match documentProfile.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before a precision-loss false green from an inexact even-window median', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-memory-settling-precision-')); + try { + const maxSafe = Number.MAX_SAFE_INTEGER; + const result = runAnalysis( + root, + evidence({ + warmupSamples: 0, + samples: [maxSafe - 1, maxSafe - 1, maxSafe - 1, maxSafe], + }), + '0.25', + '2', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Memory settling window median must be exactly representable.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceOfficeFixtureContract.test.ts b/src/performanceOfficeFixtureContract.test.ts new file mode 100644 index 00000000..c55b36dd --- /dev/null +++ b/src/performanceOfficeFixtureContract.test.ts @@ -0,0 +1,229 @@ +import { execFileSync } from 'node:child_process'; +import { + linkSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +interface OfficeFixtureProfileLock { + readonly units: number; + readonly bytes: number; + readonly sha256: string; +} + +interface OfficeFixtureLock { + readonly contractVersion: 1; + readonly synthetic: true; + readonly formats: Readonly<{ + docx: Readonly< + Record<'small' | 'page120', OfficeFixtureProfileLock> + >; + xlsx: Readonly< + Record<'small' | 'wide16384', OfficeFixtureProfileLock> + >; + pptx: Readonly< + Record<'small' | 'slide120', OfficeFixtureProfileLock> + >; + }>; +} + +function runGenerator(outputDirectory: string): OfficeFixtureLock { + const script = resolve(process.cwd(), 'benchmarks/generate-office-fixtures.mjs'); + execFileSync(process.execPath, [script, '--output', outputDirectory], { + cwd: process.cwd(), + stdio: ['ignore', 'pipe', 'pipe'], + }); + return JSON.parse( + readFileSync(join(outputDirectory, 'manifest.json'), 'utf8'), + ) as OfficeFixtureLock; +} + +function expectDeterministicFixture( + first: string, + second: string, + fileName: string, + expectedBytes: number, +): void { + const firstBytes = readFileSync(join(first, fileName)); + const secondBytes = readFileSync(join(second, fileName)); + expect(firstBytes.equals(secondBytes)).toBe(true); + expect(firstBytes.byteLength).toBe(expectedBytes); +} + +describe('deterministic synthetic Office performance fixtures', () => { + it('reproduces bounded DOCX, XLSX, and PPTX corpora including 100+ unit fixtures', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-')); + const first = join(root, 'first'); + const second = join(root, 'second'); + try { + const expected = JSON.parse( + readFileSync( + resolve(process.cwd(), 'benchmarks/office-fixtures.lock.json'), + 'utf8', + ), + ) as OfficeFixtureLock; + const firstManifest = runGenerator(first); + const secondManifest = runGenerator(second); + + expect(firstManifest).toEqual(expected); + expect(secondManifest).toEqual(expected); + expect(firstManifest).toEqual({ + contractVersion: 1, + synthetic: true, + formats: expected.formats, + }); + + for (const profile of ['small', 'page120'] as const) { + expectDeterministicFixture( + first, + second, + `docx-${profile}.json`, + expected.formats.docx[profile].bytes, + ); + } + for (const profile of ['small', 'wide16384'] as const) { + expectDeterministicFixture( + first, + second, + `xlsx-${profile}.json`, + expected.formats.xlsx[profile].bytes, + ); + } + for (const profile of ['small', 'slide120'] as const) { + expectDeterministicFixture( + first, + second, + `pptx-${profile}.json`, + expected.formats.pptx[profile].bytes, + ); + } + + const page120 = JSON.parse( + readFileSync(join(first, 'docx-page120.json'), 'utf8'), + ) as { + format: string; + blocks: Array<{ type: string; text?: string }>; + }; + expect(page120.format).toBe('docx'); + expect(page120.blocks.filter(({ type }) => type === 'heading')).toHaveLength(120); + expect(page120.blocks.filter(({ type }) => type === 'page_break')).toHaveLength(119); + expect(page120.blocks.some(({ text }) => text?.includes('한국어'))).toBe(true); + expect(page120.blocks.some(({ text }) => text?.includes('日本語'))).toBe(true); + expect(page120.blocks.some(({ text }) => text?.includes('中文'))).toBe(true); + expect(page120.blocks.some(({ text }) => text?.includes('Tiếng Việt'))).toBe(true); + + const wide = JSON.parse( + readFileSync(join(first, 'xlsx-wide16384.json'), 'utf8'), + ) as { + format: string; + sheets: Array<{ rows: unknown[][]; freeze_panes?: string }>; + }; + expect(wide.format).toBe('xlsx'); + expect(wide.sheets).toHaveLength(1); + expect(wide.sheets[0]?.rows[0]).toHaveLength(16_384); + expect(wide.sheets[0]?.freeze_panes).toBe('XFD1048576'); + + const slide120 = JSON.parse( + readFileSync(join(first, 'pptx-slide120.json'), 'utf8'), + ) as { + format: string; + slides: Array<{ title: string; bullets?: Array }>; + }; + expect(slide120.format).toBe('pptx'); + expect(slide120.slides).toHaveLength(120); + expect(slide120.slides.some(({ title }) => title.includes('한국어'))).toBe(true); + expect( + slide120.slides.some(({ bullets }) => + bullets?.some((bullet) => + typeof bullet === 'string' + ? bullet.includes('日本語') + : bullet.text.includes('日本語'), + ), + ), + ).toBe(true); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed instead of overwriting a file through an Office fixture output symlink', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-symlink-')); + const outputDirectory = join(root, 'output'); + const victimPath = join(root, 'victim.json'); + try { + mkdirSync(outputDirectory, { recursive: true }); + writeFileSync(victimPath, 'buyer-owned evidence\n', 'utf8'); + symlinkSync(victimPath, join(outputDirectory, 'docx-small.json')); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readFileSync(victimPath, 'utf8')).toBe('buyer-owned evidence\n'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed instead of overwriting a multiply linked Office fixture output', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-hardlink-')); + const outputDirectory = join(root, 'output'); + const victimPath = join(root, 'buyer-owned.json'); + try { + mkdirSync(outputDirectory, { recursive: true }); + writeFileSync(victimPath, 'buyer-owned evidence\n', 'utf8'); + linkSync(victimPath, join(outputDirectory, 'docx-small.json')); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readFileSync(victimPath, 'utf8')).toBe('buyer-owned evidence\n'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed instead of publishing through a symlinked Office fixture output directory', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-output-dir-')); + const victimDirectory = join(root, 'buyer-owned'); + const outputDirectory = join(root, 'output'); + try { + mkdirSync(victimDirectory, { recursive: true }); + writeFileSync(join(victimDirectory, 'sentinel.txt'), 'buyer-owned evidence\n', 'utf8'); + symlinkSync(victimDirectory, outputDirectory, 'dir'); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readdirSync(victimDirectory)).toEqual(['sentinel.txt']); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed instead of publishing through a symlinked Office fixture output ancestor', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-office-benchmark-output-ancestor-')); + const victimDirectory = join(root, 'buyer-owned-parent'); + const linkedParent = join(root, 'linked-parent'); + const outputDirectory = join(linkedParent, 'nested-output'); + try { + mkdirSync(victimDirectory, { recursive: true }); + writeFileSync(join(victimDirectory, 'sentinel.txt'), 'buyer-owned evidence\n', 'utf8'); + symlinkSync(victimDirectory, linkedParent, 'dir'); + + expect(() => runGenerator(outputDirectory)).toThrow(); + expect(readdirSync(victimDirectory)).toEqual(['sentinel.txt']); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); \ No newline at end of file diff --git a/src/performancePackedArtifactPathStability.test.ts b/src/performancePackedArtifactPathStability.test.ts new file mode 100644 index 00000000..7ff8c0df --- /dev/null +++ b/src/performancePackedArtifactPathStability.test.ts @@ -0,0 +1,201 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + chmodSync, + copyFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { delimiter, join, resolve } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const temporaryDirectories: string[] = []; +const activeRuntimeId = `node-${process.versions.node}`; +const sourceCommitSha = execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], +}).trim(); +const referenceHardwareId = `refhw-sha256-${'d'.repeat(64)}`; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function sha256(bytes: Buffer | string): string { + return createHash('sha256').update(bytes).digest('hex'); +} + +function createPackedBenchmarkFixture( + directory: string, + name: string, + markdownMarker: string, +): { + markdownModuleSha256: string; + packageSha256: string; + tarballPath: string; +} { + const packageDirectory = join(directory, `${name}-package-source`); + const distDirectory = join(packageDirectory, 'dist'); + const packDirectory = join(directory, `${name}-packed`); + mkdirSync(distDirectory, { recursive: true }); + mkdirSync(packDirectory, { recursive: true }); + + const markdownModule = `export function markdownToHtml(source) { return \`

\${source}

\`; }\n`; + writeFileSync( + join(packageDirectory, 'package.json'), + `${JSON.stringify( + { + name: '@contextualwisdomlab/cwl-editor', + version: '0.0.0-benchmark-fixture', + type: 'module', + files: ['dist'], + }, + null, + 2, + )}\n`, + 'utf8', + ); + writeFileSync( + join(distDirectory, 'cwl-markdown.js'), + markdownModule, + 'utf8', + ); + writeFileSync( + join(distDirectory, 'cwl-revision-evidence.js'), + `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'e'.repeat(64)}' } }; }\n`, + 'utf8', + ); + + const packResult = JSON.parse( + execFileSync( + 'npm', + [ + 'pack', + '--json', + '--ignore-scripts', + '--pack-destination', + packDirectory, + ], + { + cwd: packageDirectory, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, + ), + )[0] as { filename: string }; + const tarballPath = join(packDirectory, packResult.filename); + return { + markdownModuleSha256: sha256(markdownModule), + packageSha256: sha256(readFileSync(tarballPath)), + tarballPath, + }; +} + +function createTarInterpositionShim( + directory: string, + originalTarballPath: string, + adversarialTarballPath: string, +): { environment: NodeJS.ProcessEnv; originalBackupPath: string } { + const shimDirectory = join(directory, 'shim'); + mkdirSync(shimDirectory, { recursive: true }); + const originalBackupPath = join(directory, 'original-package-backup.tgz'); + copyFileSync(originalTarballPath, originalBackupPath); + const realTar = execFileSync('sh', ['-c', 'command -v tar'], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }).trim(); + const shimPath = join(shimDirectory, 'tar'); + writeFileSync( + shimPath, + `#!/usr/bin/env node\nimport { copyFileSync } from 'node:fs';\nimport { spawnSync } from 'node:child_process';\n\nconst args = process.argv.slice(2);\nconst target = args[1];\nconst original = process.env.INKSPAN_TEST_ORIGINAL_TARBALL;\nif (target === original) copyFileSync(process.env.INKSPAN_TEST_ADVERSARIAL_TARBALL, original);\ntry {\n const result = spawnSync(process.env.INKSPAN_TEST_REAL_TAR, args, { stdio: ['ignore', 'pipe', 'pipe'] });\n if (result.stdout) process.stdout.write(result.stdout);\n if (result.stderr) process.stderr.write(result.stderr);\n process.exitCode = result.status ?? 1;\n} finally {\n if (target === original) copyFileSync(process.env.INKSPAN_TEST_ORIGINAL_BACKUP, original);\n}\n`, + 'utf8', + ); + chmodSync(shimPath, 0o755); + return { + originalBackupPath, + environment: { + ...process.env, + PATH: `${shimDirectory}${delimiter}${process.env.PATH ?? ''}`, + INKSPAN_TEST_REAL_TAR: realTar, + INKSPAN_TEST_ORIGINAL_TARBALL: originalTarballPath, + INKSPAN_TEST_ADVERSARIAL_TARBALL: adversarialTarballPath, + INKSPAN_TEST_ORIGINAL_BACKUP: originalBackupPath, + }, + }; +} + +describe('packed artifact benchmark path stability', () => { + it('measures the same tarball bytes whose package digest was verified', () => { + if (process.platform === 'win32') return; + + const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-path-stability-')); + temporaryDirectories.push(directory); + const original = createPackedBenchmarkFixture(directory, 'original', 'verified'); + const adversarial = createPackedBenchmarkFixture(directory, 'adversarial', 'interposed'); + const { environment } = createTarInterpositionShim( + directory, + original.tarballPath, + adversarial.tarballPath, + ); + const markdownInputPath = join(directory, 'input.md'); + const revisionInputPath = join(directory, 'document-envelope.json'); + const outputDirectory = join(directory, 'evidence'); + writeFileSync(markdownInputPath, '# Stable packed artifact\n', 'utf8'); + writeFileSync( + revisionInputPath, + '{"contractVersion":1,"mode":"markdown","document":"# Stable packed artifact"}\n', + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + suitePath, + '--input', + markdownInputPath, + '--revision-input', + revisionInputPath, + '--package-tarball', + original.tarballPath, + '--package-sha256', + original.packageSha256, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + sourceCommitSha, + '--runtime-id', + activeRuntimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + outputDirectory, + ], + { + cwd: repositoryRoot, + encoding: 'utf8', + env: environment, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 15_000, + }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + const markdownEvidence = JSON.parse( + readFileSync(join(outputDirectory, 'markdown', 'samples.json'), 'utf8'), + ) as { artifactSha256: string }; + expect(markdownEvidence.artifactSha256).toBe(original.markdownModuleSha256); + }); +}); diff --git a/src/performancePackedArtifactSuiteContract.test.ts b/src/performancePackedArtifactSuiteContract.test.ts new file mode 100644 index 00000000..5e3eb674 --- /dev/null +++ b/src/performancePackedArtifactSuiteContract.test.ts @@ -0,0 +1,251 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const temporaryDirectories: string[] = []; +const activeRuntimeId = `node-${process.versions.node}`; +const sourceCommitSha = execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], +}).trim(); +const referenceHardwareId = `refhw-sha256-${'b'.repeat(64)}`; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function sha256(bytes: Buffer | string): string { + return createHash('sha256').update(bytes).digest('hex'); +} + +function createPackedBenchmarkFixture(directory: string): { + packageSha256: string; + tarballPath: string; +} { + const packageDirectory = join(directory, 'package-source'); + const distDirectory = join(packageDirectory, 'dist'); + const packDirectory = join(directory, 'packed'); + mkdirSync(distDirectory, { recursive: true }); + mkdirSync(packDirectory, { recursive: true }); + + writeFileSync( + join(packageDirectory, 'package.json'), + `${JSON.stringify( + { + name: '@contextualwisdomlab/cwl-editor', + version: '0.0.0-benchmark-fixture', + type: 'module', + files: ['dist'], + }, + null, + 2, + )}\n`, + 'utf8', + ); + writeFileSync( + join(distDirectory, 'cwl-markdown.js'), + [ + "export function markdownToHtml(source) { return `

${source}

`; }", + "export function htmlToMarkdown(source) { return source.replace(/<[^>]+>/gu, '').trim(); }", + '', + ].join('\n'), + 'utf8', + ); + writeFileSync( + join(distDirectory, 'cwl-revision-evidence.js'), + `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'c'.repeat(64)}' } }; }\n`, + 'utf8', + ); + + const packResult = JSON.parse( + execFileSync( + 'npm', + [ + 'pack', + '--json', + '--ignore-scripts', + '--pack-destination', + packDirectory, + ], + { + cwd: packageDirectory, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, + ), + )[0] as { filename: string }; + const tarballPath = join(packDirectory, packResult.filename); + return { + packageSha256: sha256(readFileSync(tarballPath)), + tarballPath, + }; +} + +function packedSuiteArguments(options: { + directory: string; + packageSha256: string; + runtimeId: string; + sourceCommitSha?: string; + tarballPath: string; +}): string[] { + const markdownInputPath = join(options.directory, 'input.md'); + const htmlInputPath = join(options.directory, 'input.html'); + const revisionInputPath = join(options.directory, 'document-envelope.json'); + writeFileSync(markdownInputPath, '# Packed buyer benchmark\n', 'utf8'); + writeFileSync(htmlInputPath, '

Packed buyer benchmark

\n', 'utf8'); + writeFileSync( + revisionInputPath, + '{"contractVersion":1,"mode":"markdown","document":"# Packed buyer benchmark"}\n', + 'utf8', + ); + return [ + suitePath, + '--input', + markdownInputPath, + '--html-input', + htmlInputPath, + '--revision-input', + revisionInputPath, + '--package-tarball', + options.tarballPath, + '--package-sha256', + options.packageSha256, + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + options.sourceCommitSha ?? sourceCommitSha, + '--runtime-id', + options.runtimeId, + '--reference-hardware-id', + referenceHardwareId, + '--output', + join(options.directory, 'evidence'), + ]; +} + +describe('packed artifact benchmark suite contract', () => { + it('binds one-command benchmark evidence to packed artifact and run provenance', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-benchmark-')); + temporaryDirectories.push(directory); + const packed = createPackedBenchmarkFixture(directory); + + const result = spawnSync( + process.execPath, + packedSuiteArguments({ + directory, + packageSha256: packed.packageSha256, + runtimeId: activeRuntimeId, + tarballPath: packed.tarballPath, + }), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 15_000, + }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout.trim())).toMatchObject({ + contractVersion: 1, + documentProfile: 'small', + sampleCount: 2, + sourceCommitSha, + runtimeId: activeRuntimeId, + referenceHardwareId, + packageName: '@contextualwisdomlab/cwl-editor', + packageVersion: '0.0.0-benchmark-fixture', + packageSha256: packed.packageSha256, + htmlSerializationSamples: 'html-serialization/samples.json', + htmlSerializationSummaryJson: + 'html-serialization/summary/summary.json', + htmlSerializationSummaryText: + 'html-serialization/summary/summary.txt', + status: 'completed', + }); + + const htmlSamples = JSON.parse( + readFileSync( + join(directory, 'evidence', 'html-serialization', 'samples.json'), + 'utf8', + ), + ) as { benchmarkId?: unknown; samples?: unknown[] }; + expect(htmlSamples.benchmarkId).toBe('html-serialization-small'); + expect(htmlSamples.samples).toHaveLength(2); + }); + + it('rejects a runtime identifier that does not match the active Node process', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-runtime-')); + temporaryDirectories.push(directory); + const packed = createPackedBenchmarkFixture(directory); + + const result = spawnSync( + process.execPath, + packedSuiteArguments({ + directory, + packageSha256: packed.packageSha256, + runtimeId: 'node-0.0.0', + tarballPath: packed.tarballPath, + }), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 15_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite runtime ID must match the active Node runtime.\n', + ); + }); + + it('rejects source provenance that does not match the benchmark checkout', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-packed-source-')); + temporaryDirectories.push(directory); + const packed = createPackedBenchmarkFixture(directory); + + const result = spawnSync( + process.execPath, + packedSuiteArguments({ + directory, + packageSha256: packed.packageSha256, + runtimeId: activeRuntimeId, + sourceCommitSha: '0'.repeat(40), + tarballPath: packed.tarballPath, + }), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 15_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite source commit SHA must match the current benchmark checkout.\n', + ); + }); +}); diff --git a/src/performanceRegressionComparatorContract.test.ts b/src/performanceRegressionComparatorContract.test.ts new file mode 100644 index 00000000..d51ae9f1 --- /dev/null +++ b/src/performanceRegressionComparatorContract.test.ts @@ -0,0 +1,325 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/compare-summaries.mjs'); +const SOURCE_COMMIT_SHA = 'a'.repeat(40); +const ARTIFACT_SHA256 = 'b'.repeat(64); +const CURRENT_ARTIFACT_SHA256 = 'c'.repeat(64); + +type SummaryOverrides = Partial<{ + benchmarkId: string; + unit: string; + sourceCommitSha: string; + artifactSha256: string; + documentProfile: string; + runtimeId: string; + referenceHardwareId: string; + sampleCount: number; + percentileMethod: string; + minimum: number; + p50: number; + p75: number; + p95: number; + maximum: number; +}>; + +function summary(overrides: SummaryOverrides = {}) { + return { + contractVersion: 1, + benchmarkId: 'editor-input-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: ARTIFACT_SHA256, + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + minimum: 70, + p50: 80, + p75: 90, + p95: 100, + maximum: 110, + ...overrides, + }; +} + +function runComparison( + root: string, + baseline: ReturnType, + current: ReturnType, + tolerancePercent: string, +) { + const baselinePath = join(root, 'baseline.json'); + const currentPath = join(root, 'current.json'); + writeFileSync(baselinePath, `${JSON.stringify(baseline)}\n`, 'utf8'); + writeFileSync(currentPath, `${JSON.stringify(current)}\n`, 'utf8'); + return spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + tolerancePercent, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); +} + +describe('benchmark regression comparator contract', () => { + it('passes only when a current exact-context metric stays within an explicit tolerance', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-pass-')); + try { + const result = runComparison( + root, + summary(), + summary({ artifactSha256: CURRENT_ARTIFACT_SHA256, p95: 104 }), + '5', + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toEqual({ + contractVersion: 1, + benchmarkId: 'editor-input-large', + unit: 'ms', + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + metric: 'p95', + baselineSourceCommitSha: SOURCE_COMMIT_SHA, + baselineArtifactSha256: ARTIFACT_SHA256, + currentSourceCommitSha: SOURCE_COMMIT_SHA, + currentArtifactSha256: CURRENT_ARTIFACT_SHA256, + baselineValue: 100, + currentValue: 104, + maxRegressionPercent: 5, + regressionPercent: 4, + passed: true, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails a material unapproved regression without hiding the measured receipt', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-fail-')); + try { + const result = runComparison( + root, + summary(), + summary({ artifactSha256: CURRENT_ARTIFACT_SHA256, p95: 106 }), + '5', + ); + + expect(result.status).toBe(1); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toEqual({ + contractVersion: 1, + benchmarkId: 'editor-input-large', + unit: 'ms', + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + metric: 'p95', + baselineSourceCommitSha: SOURCE_COMMIT_SHA, + baselineArtifactSha256: ARTIFACT_SHA256, + currentSourceCommitSha: SOURCE_COMMIT_SHA, + currentArtifactSha256: CURRENT_ARTIFACT_SHA256, + baselineValue: 100, + currentValue: 106, + maxRegressionPercent: 5, + regressionPercent: 6, + passed: false, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects incomparable runtime or hardware evidence instead of laundering it through a tolerance', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-context-')); + try { + const result = runComparison( + root, + summary(), + summary({ referenceHardwareId: 'github-actions-ubuntu-22.04-x64' }), + '5', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summaries are not comparable: referenceHardwareId differs.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects private-looking units at the direct summary-comparison boundary', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-unit-')); + try { + const result = runComparison( + root, + summary({ unit: 'tenant-acme' }), + summary({ artifactSha256: CURRENT_ARTIFACT_SHA256, unit: 'tenant-acme' }), + '5', + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe('Benchmark summary unit is invalid.'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('requires an explicit finite non-negative regression tolerance', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-tolerance-')); + try { + const result = runComparison(root, summary(), summary(), '-1'); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark max regression percent must be a finite non-negative number.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects symlinked summary inputs instead of comparing mutable aliases', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-symlink-')); + const baselineTargetPath = join(root, 'baseline-target.json'); + const baselinePath = join(root, 'baseline-link.json'); + const currentPath = join(root, 'current.json'); + try { + writeFileSync(baselineTargetPath, `${JSON.stringify(summary())}\n`, 'utf8'); + symlinkSync(baselineTargetPath, baselinePath); + writeFileSync( + currentPath, + `${JSON.stringify(summary({ artifactSha256: CURRENT_ARTIFACT_SHA256 }))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary input must be a regular non-symlink file.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts a summary input path when a parent component is not a directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-path-privacy-')); + try { + const privateMarker = 'tenant-private-performance-baseline-parent'; + const privateParentPath = join(root, privateMarker); + const baselinePath = join(privateParentPath, 'baseline.json'); + const currentPath = join(root, 'current.json'); + writeFileSync(privateParentPath, 'not-a-directory', 'utf8'); + writeFileSync( + currentPath, + `${JSON.stringify(summary({ artifactSha256: CURRENT_ARTIFACT_SHA256 }))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary input must be a regular file.', + ); + expect(result.stderr).not.toContain(privateMarker); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed on a named-pipe summary instead of blocking before regular-file validation', () => { + if (process.platform === 'win32') return; + + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-compare-fifo-')); + const baselinePath = join(root, 'baseline.pipe'); + const currentPath = join(root, 'current.json'); + try { + const mkfifo = spawnSync('mkfifo', [baselinePath], { encoding: 'utf8' }); + expect(mkfifo.status).toBe(0); + writeFileSync(currentPath, `${JSON.stringify(summary())}\n`, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8', timeout: 1000 }, + ); + + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary input must be a regular file.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceRegressionOverflowContract.test.ts b/src/performanceRegressionOverflowContract.test.ts new file mode 100644 index 00000000..2676c458 --- /dev/null +++ b/src/performanceRegressionOverflowContract.test.ts @@ -0,0 +1,71 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/compare-summaries.mjs'); + +function summary(measurement: number, digestCharacter: string) { + return { + contractVersion: 1, + benchmarkId: 'markdown-serialization-large', + unit: 'ms', + sourceCommitSha: digestCharacter.repeat(40), + artifactSha256: digestCharacter.repeat(64), + documentProfile: 'large', + runtimeId: 'node-22.18.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 1, + percentileMethod: 'nearest-rank', + minimum: measurement, + p50: measurement, + p75: measurement, + p95: measurement, + maximum: measurement, + }; +} + +describe('benchmark regression comparator overflow contract', () => { + it('fails closed instead of serializing an overflowing regression percentage as JSON null', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-overflow-')); + const baseline = join(root, 'baseline.json'); + const current = join(root, 'current.json'); + try { + writeFileSync( + baseline, + `${JSON.stringify(summary(1e-308, 'a'))}\n`, + 'utf8', + ); + writeFileSync( + current, + `${JSON.stringify(summary(1e308, 'b'))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baseline, + '--current', + current, + '--metric', + 'p95', + '--max-regression-percent', + '10', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark regression percent is not finite for the supplied measurements.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceRegressionProfileConsistencyContract.test.ts b/src/performanceRegressionProfileConsistencyContract.test.ts new file mode 100644 index 00000000..bf787527 --- /dev/null +++ b/src/performanceRegressionProfileConsistencyContract.test.ts @@ -0,0 +1,64 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/compare-summaries.mjs'); + +function summary(artifactSha256: string) { + return { + contractVersion: 1, + benchmarkId: 'editor-input-small', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256, + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + minimum: 70, + p50: 80, + p75: 90, + p95: 100, + maximum: 110, + }; +} + +describe('benchmark regression profile consistency contract', () => { + it('rejects summaries whose benchmarkId profile disagrees with documentProfile', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-profile-')); + const baselinePath = join(root, 'baseline.json'); + const currentPath = join(root, 'current.json'); + + try { + writeFileSync(baselinePath, `${JSON.stringify(summary('b'.repeat(64)))}\n`, 'utf8'); + writeFileSync(currentPath, `${JSON.stringify(summary('c'.repeat(64)))}\n`, 'utf8'); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark summary profile must match documentProfile.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceRegressionProvenanceContract.test.ts b/src/performanceRegressionProvenanceContract.test.ts new file mode 100644 index 00000000..55e90cda --- /dev/null +++ b/src/performanceRegressionProvenanceContract.test.ts @@ -0,0 +1,135 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/compare-summaries.mjs'); + +function summary(sourceCommitSha: string, artifactSha256: string) { + return { + contractVersion: 1, + benchmarkId: 'editor-input-large', + unit: 'ms', + sourceCommitSha, + artifactSha256, + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + minimum: 70, + p50: 80, + p75: 90, + p95: 100, + maximum: 110, + }; +} + +describe('benchmark regression provenance contract', () => { + it('binds each comparison receipt to both exact measured artifacts and the shared context', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-provenance-')); + try { + const baselinePath = join(root, 'baseline.json'); + const currentPath = join(root, 'current.json'); + const baselineSourceCommitSha = 'a'.repeat(40); + const currentSourceCommitSha = 'c'.repeat(40); + const baselineArtifactSha256 = 'b'.repeat(64); + const currentArtifactSha256 = 'd'.repeat(64); + writeFileSync( + baselinePath, + `${JSON.stringify(summary(baselineSourceCommitSha, baselineArtifactSha256))}\n`, + 'utf8', + ); + writeFileSync( + currentPath, + `${JSON.stringify(summary(currentSourceCommitSha, currentArtifactSha256))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + expect(JSON.parse(result.stdout)).toEqual({ + contractVersion: 1, + benchmarkId: 'editor-input-large', + unit: 'ms', + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + sampleCount: 20, + percentileMethod: 'nearest-rank', + metric: 'p95', + baselineSourceCommitSha, + baselineArtifactSha256, + currentSourceCommitSha, + currentArtifactSha256, + baselineValue: 100, + currentValue: 100, + maxRegressionPercent: 5, + regressionPercent: 0, + passed: true, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('rejects a vacuous comparison when both summaries identify the same exact artifact', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-provenance-')); + try { + const baselinePath = join(root, 'baseline.json'); + const currentPath = join(root, 'current.json'); + const artifactSha256 = 'b'.repeat(64); + writeFileSync( + baselinePath, + `${JSON.stringify(summary('a'.repeat(40), artifactSha256))}\n`, + 'utf8', + ); + writeFileSync( + currentPath, + `${JSON.stringify(summary('c'.repeat(40), artifactSha256))}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [ + script, + '--baseline', + baselinePath, + '--current', + currentPath, + '--metric', + 'p95', + '--max-regression-percent', + '5', + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark summaries must identify distinct measured artifacts.\n', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceRevisionMeasurementContract.test.ts b/src/performanceRevisionMeasurementContract.test.ts new file mode 100644 index 00000000..8df70188 --- /dev/null +++ b/src/performanceRevisionMeasurementContract.test.ts @@ -0,0 +1,279 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +interface BenchmarkSamples { + readonly contractVersion: 1; + readonly benchmarkId: string; + readonly unit: 'ms'; + readonly sourceCommitSha: string; + readonly artifactSha256: string; + readonly documentProfile: 'small' | 'medium' | 'large' | 'stress'; + readonly runtimeId: string; + readonly referenceHardwareId: string; + readonly samples: number[]; +} + +const repositoryRoot = process.cwd(); +const measurementScript = resolve( + repositoryRoot, + 'benchmarks/measure-revision-evidence.mjs', +); +const summaryScript = resolve(repositoryRoot, 'benchmarks/summarize-samples.mjs'); +const SOURCE_COMMIT_SHA = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const RUNTIME_ID = `node-${process.versions.node}`; +const HARDWARE_ID = 'github-actions-ubuntu-24.04-x64'; + +function fileSha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +function argumentsFor( + input: string, + modulePath: string, + output: string, +): string[] { + return [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '3', + '--source-commit-sha', + SOURCE_COMMIT_SHA, + '--artifact-sha256', + fileSha256(modulePath), + '--runtime-id', + RUNTIME_ID, + '--reference-hardware-id', + HARDWARE_ID, + '--output', + output, + ]; +} + +function writeSyntheticEnvelope(path: string): void { + writeFileSync( + path, + JSON.stringify({ + schemaId: 'https://inkspan.io/schemas/document-envelope/v1', + schemaVersion: 1, + documentJson: { + type: 'doc', + content: [ + { + type: 'paragraph', + content: [{ type: 'text', text: 'Synthetic benchmark content' }], + }, + ], + }, + }), + 'utf8', + ); +} + +describe('revision-evidence runtime measurement contract', () => { + it('writes privacy-safe revision samples consumable by the canonical summarizer', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const samplesPath = join(root, 'samples.json'); + const summaryDirectory = join(root, 'summary'); + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) { return { revision: { digestHex: String(source.byteLength).padStart(64, "0") } }; }\n', + 'utf8', + ); + + execFileSync(process.execPath, argumentsFor(input, modulePath, samplesPath), { + cwd: repositoryRoot, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + const samples = JSON.parse( + readFileSync(samplesPath, 'utf8'), + ) as BenchmarkSamples; + expect(samples).toMatchObject({ + contractVersion: 1, + benchmarkId: 'revision-evidence-large', + unit: 'ms', + sourceCommitSha: SOURCE_COMMIT_SHA, + artifactSha256: fileSha256(modulePath), + documentProfile: 'large', + runtimeId: RUNTIME_ID, + referenceHardwareId: HARDWARE_ID, + }); + expect(samples.samples).toHaveLength(3); + expect( + samples.samples.every( + (sample) => Number.isFinite(sample) && sample >= 0, + ), + ).toBe(true); + expect(readFileSync(samplesPath, 'utf8')).not.toContain( + 'Synthetic benchmark content', + ); + + execFileSync( + process.execPath, + [summaryScript, '--input', samplesPath, '--output', summaryDirectory], + { cwd: repositoryRoot, stdio: ['ignore', 'pipe', 'pipe'] }, + ); + expect( + JSON.parse(readFileSync(join(summaryDirectory, 'summary.json'), 'utf8')), + ).toMatchObject({ + sampleCount: 3, + benchmarkId: 'revision-evidence-large', + unit: 'ms', + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('fails closed before output when the measured module lacks the revision API', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-export-')); + const input = join(root, 'small.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const samplesPath = join(root, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync(modulePath, 'export const other = true;\n', 'utf8'); + + const result = spawnSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath), + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured revision module must export createDocumentEnvelopeRevisionEvidenceBytes().', + ); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts hostile revision-result accessors before publishing output', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-result-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const samplesPath = join(root, 'samples.json'); + const privateSentinel = 'tenant-private-revision-result-sentinel'; + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return new Proxy({}, { get(_target, property) { if (property === 'revision') throw new Error('${privateSentinel}'); return undefined; } }); }\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath), + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured revision-evidence result is invalid.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts filesystem details when output path traversal fails', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-output-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const privateSentinel = 'private-revision-output-sentinel-must-not-leak'; + const blockedParent = join(root, privateSentinel); + const samplesPath = join(blockedParent, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) { return { revision: { digestHex: String(source.byteLength).padStart(64, "0") } }; }\n', + 'utf8', + ); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath), + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Revision benchmark output path could not be inspected.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('redacts filesystem details when output publication cannot create its directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-measurement-publication-')); + const input = join(root, 'large.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const privateSentinel = `private-revision-publication-${process.pid}`; + const samplesPath = join('/sys', privateSentinel, 'samples.json'); + try { + writeSyntheticEnvelope(input); + writeFileSync( + modulePath, + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) { return { revision: { digestHex: String(source.byteLength).padStart(64, "0") } }; }\n', + 'utf8', + ); + + const result = spawnSync( + process.execPath, + argumentsFor(input, modulePath, samplesPath), + { cwd: repositoryRoot, encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Revision benchmark output could not be written.', + ); + expect(result.stderr).not.toContain(privateSentinel); + expect(existsSync(samplesPath)).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceRevisionMeasurementInputPathPrivacyContract.test.ts b/src/performanceRevisionMeasurementInputPathPrivacyContract.test.ts new file mode 100644 index 00000000..fee61353 --- /dev/null +++ b/src/performanceRevisionMeasurementInputPathPrivacyContract.test.ts @@ -0,0 +1,60 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-revision-evidence.mjs', +); + +describe('revision benchmark input-path privacy contract', () => { + it('redacts filesystem details when input inspection crosses a non-directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-input-path-')); + const privateSentinel = 'tenant-private-revision-input-parent'; + const blockedParent = join(root, privateSentinel); + const input = join(blockedParent, 'document-envelope.json'); + const output = join(root, 'samples.json'); + + try { + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + join(root, 'unused-module.mjs'), + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'b'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Revision benchmark input must be a regular non-symlink file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceRevisionMeasurementModulePathPrivacyContract.test.ts b/src/performanceRevisionMeasurementModulePathPrivacyContract.test.ts new file mode 100644 index 00000000..814f35c6 --- /dev/null +++ b/src/performanceRevisionMeasurementModulePathPrivacyContract.test.ts @@ -0,0 +1,82 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +const measurementScript = resolve( + process.cwd(), + 'benchmarks/measure-revision-evidence.mjs', +); + +function writeSyntheticEnvelope(path: string): void { + writeFileSync( + path, + JSON.stringify({ + schemaId: 'https://inkspan.io/schemas/document-envelope/v1', + schemaVersion: 1, + documentJson: { + type: 'doc', + content: [ + { + type: 'paragraph', + content: [{ type: 'text', text: 'Synthetic benchmark content' }], + }, + ], + }, + }), + 'utf8', + ); +} + +describe('revision benchmark module-path privacy contract', () => { + it('redacts filesystem details when module-path resolution crosses a non-directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-module-path-')); + const input = join(root, 'small.json'); + const privateSentinel = 'tenant-private-revision-module-parent'; + const blockedParent = join(root, privateSentinel); + const modulePath = join(blockedParent, 'packed-revision-evidence.mjs'); + const output = join(root, 'samples.json'); + + try { + writeSyntheticEnvelope(input); + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [ + measurementScript, + '--input', + input, + '--module', + modulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + 'b'.repeat(64), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Measured revision module must be a local regular file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceRevisionMeasurementOutputAncestorSymlink.test.ts b/src/performanceRevisionMeasurementOutputAncestorSymlink.test.ts new file mode 100644 index 00000000..48348e12 --- /dev/null +++ b/src/performanceRevisionMeasurementOutputAncestorSymlink.test.ts @@ -0,0 +1,84 @@ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/measure-revision-evidence.mjs'); + +function sha256(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +describe('revision benchmark output path ancestry', () => { + it('fails closed before writing beneath a symlinked output ancestor', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-revision-output-ancestor-')); + const input = join(root, 'document-envelope.json'); + const modulePath = join(root, 'packed-revision-evidence.mjs'); + const outside = join(root, 'outside-target'); + const alias = join(root, 'aliased-parent'); + const output = join(alias, 'nested-output', 'samples.json'); + try { + writeFileSync( + input, + JSON.stringify({ + schemaId: 'https://inkspan.io/schemas/document-envelope/v1', + schemaVersion: 1, + documentJson: { type: 'doc', content: [] }, + }), + 'utf8', + ); + writeFileSync( + modulePath, + 'export async function createDocumentEnvelopeRevisionEvidenceBytes(source) { return { revision: { digestHex: String(source.byteLength).padStart(64, "0") } }; }\n', + 'utf8', + ); + mkdirSync(outside); + symlinkSync(outside, alias, process.platform === 'win32' ? 'junction' : 'dir'); + + const result = spawnSync( + process.execPath, + [ + script, + '--input', + input, + '--module', + modulePath, + '--profile', + 'large', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + sha256(modulePath), + '--runtime-id', + 'node-22.18.0', + '--reference-hardware-id', + 'github-actions-ubuntu-24.04-x64', + '--output', + output, + ], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Revision benchmark output directory must be a non-symlink directory.', + ); + expect(existsSync(join(outside, 'nested-output', 'samples.json'))).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceSingleCommandSuiteContract.test.ts b/src/performanceSingleCommandSuiteContract.test.ts new file mode 100644 index 00000000..ca04e242 --- /dev/null +++ b/src/performanceSingleCommandSuiteContract.test.ts @@ -0,0 +1,322 @@ +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const temporaryDirectories: string[] = []; +const currentSourceCommitSha = execFileSync( + 'git', + ['rev-parse', '--verify', 'HEAD'], + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }, +).trim(); +const currentRuntimeId = `node-${process.versions.node}`; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function benchmarkArguments( + markdownInputPath: string, + markdownModulePath: string, + markdownArtifactSha256: string, + revisionInputPath: string, + revisionModulePath: string, + revisionArtifactSha256: string, + outputDirectory: string, + sourceCommitSha = currentSourceCommitSha, +): string[] { + return [ + suitePath, + '--input', + markdownInputPath, + '--module', + markdownModulePath, + '--revision-input', + revisionInputPath, + '--revision-module', + revisionModulePath, + '--profile', + 'small', + '--samples', + '2', + '--source-commit-sha', + sourceCommitSha, + '--artifact-sha256', + markdownArtifactSha256, + '--revision-artifact-sha256', + revisionArtifactSha256, + '--runtime-id', + currentRuntimeId, + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + outputDirectory, + ]; +} + +function writeBenchmarkInputs(directory: string): { + markdownArtifactSha256: string; + markdownInputPath: string; + markdownModulePath: string; + revisionArtifactSha256: string; + revisionInputPath: string; + revisionModulePath: string; +} { + const markdownInputPath = join(directory, 'input.md'); + const markdownModulePath = join(directory, 'markdown-measured.mjs'); + const markdownModuleSource = + "export function markdownToHtml(source) { return `

${source}

`; }\n"; + const revisionInputPath = join(directory, 'document-envelope.json'); + const revisionModulePath = join(directory, 'revision-measured.mjs'); + const revisionModuleSource = `export async function createDocumentEnvelopeRevisionEvidenceBytes() { return { revision: { digestHex: '${'c'.repeat(64)}' } }; }\n`; + + writeFileSync(markdownInputPath, '# Buyer benchmark\n', 'utf8'); + writeFileSync(markdownModulePath, markdownModuleSource, 'utf8'); + writeFileSync( + revisionInputPath, + '{"contractVersion":1,"mode":"markdown","document":"# Buyer benchmark"}\n', + 'utf8', + ); + writeFileSync(revisionModulePath, revisionModuleSource, 'utf8'); + + return { + markdownArtifactSha256: createHash('sha256') + .update(markdownModuleSource) + .digest('hex'), + markdownInputPath, + markdownModulePath, + revisionArtifactSha256: createHash('sha256') + .update(revisionModuleSource) + .digest('hex'), + revisionInputPath, + revisionModulePath, + }; +} + +describe('single-command benchmark suite contract', () => { + it('measures and summarizes Markdown serialization and revision evidence with one command', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); + temporaryDirectories.push(directory); + const outputDirectory = join(directory, 'evidence'); + const inputs = writeBenchmarkInputs(directory); + + const output = execFileSync( + process.execPath, + benchmarkArguments( + inputs.markdownInputPath, + inputs.markdownModulePath, + inputs.markdownArtifactSha256, + inputs.revisionInputPath, + inputs.revisionModulePath, + inputs.revisionArtifactSha256, + outputDirectory, + ), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(JSON.parse(output.trim())).toEqual({ + contractVersion: 1, + documentProfile: 'small', + sampleCount: 2, + sourceCommitSha: currentSourceCommitSha, + runtimeId: currentRuntimeId, + referenceHardwareId: `refhw-sha256-${'b'.repeat(64)}`, + markdownSamples: 'markdown/samples.json', + markdownSummaryJson: 'markdown/summary/summary.json', + markdownSummaryText: 'markdown/summary/summary.txt', + revisionSamples: 'revision/samples.json', + revisionSummaryJson: 'revision/summary/summary.json', + revisionSummaryText: 'revision/summary/summary.txt', + status: 'completed', + }); + + const markdownSamples = JSON.parse( + readFileSync(join(outputDirectory, 'markdown', 'samples.json'), 'utf8'), + ) as { benchmarkId?: unknown; documentProfile?: unknown; samples?: unknown }; + expect(markdownSamples.benchmarkId).toBe('markdown-serialization-small'); + expect(markdownSamples.documentProfile).toBe('small'); + expect(markdownSamples.samples).toHaveLength(2); + + const markdownSummary = JSON.parse( + readFileSync( + join(outputDirectory, 'markdown', 'summary', 'summary.json'), + 'utf8', + ), + ) as { benchmarkId?: unknown; documentProfile?: unknown }; + expect(markdownSummary.benchmarkId).toBe('markdown-serialization-small'); + expect(markdownSummary.documentProfile).toBe('small'); + expect( + readFileSync( + join(outputDirectory, 'markdown', 'summary', 'summary.txt'), + 'utf8', + ), + ).toContain('markdown-serialization-small'); + + const revisionSamples = JSON.parse( + readFileSync(join(outputDirectory, 'revision', 'samples.json'), 'utf8'), + ) as { benchmarkId?: unknown; documentProfile?: unknown; samples?: unknown }; + expect(revisionSamples.benchmarkId).toBe('revision-evidence-small'); + expect(revisionSamples.documentProfile).toBe('small'); + expect(revisionSamples.samples).toHaveLength(2); + + const revisionSummary = JSON.parse( + readFileSync( + join(outputDirectory, 'revision', 'summary', 'summary.json'), + 'utf8', + ), + ) as { benchmarkId?: unknown; documentProfile?: unknown }; + expect(revisionSummary.benchmarkId).toBe('revision-evidence-small'); + expect(revisionSummary.documentProfile).toBe('small'); + expect( + readFileSync( + join(outputDirectory, 'revision', 'summary', 'summary.txt'), + 'utf8', + ), + ).toContain('revision-evidence-small'); + }); + + it('rejects a claimed source commit that is not the checked-out HEAD', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-source-')); + temporaryDirectories.push(directory); + const outputDirectory = join(directory, 'evidence'); + const inputs = writeBenchmarkInputs(directory); + const mismatchedCommit = + currentSourceCommitSha === 'f'.repeat(40) ? 'e'.repeat(40) : 'f'.repeat(40); + + const result = spawnSync( + process.execPath, + benchmarkArguments( + inputs.markdownInputPath, + inputs.markdownModulePath, + inputs.markdownArtifactSha256, + inputs.revisionInputPath, + inputs.revisionModulePath, + inputs.revisionArtifactSha256, + outputDirectory, + mismatchedCommit, + ), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite source commit does not match checked-out HEAD.\n', + ); + expect(result.stderr).not.toContain(currentSourceCommitSha); + expect(result.stderr).not.toContain(mismatchedCommit); + expect(existsSync(outputDirectory)).toBe(false); + }); + + it('removes partial suite evidence when a downstream measurement fails', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); + temporaryDirectories.push(directory); + const outputDirectory = join(directory, 'evidence'); + const inputs = writeBenchmarkInputs(directory); + const failingRevisionModuleSource = + "export async function createDocumentEnvelopeRevisionEvidenceBytes() { throw new Error('private benchmark failure'); }\n"; + writeFileSync( + inputs.revisionModulePath, + failingRevisionModuleSource, + 'utf8', + ); + const failingRevisionArtifactSha256 = createHash('sha256') + .update(failingRevisionModuleSource) + .digest('hex'); + + const result = spawnSync( + process.execPath, + benchmarkArguments( + inputs.markdownInputPath, + inputs.markdownModulePath, + inputs.markdownArtifactSha256, + inputs.revisionInputPath, + inputs.revisionModulePath, + failingRevisionArtifactSha256, + outputDirectory, + ), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).not.toBe(0); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite revision measurement failed.\n', + ); + expect(existsSync(outputDirectory)).toBe(false); + }); + + it('fails closed before writing evidence through a symlink output directory', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-suite-')); + temporaryDirectories.push(directory); + const inputs = writeBenchmarkInputs(directory); + const actualOutputDirectory = join(directory, 'outside-target'); + const outputDirectory = join(directory, 'evidence-link'); + mkdirSync(actualOutputDirectory); + symlinkSync( + actualOutputDirectory, + outputDirectory, + process.platform === 'win32' ? 'junction' : 'dir', + ); + + const result = spawnSync( + process.execPath, + benchmarkArguments( + inputs.markdownInputPath, + inputs.markdownModulePath, + inputs.markdownArtifactSha256, + inputs.revisionInputPath, + inputs.revisionModulePath, + inputs.revisionArtifactSha256, + outputDirectory, + ), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).not.toBe(0); + expect(result.stderr).toBe( + 'Benchmark suite output directory must be a non-symlink directory.\n', + ); + expect(existsSync(join(actualOutputDirectory, 'markdown'))).toBe(false); + expect(existsSync(join(actualOutputDirectory, 'revision'))).toBe(false); + }); +}); diff --git a/src/performanceSourceCheckoutProvenanceContract.test.ts b/src/performanceSourceCheckoutProvenanceContract.test.ts new file mode 100644 index 00000000..34ab09d0 --- /dev/null +++ b/src/performanceSourceCheckoutProvenanceContract.test.ts @@ -0,0 +1,115 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const helperUrl = pathToFileURL( + resolve(repositoryRoot, 'benchmarks/source-checkout-provenance.mjs'), +).href; +const temporaryDirectories: string[] = []; + +const probe = ` +import { assertCleanSourceCheckout } from ${JSON.stringify(helperUrl)}; +try { + assertCleanSourceCheckout(process.argv[1], process.argv[2]); + process.stdout.write('clean\\n'); +} catch (error) { + process.stderr.write(\`${'${error instanceof Error ? error.message : "verification failed"}'}\\n\`); + process.exitCode = 1; +} +`; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function createRepository(): string { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-benchmark-provenance-')); + temporaryDirectories.push(directory); + execFileSync('git', ['init', '--quiet'], { cwd: directory }); + execFileSync('git', ['config', 'user.email', 'test@example.invalid'], { + cwd: directory, + }); + execFileSync('git', ['config', 'user.name', 'Inkspan Test'], { + cwd: directory, + }); + writeFileSync(join(directory, 'tracked.txt'), 'committed\n'); + execFileSync('git', ['add', 'tracked.txt'], { cwd: directory }); + execFileSync('git', ['commit', '--quiet', '-m', 'fixture'], { cwd: directory }); + return directory; +} + +function headSha(directory: string): string { + return execFileSync('git', ['rev-parse', '--verify', 'HEAD'], { + cwd: directory, + encoding: 'utf8', + }).trim(); +} + +function probeCheckout(directory: string, expectedCommitSha = headSha(directory)) { + return spawnSync( + process.execPath, + ['--input-type=module', '--eval', probe, directory, expectedCommitSha], + { + cwd: repositoryRoot, + encoding: 'utf8', + maxBuffer: 1024 * 1024, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); +} + +describe('benchmark source checkout provenance', () => { + it('accepts a clean source checkout at the claimed source commit', () => { + const directory = createRepository(); + const result = probeCheckout(directory); + + expect(result.error).toBeUndefined(); + expect(result.signal).toBeNull(); + expect(result.status).toBe(0); + expect(result.stdout).toBe('clean\n'); + expect(result.stderr).toBe(''); + }); + + it('rejects a clean checkout when the claimed source commit is not HEAD', () => { + const directory = createRepository(); + const actualHead = headSha(directory); + const mismatchedCommit = actualHead === 'f'.repeat(40) ? 'e'.repeat(40) : 'f'.repeat(40); + const result = probeCheckout(directory, mismatchedCommit); + + expect(result.error).toBeUndefined(); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite source commit does not match checked-out HEAD.\n', + ); + expect(result.stderr).not.toContain(actualHead); + expect(result.stderr).not.toContain(mismatchedCommit); + }); + + it('rejects untracked source state without disclosing paths', () => { + const directory = createRepository(); + const untrackedPath = join(directory, 'untracked-secret-name.txt'); + writeFileSync(untrackedPath, 'not part of the committed source\n'); + + const result = probeCheckout(directory); + + expect(result.error).toBeUndefined(); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite source checkout must be clean before acquisition evidence is recorded.\n', + ); + expect(result.stderr).not.toContain(untrackedPath); + expect(result.stderr).not.toContain('untracked-secret-name.txt'); + }); +}); diff --git a/src/performanceSuiteExistingOutputAtomicity.test.ts b/src/performanceSuiteExistingOutputAtomicity.test.ts new file mode 100644 index 00000000..063b0ce4 --- /dev/null +++ b/src/performanceSuiteExistingOutputAtomicity.test.ts @@ -0,0 +1,179 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const repositoryRoot = process.cwd(); +const suitePath = resolve(repositoryRoot, 'benchmarks/run-current-suite.mjs'); +const coreSuitePath = resolve( + repositoryRoot, + 'benchmarks/run-current-suite-core.mjs', +); +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function sha256(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +function existingOutputArguments({ + suite, + markdownInputPath, + markdownModulePath, + revisionInputPath, + revisionModulePath, + markdownModuleSource, + revisionModuleSource, + outputDirectory, +}: { + suite: string; + markdownInputPath: string; + markdownModulePath: string; + revisionInputPath: string; + revisionModulePath: string; + markdownModuleSource: string; + revisionModuleSource: string; + outputDirectory: string; +}) { + return [ + suite, + '--input', + markdownInputPath, + '--module', + markdownModulePath, + '--revision-input', + revisionInputPath, + '--revision-module', + revisionModulePath, + '--profile', + 'small', + '--samples', + '1', + '--source-commit-sha', + 'a'.repeat(40), + '--artifact-sha256', + sha256(markdownModuleSource), + '--revision-artifact-sha256', + sha256(revisionModuleSource), + '--runtime-id', + 'node-22.0.0', + '--reference-hardware-id', + `refhw-sha256-${'b'.repeat(64)}`, + '--output', + outputDirectory, + ]; +} + +function makeExistingOutputFixture(directory: string) { + const markdownInputPath = join(directory, 'input.md'); + const revisionInputPath = join(directory, 'document-envelope.json'); + const markdownModulePath = join(directory, 'markdown.mjs'); + const revisionModulePath = join(directory, 'revision.mjs'); + const outputDirectory = join(directory, 'evidence'); + const priorEvidencePath = join(outputDirectory, 'accepted-evidence.json'); + const markdownModuleSource = + "export function markdownToHtml(source) { return `

${source}

`; }\n"; + const revisionModuleSource = + "export async function createDocumentEnvelopeRevisionEvidenceBytes() { throw new Error('private downstream failure'); }\n"; + + writeFileSync(markdownInputPath, '# Existing evidence must stay immutable\n', 'utf8'); + writeFileSync( + revisionInputPath, + '{"contractVersion":1,"mode":"markdown","document":"# Existing evidence must stay immutable"}\n', + 'utf8', + ); + writeFileSync(markdownModulePath, markdownModuleSource, 'utf8'); + writeFileSync(revisionModulePath, revisionModuleSource, 'utf8'); + mkdirSync(outputDirectory); + writeFileSync(priorEvidencePath, '{"status":"accepted"}\n', 'utf8'); + + return { + markdownInputPath, + revisionInputPath, + markdownModulePath, + revisionModulePath, + outputDirectory, + priorEvidencePath, + markdownModuleSource, + revisionModuleSource, + }; +} + +function expectExistingEvidenceUntouched({ + outputDirectory, + priorEvidencePath, +}: { + outputDirectory: string; + priorEvidencePath: string; +}) { + expect(readFileSync(priorEvidencePath, 'utf8')).toBe( + '{"status":"accepted"}\n', + ); + expect(existsSync(join(outputDirectory, 'markdown'))).toBe(false); + expect(existsSync(join(outputDirectory, 'revision'))).toBe(false); +} + +describe('benchmark suite existing-output atomicity', () => { + it('rejects an existing evidence directory before mutating prior evidence', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-suite-existing-output-')); + temporaryDirectories.push(directory); + const fixture = makeExistingOutputFixture(directory); + + const result = spawnSync( + process.execPath, + existingOutputArguments({ suite: suitePath, ...fixture }), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite output directory must not already exist.\n', + ); + expectExistingEvidenceUntouched(fixture); + }); + + it('rejects an output directory created after the wrapper preflight', () => { + const directory = mkdtempSync(join(tmpdir(), 'inkspan-suite-core-existing-output-')); + temporaryDirectories.push(directory); + const fixture = makeExistingOutputFixture(directory); + + const result = spawnSync( + process.execPath, + existingOutputArguments({ suite: coreSuitePath, ...fixture }), + { + cwd: repositoryRoot, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr).toBe( + 'Benchmark suite output directory must not already exist.\n', + ); + expectExistingEvidenceUntouched(fixture); + }); +}); diff --git a/src/performanceSummaryInputPathPrivacyContract.test.ts b/src/performanceSummaryInputPathPrivacyContract.test.ts new file mode 100644 index 00000000..64ca6050 --- /dev/null +++ b/src/performanceSummaryInputPathPrivacyContract.test.ts @@ -0,0 +1,37 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +const summarizer = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +describe('benchmark summary input-path privacy contract', () => { + it('redacts filesystem details when input inspection crosses a non-directory', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-summary-input-path-')); + const privateSentinel = 'tenant-private-summary-input-parent'; + const blockedParent = join(root, privateSentinel); + const input = join(blockedParent, 'samples.json'); + const output = join(root, 'summary'); + + try { + writeFileSync(blockedParent, 'not a directory', 'utf8'); + + const result = spawnSync( + process.execPath, + [summarizer, '--input', input, '--output', output], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample input must be a regular file.', + ); + expect(result.stderr).not.toContain(privateSentinel); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/performanceSummaryProfileConsistencyContract.test.ts b/src/performanceSummaryProfileConsistencyContract.test.ts new file mode 100644 index 00000000..118346de --- /dev/null +++ b/src/performanceSummaryProfileConsistencyContract.test.ts @@ -0,0 +1,47 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const script = resolve(process.cwd(), 'benchmarks/summarize-samples.mjs'); + +describe('benchmark summary profile consistency contract', () => { + it('rejects sample evidence whose benchmarkId profile disagrees with documentProfile', () => { + const root = mkdtempSync(join(tmpdir(), 'inkspan-summary-profile-')); + const inputPath = join(root, 'samples.json'); + const outputDirectory = join(root, 'summary'); + + try { + writeFileSync( + inputPath, + `${JSON.stringify({ + contractVersion: 1, + benchmarkId: 'editor-input-small', + unit: 'ms', + sourceCommitSha: 'a'.repeat(40), + artifactSha256: 'b'.repeat(64), + documentProfile: 'large', + runtimeId: 'chromium-1.62.0', + referenceHardwareId: 'github-actions-ubuntu-24.04-x64', + samples: [70, 80, 90, 100], + })}\n`, + 'utf8', + ); + + const result = spawnSync( + process.execPath, + [script, '--input', inputPath, '--output', outputDirectory], + { cwd: process.cwd(), encoding: 'utf8' }, + ); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(''); + expect(result.stderr.trim()).toBe( + 'Benchmark sample profile must match documentProfile.', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/vite.demo.chunking.ts b/vite.demo.chunking.ts new file mode 100644 index 00000000..87cc2834 --- /dev/null +++ b/vite.demo.chunking.ts @@ -0,0 +1,54 @@ +const NODE_MODULES = '/node_modules/'; + +function hasPackage(moduleId: string, packageName: string): boolean { + return moduleId.includes(`${NODE_MODULES}${packageName}/`); +} + +/** + * Assign large demo-only dependency families to stable Rollup vendor chunks. + * Product/library entry points are unchanged; this only keeps the standalone + * buyer demo from regressing into one oversized JavaScript payload. + */ +export function demoVendorChunk(id: string): string | undefined { + const moduleId = id.replace(/\\/g, '/'); + + if (!moduleId.includes(NODE_MODULES)) { + return undefined; + } + + if ( + hasPackage(moduleId, 'react') || + hasPackage(moduleId, 'react-dom') || + hasPackage(moduleId, 'scheduler') + ) { + return 'react-vendor'; + } + + if ( + hasPackage(moduleId, '@tiptap/pm') || + moduleId.includes(`${NODE_MODULES}prosemirror-`) + ) { + return 'prosemirror-vendor'; + } + + if (moduleId.includes(`${NODE_MODULES}@tiptap/`)) { + return 'tiptap-vendor'; + } + + if ( + hasPackage(moduleId, 'marked') || + hasPackage(moduleId, 'turndown') || + hasPackage(moduleId, 'turndown-plugin-gfm') + ) { + return 'serialization-vendor'; + } + + if ( + hasPackage(moduleId, 'yjs') || + hasPackage(moduleId, 'y-prosemirror') + ) { + return 'collaboration-vendor'; + } + + return 'vendor'; +} diff --git a/vite.demo.config.ts b/vite.demo.config.ts index dd5e2d18..e5d2a0b6 100644 --- a/vite.demo.config.ts +++ b/vite.demo.config.ts @@ -1,6 +1,7 @@ import { resolve } from 'node:path'; import { defineConfig } from 'vite'; import react from '@vitejs/plugin-react'; +import { demoVendorChunk } from './vite.demo.chunking'; // Standalone demo build (Vite). `pnpm build:demo` emits a static site to // dist-demo/ that can be served by any static host or the provided Dockerfile. @@ -11,5 +12,10 @@ export default defineConfig({ build: { outDir: resolve(__dirname, 'dist-demo'), emptyOutDir: true, + rollupOptions: { + output: { + manualChunks: demoVendorChunk, + }, + }, }, });