diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml new file mode 100644 index 00000000..0f4729d9 --- /dev/null +++ b/.github/workflows/perf.yml @@ -0,0 +1,199 @@ +name: Performance regression + +on: + pull_request: + paths: + - ".github/workflows/perf.yml" + - "apps/**" + - "assets/**" + - "bun.lock" + - "contracts/**" + - "engine/Cargo.lock" + - "engine/Cargo.toml" + - "engine/core/**" + - "engine/crates/pocket-mod/**" + - "engine/crates/pocket-ui-surface/**" + - "engine/wasm/**" + - "framework/**" + - "hosts/web/**" + - "package.json" + - "pocket.config.ts" + - "pocket.json" + - "tests/perf-*.test.ts" + - "tools/build.ts" + - "tools/perf.ts" + - "tools/perf/**" + - "tools/test.ts" + - "tools/wasm.ts" + - "tsconfig.json" + - "vapor/**" + push: + branches: + - main + - "agent/**" + paths: + - ".github/workflows/perf.yml" + - "apps/**" + - "assets/**" + - "bun.lock" + - "contracts/**" + - "engine/Cargo.lock" + - "engine/Cargo.toml" + - "engine/core/**" + - "engine/crates/pocket-mod/**" + - "engine/crates/pocket-ui-surface/**" + - "engine/wasm/**" + - "framework/**" + - "hosts/web/**" + - "package.json" + - "pocket.config.ts" + - "pocket.json" + - "tests/perf-*.test.ts" + - "tools/build.ts" + - "tools/perf.ts" + - "tools/perf/**" + - "tools/test.ts" + - "tools/wasm.ts" + - "tsconfig.json" + - "vapor/**" + workflow_dispatch: {} + +permissions: + contents: read + +concurrency: + group: perf-${{ github.repository }}-${{ github.ref }} + cancel-in-progress: true + +env: + CI: "true" + TZ: UTC + LANG: C.UTF-8 + LC_ALL: C.UTF-8 + CARGO_TERM_COLOR: always + +jobs: + contracts: + name: Contracts and Native A/A + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + + - uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c + with: + toolchain: 1.93.0 + targets: wasm32-unknown-unknown + components: rustfmt + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Test performance protocol and executors + run: bun test tests/perf-*.test.ts + + - name: Generate clean-checkout build inputs + run: bun tools/build.ts hero + + - name: Typecheck + run: bunx tsc --noEmit + + - name: Test benchmark Rust harnesses + run: | + cargo test --locked --manifest-path tools/perf/guest/Cargo.toml + cargo fmt --check --manifest-path tools/perf/guest/Cargo.toml + cargo test --locked --manifest-path tools/perf/damage-fixture/Cargo.toml + cargo fmt --check --manifest-path tools/perf/damage-fixture/Cargo.toml + + - name: Compare two isolated Native runs + run: | + mkdir -p "$RUNNER_TEMP/pocketjs-perf/native" + bun perf local --base HEAD --executor native --suite quick \ + --format markdown \ + --out "$RUNNER_TEMP/pocketjs-perf/native/comparison.md" + cat "$RUNNER_TEMP/pocketjs-perf/native/comparison.md" >> "$GITHUB_STEP_SUMMARY" + + - name: Upload Native comparison + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: perf-native-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/pocketjs-perf/native/comparison.md + if-no-files-found: warn + retention-days: 7 + + qemu: + name: ${{ matrix.executor }} A/A + runs-on: ubuntu-24.04 + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + executor: + - qemu-armv7-thumb2 + - qemu-aarch64 + env: + PERF_ROOT: perf-results/${{ matrix.executor }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + + - uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c + with: + toolchain: 1.93.0 + targets: wasm32-unknown-unknown + components: rustfmt + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Build pinned QEMU runner + run: tools/perf/qemu/docker.sh build + + - name: Test QEMU marker and counter protocol + if: matrix.executor == 'qemu-armv7-thumb2' + run: docker run --rm pocketjs-perf-qemu:11.0.3 + + - name: Check executor prerequisites + run: | + mkdir -p "$PERF_ROOT" + bun perf doctor --json | tee "$PERF_ROOT/doctor.json" + + - name: Run and compare the quick suite twice + env: + PERF_EXECUTOR: ${{ matrix.executor }} + run: | + bun perf run --executor "$PERF_EXECUTOR" --suite quick \ + --out-dir "$PERF_ROOT/base" + bun perf run --executor "$PERF_EXECUTOR" --suite quick \ + --out-dir "$PERF_ROOT/candidate" + bun perf compare --base "$PERF_ROOT/base" --candidate "$PERF_ROOT/candidate" \ + --format markdown --out "$PERF_ROOT/comparison.md" + cat "$PERF_ROOT/comparison.md" >> "$GITHUB_STEP_SUMMARY" + + - name: Upload QEMU receipts and diagnostics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: perf-${{ matrix.executor }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ env.PERF_ROOT }}/doctor.json + ${{ env.PERF_ROOT }}/comparison.md + ${{ env.PERF_ROOT }}/**/*.receipt.json + ${{ env.PERF_ROOT }}/**/run.json + ${{ env.PERF_ROOT }}/**/*.log + if-no-files-found: warn + retention-days: 7 diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md new file mode 100644 index 00000000..02876722 --- /dev/null +++ b/docs/PERFORMANCE.md @@ -0,0 +1,201 @@ +# Local performance regression checks + +PocketJS compares a baseline checkout with the tracked contents of the current +checkout. The comparison uses versioned scenarios, input tapes, receipts and +budgets under `tools/perf/`. + +## Commands + +```sh +bun perf doctor +bun perf run --executor native --suite quick +bun perf run --executor qemu-armv7-thumb2 --suite quick +bun perf run --executor qemu-aarch64 --suite quick +bun perf compare --base --candidate +bun perf local --base +``` + +`doctor` reports missing tools and artifacts without installing or building +anything. Build the pinned QEMU image with: + +```sh +tools/perf/qemu/docker.sh build +``` + +`run` writes a `run.json` summary. QEMU writes one receipt per measured phase; +Native writes one scenario receipt with its measured phases aggregated. Pass +`--out-dir` to retain them at a known path. New summaries store receipt paths +relative to that output directory, so a complete run directory remains +portable; comparison also safely relocates absolute paths written by earlier +version 1 summaries. `compare` accepts either two receipt files or two +directories containing matching receipt identities. When a +directory tree contains `run.json`, comparison requires every summary to be +schema-valid and successful, requires baseline and candidate executor/suite +runs to match, and verifies that the summaries list every receipt exactly once. +An invalid summary, malformed receipt, missing listed receipt or unlisted stale +receipt makes the whole comparison `invalid`. A directory comparison requires +this inventory on both sides; use two receipt file paths when intentionally +comparing one receipt pair. + +`local` creates detached baseline and candidate worktrees below the system +temporary directory. It applies `git diff HEAD` to the candidate worktree, +then runs `bun install --frozen-lockfile` independently in each worktree. Bun's +global download cache may be shared, but baseline and candidate never share a +`node_modules` tree: each dependency graph comes from that snapshot's own +package manifest and lockfile. All generated artifacts remain inside the two +worktrees. The versioned workload sources under +`tools/perf/apps` are staged from the current benchmark harness into both +worktrees, so both revisions compile the same fixture against their own +framework and Core. **Other untracked files are neither copied into the +candidate nor included in its source hash.** Both temporary worktrees are +removed after the comparison. + +The benchmark guest and its diagnostic hooks come from the current harness. +The measured QuickJS, framework, `UiSurface`, Core, bundles and assets come from +each source worktree, so a baseline does not need to contain the performance +runner itself. + +## Executors + +Native runs the framework, `UiSurface`, WASM core and software renderer. It +provides deterministic correctness results, bundle and PAK sizes, plus host wall +time as a diagnostic. **Native wall time is not a regression gate.** The +versioned measurement host comes from the current checkout, while the measured +WASM, bundle and PAK come from the source worktree. + +Receipts record both the scenario's declared gate metrics and the subset that +an executor cannot observe. Native must declare every unavailable gate +explicitly. A Native comparison reports those metrics as unsupported only when +the selected budget does not apply to Native; an applicable Native budget with +no observation makes the comparison `invalid`. + +Each Native scenario runs in a fresh Bun process. This prevents framework or +oracle globals from one app changing a later scenario while retaining two +independent worlds for its correctness and measurement replays. + +QEMU uses pinned QEMU 11.0.3 linux-user and a plugin built from the same source. +A controlled guest syscall marks the beginning and end of each phase. Missing, +nested, mismatched or unclosed markers make the run `invalid`. The plugin counts: + +- dispatched guest instructions; +- dynamic guest instruction bytes; +- 16-bit and 32-bit instruction counts; +- successful guest load and store events. + +Every ARM and AArch64 invocation includes `-seed 1`, which fixes the entropy +QEMU supplies through the ELF `AT_RANDOM` auxiliary vector. **QEMU's seed does +not intercept a guest `getrandom(2)` syscall.** The QuickJS benchmark guest +therefore defines a benchmark-only `getrandom` symbol that supplies Rust +`RandomState` with a fixed, call-indexed nonzero byte stream. The plugin rejects +any raw `getrandom` syscall observed while a measurement marker is active, so a +future workload cannot silently restore entropy-dependent instruction paths. +The receipt profile records this as `seed-1+guest-shim-v1`; a profile change +requires a new baseline. + +The guest also records allocation calls, allocated bytes, current bytes, the +peak increase above the phase-start allocation level, and QuickJS live memory +after forced collection. Artifact collection records bundle, PAK and ELF +`.text + .rodata` sizes where the subject produces them. + +The QEMU guest preallocates one RGBA framebuffer and executes Core's +deterministic software rasterizer inside every measured frame. Its separate +correctness replay hashes every raw framebuffer into the same full-trace digest +as Native, and writes the declared raw framebuffer checkpoints and final frame +outside all markers. The host requires the full trace, each checkpoint and the +final frame to equal the Native replay. Hashing, file output and DevTools/effect +probes do not run inside a measurement interval. + +**These QEMU counters describe one executed instruction path.** They do not +represent cycles, cache traffic, power use or device frame rate. + +The ARMv7 executor uses: + +```text +Rust target: armv7-unknown-linux-gnueabihf +Rust flags: -C target-feature=+thumb-mode +QuickJS C: -mthumb -march=armv7-a -mfpu=vfpv3-d16 -mfloat-abi=hard +QEMU CPU: cortex-a9,neon=off,vfp-d32=off +``` + +The AArch64 executor fixes its QEMU CPU to `cortex-a53`. The ARMv7 reference +configuration does not expose NEON to the guest. The build configuration and +CPU model are recorded in every receipt; no ELF instruction-mode check is part +of the normal suite. ARMv7 Thumb-2 and AArch64 receipts are compared only with +receipts from the same executor, CPU model, toolchain, sysroot, build profile +and QEMU image. + +## Replay and correctness + +Every scenario has two executions. The correctness execution hashes the +framebuffer trace, DrawList, final state and effects. The measurement execution +replays the same input and excludes hash, PNG and diagnostic serialization from +the marked intervals. A replay mismatch makes the receipt `invalid` before any +metric threshold is considered. + +The quick suite contains framework startup and first frame for Solid, Vue Vapor +and Octane, an idle screen, fixed-size text updates, list mutation, paint and +layout updates, timers and animation, damage regions, touch input, a generated-C +Vapor reactive grid, and DeepZoom tile uploads. Scenario manifests carry a +total cold-run estimate, and the command rejects a suite above the configured +`--max-estimated-seconds` ceiling. The default ceiling is 1,500 seconds. + +For the generated-C Vapor grid, `correctness.framebuffer` is the full trace of +the character-and-palette presentation buffer. The guest produces that trace +after its generated state has matched the independent Vue Vapor oracle. + +## Budgets + +A relative threshold and its absolute floor must both be exceeded. Equality is +within budget. + +| Metric | Warning | Regression | +| --- | ---: | ---: | +| Guest instructions | >0.5% and >5,000 | >1% and >10,000 | +| Dynamic instruction bytes | >0.5% and >10 KiB | >1% and >20 KiB | +| Load and store events | >1% and >10,000 | >2% and >20,000 | +| Allocated bytes | >1% and >4 KiB | >2% and >8 KiB | +| QuickJS live bytes after GC | >1% and >32 KiB | >2% and >64 KiB | +| Bundle bytes | >1% and >2 KiB | >3% and >4 KiB | +| ELF `.text + .rodata` | >0.5% and >2 KiB | >1% and >4 KiB | + +The generated-C Vapor grid has a scenario-specific `memory.allocations` +`hardMax` of zero. Instruction-width distribution, separate load/store counts, +current and peak allocation gauges, and Native wall time remain diagnostics. +The budget schema rejects those diagnostic metrics, including in +scenario-specific overrides, so a custom budget cannot silently turn them into +regression gates. + +`params.gateMetrics` is the scenario's required regression-observation list. +Each gate must be a non-diagnostic catalog metric. A supported gate must be +present in the receipt and have a budget applicable to the receipt's executor +and scenario. An executor may instead declare the gate unsupported only when +the metric has a configured budget that does not apply to that executor. +Missing observations, silent omissions and missing budgets make a comparison +`invalid`. A metric outside `gateMetrics` is still checked when the receipt +emits it and the selected budget applies; this keeps shared counters such as +dynamic instruction bytes covered without requiring every scenario to repeat +the full metric catalog. + +`pass` means no applicable, supported budget was exceeded. The comparison and +receipts retain the explicit unsupported-gate list. `warn` reports a warning +threshold or an inconclusive sampled regression. `regression` reports a +conclusive regression or a hard-limit violation. `invalid` reports incomparable +provenance, correctness drift, missing required observations or protocol errors. +The command exits with 0 for `pass` and `warn`, 1 for `regression`, and 2 for +invalid input or execution. + +PPSSPP will use the same receipt and comparison formats after the deterministic +QEMU paths have completed local calibration. PSP receipts will compare only with +PSP baselines. + +## GitHub validation + +`.github/workflows/perf.yml` runs the versioned harness on an Ubuntu runner when +performance-sensitive paths change, and it can also be started manually. The +workflow runs the JavaScript, WASM-host and Rust harness checks plus an isolated +Native A/A comparison. Two parallel jobs independently build the pinned QEMU +image from its verified source archive; one verifies the marker/plugin fixtures, +and both run the complete quick suite twice for their ARMv7 Thumb-2 or AArch64 +target. **The workflow has read-only repository permissions and contains no +publish or deployment step.** Receipts, comparison reports and failure logs are +retained for 7 days. diff --git a/package.json b/package.json index 53900ec8..f2564017 100644 --- a/package.json +++ b/package.json @@ -142,6 +142,7 @@ "bootstrap": "bun tools/bootstrap.ts", "pocket": "bun tools/pocket.ts", "build": "bun tools/build.ts", + "perf": "bun tools/perf.ts", "play": "bun tools/play.ts", "widget": "bun tools/widget.ts", "widget:ipod": "bun tools/widget.ts --stage ipod", diff --git a/tests/perf-cli.test.ts b/tests/perf-cli.test.ts new file mode 100644 index 00000000..f325ea0d --- /dev/null +++ b/tests/perf-cli.test.ts @@ -0,0 +1,529 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { parsePerfCommand, UsageError } from "../tools/perf/cli/args.ts"; +import { comparePaths, comparisonExitCode } from "../tools/perf/cli/compare-paths.ts"; +import { runLocal } from "../tools/perf/cli/local.ts"; +import { runPerfCli } from "../tools/perf/cli/main.ts"; +import { nativeResultToReceipt, writeReceipt } from "../tools/perf/cli/receipts.ts"; +import type { PerfRunSummaryV1 } from "../tools/perf/cli/types.ts"; +import { parseScenarioV1, type ReceiptV1 } from "../tools/perf/core/index.ts"; +import type { NativeRunResult } from "../tools/perf/runner/native.ts"; + +const roots: string[] = []; + +function temporaryRoot(prefix = "pocketjs-perf-cli-test-"): string { + const root = mkdtempSync(join(tmpdir(), prefix)); + roots.push(root); + return root; +} + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function receipt(value: number, revision = "a".repeat(40)): ReceiptV1 { + return { + schemaVersion: 1, + kind: "pocketjs.perf.receipt", + createdAt: "2026-08-09T00:00:00.000Z", + status: "valid", + invalidReasons: [], + provenance: { + source: { revision, dirty: false, contentHash: "b".repeat(64) }, + scenario: { + id: "fixture.v1", + suite: "quick", + framework: "solid", + manifestHash: "9".repeat(64), + inputTapeHash: "c".repeat(64), + }, + toolchain: { rustc: "rustc fixture", cCompiler: "cc fixture", sysroot: "/fixture" }, + build: { + target: "fixture-target", + profile: "perf", + rustFlags: [], + cFlags: [], + linkerFlags: [], + }, + executor: { + id: "native", + version: "fixture", + profile: "host-diagnostic", + fingerprint: "a".repeat(64), + }, + binary: { sha256: value === 1 ? "d".repeat(64) : "e".repeat(64) }, + }, + correctness: { + framebufferHash: "1".repeat(64), + drawListHash: "2".repeat(64), + stateHash: "3".repeat(64), + effectHash: "4".repeat(64), + }, + gateMetrics: ["artifact.bundle_bytes"], + unsupportedMetrics: [], + metrics: { "artifact.bundle_bytes": { kind: "exact", value, unit: "bytes" } }, + }; +} + +function writeBudget(root: string): string { + const path = join(root, "budget.json"); + writeFileSync(path, JSON.stringify({ + schemaVersion: 1, + kind: "pocketjs.perf.budget-set", + id: "fixture", + metrics: { + "artifact.bundle_bytes": { + warn: { relative: 100, absolute: 100 }, + regression: { relative: 200, absolute: 200 }, + }, + }, + })); + return path; +} + +function git(root: string, ...args: string[]): string { + const result = Bun.spawnSync(["git", ...args], { cwd: root, stdout: "pipe", stderr: "pipe" }); + if (result.exitCode !== 0) throw new Error(result.stderr.toString()); + return result.stdout.toString().trim(); +} + +function localDependencyPackage(relativePath: string): string { + return `${JSON.stringify({ + name: "fixture", + private: true, + dependencies: { "fixture-dependency": `file:${relativePath}` }, + }, null, 2)}\n`; +} + +function localDependencyLock(relativePath: string): string { + return `{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "fixture", + "dependencies": { + "fixture-dependency": "file:${relativePath}", + }, + }, + }, + "packages": { + "fixture-dependency": ["fixture-dependency@file:${relativePath}", {}], + } +}\n`; +} + +function writeRunSummary( + root: string, + receipts: readonly string[], + overrides: Partial = {}, +): void { + const summary: PerfRunSummaryV1 = { + schemaVersion: 1, + kind: "pocketjs.perf.run", + status: "valid", + executor: "native", + suite: "quick", + sourceRoot: root, + outputDir: root, + receipts, + invalidReasons: [], + ...overrides, + }; + writeFileSync(join(root, "run.json"), `${JSON.stringify(summary, null, 2)}\n`); +} + +describe("perf CLI arguments", () => { + test("parses the four public commands and stable defaults", () => { + expect(parsePerfCommand(["doctor"])).toEqual({ command: "doctor", format: "text" }); + expect(parsePerfCommand(["run", "--executor", "qemu-armv7-thumb2", "--suite=quick"])).toMatchObject({ + command: "run", + executor: "qemu-armv7-thumb2", + suite: "quick", + maxEstimatedSeconds: 1_500, + }); + expect(parsePerfCommand(["compare", "--base", "a", "--candidate", "b"])).toMatchObject({ + command: "compare", + format: "json", + }); + expect(parsePerfCommand(["local", "--base", "main"])).toMatchObject({ + command: "local", + executors: ["native", "qemu-armv7-thumb2", "qemu-aarch64"], + suite: "quick", + }); + }); + + test("rejects unknown executors and options", () => { + expect(() => parsePerfCommand(["run", "--executor", "arm", "--suite", "quick"])).toThrow(UsageError); + expect(() => parsePerfCommand(["doctor", "--install=yes"])).toThrow("unknown option --install"); + }); + + test("help is available without touching an executor", async () => { + let output = ""; + const exit = await runPerfCli(["--help"], { + stdout(value) { output += value; }, + stderr() {}, + }); + expect(exit).toBe(0); + expect(output).toContain("bun perf doctor"); + expect(output).toContain("bun perf local --base "); + }); +}); + +describe("perf receipt comparison paths", () => { + test("compares both a receipt pair and receipt directories", () => { + const root = temporaryRoot(); + const budget = writeBudget(root); + const baseDir = join(root, "base"); + const candidateDir = join(root, "candidate"); + const base = writeReceipt(baseDir, receipt(1)); + const candidate = writeReceipt(candidateDir, receipt(2)); + + const single = comparePaths({ base, candidate, budgetPath: budget, format: "json" }); + expect(single.result.kind).toBe("pocketjs.perf.comparison"); + expect(single.result.status).toBe("pass"); + + const directory = comparePaths({ base: baseDir, candidate: candidateDir, budgetPath: budget, format: "markdown" }); + expect(directory.result.kind).toBe("pocketjs.perf.comparison-set"); + expect(directory.result.status).toBe("invalid"); + expect(directory.rendered).toContain("has no run.json summary"); + expect(directory.rendered).toContain("performance comparison set"); + + writeRunSummary(baseDir, [base]); + writeRunSummary(candidateDir, [candidate]); + const summarized = comparePaths({ base: baseDir, candidate: candidateDir, budgetPath: budget, format: "json" }); + expect(summarized.result.status).toBe("pass"); + }); + + test("makes failed run summaries invalidate the directory comparison and CLI", async () => { + const root = temporaryRoot(); + const budget = writeBudget(root); + const baseDir = join(root, "base"); + const candidateDir = join(root, "candidate"); + const base = writeReceipt(baseDir, receipt(1)); + const candidate = writeReceipt(candidateDir, receipt(1)); + writeRunSummary(baseDir, [base], { status: "invalid", invalidReasons: ["baseline fixture failed"] }); + writeRunSummary(candidateDir, [candidate], { status: "invalid", invalidReasons: ["candidate fixture failed"] }); + + const compared = comparePaths({ base: baseDir, candidate: candidateDir, budgetPath: budget, format: "json" }); + expect(compared.result.status).toBe("invalid"); + expect(comparisonExitCode(compared.result)).toBe(2); + expect(compared.rendered).toContain("baseline fixture failed"); + expect(compared.rendered).toContain("candidate fixture failed"); + + let stdout = ""; + let stderr = ""; + const exit = await runPerfCli([ + "compare", + "--base", baseDir, + "--candidate", candidateDir, + "--budget", budget, + ], { + stdout(value) { stdout += value; }, + stderr(value) { stderr += value; }, + }); + expect(exit).toBe(2); + expect(JSON.parse(stdout).status).toBe("invalid"); + expect(stderr).toBe(""); + }); + + test("rejects missing, omitted, and malformed run artifacts", () => { + const root = temporaryRoot(); + const budget = writeBudget(root); + + const missingBase = join(root, "missing-base"); + const missingCandidate = join(root, "missing-candidate"); + const unlistedBase = writeReceipt(missingBase, receipt(1)); + const unlistedCandidate = writeReceipt(missingCandidate, receipt(1)); + writeRunSummary(missingBase, [join(missingBase, "missing.receipt.json")]); + writeRunSummary(missingCandidate, [unlistedCandidate]); + const missing = comparePaths({ + base: missingBase, + candidate: missingCandidate, + budgetPath: budget, + format: "json", + }); + expect(missing.result.status).toBe("invalid"); + expect(missing.rendered).toContain("listed receipt does not exist"); + expect(missing.rendered).toContain("receipt is not listed by any run summary"); + expect(existsSync(unlistedBase)).toBe(true); + + const malformedBase = join(root, "malformed-base"); + const malformedCandidate = join(root, "malformed-candidate"); + mkdirSync(malformedBase); + const candidate = writeReceipt(malformedCandidate, receipt(1)); + const malformedReceipt = join(malformedBase, "broken.receipt.json"); + writeFileSync(malformedReceipt, "{ definitely not JSON\n"); + writeRunSummary(malformedBase, [malformedReceipt]); + writeRunSummary(malformedCandidate, [candidate]); + const malformed = comparePaths({ + base: malformedBase, + candidate: malformedCandidate, + budgetPath: budget, + format: "json", + }); + expect(malformed.result.status).toBe("invalid"); + expect(malformed.rendered).toContain("malformed receipt"); + + writeFileSync(join(malformedCandidate, "run.json"), JSON.stringify({ + schemaVersion: 1, + kind: "pocketjs.perf.run", + status: "valid", + executor: "native", + suite: "quick", + sourceRoot: malformedCandidate, + outputDir: malformedCandidate, + receipts: [candidate], + invalidReasons: [], + unexpected: true, + })); + const malformedSummary = comparePaths({ + base: malformedBase, + candidate: malformedCandidate, + budgetPath: budget, + format: "json", + }); + expect(malformedSummary.result.status).toBe("invalid"); + expect(malformedSummary.rendered).toContain("unknown fields: unexpected"); + }); + + test("requires run executor and suite metadata to match every listed receipt and peer run", () => { + const root = temporaryRoot(); + const budget = writeBudget(root); + const baseDir = join(root, "base"); + const candidateDir = join(root, "candidate"); + const base = writeReceipt(baseDir, receipt(1)); + const candidate = writeReceipt(candidateDir, receipt(1)); + writeRunSummary(baseDir, [base]); + writeRunSummary(candidateDir, [candidate], { executor: "qemu-aarch64" }); + + const compared = comparePaths({ base: baseDir, candidate: candidateDir, budgetPath: budget, format: "json" }); + expect(compared.result.status).toBe("invalid"); + expect(compared.rendered).toContain("does not match run summary executor"); + expect(compared.rendered).toContain("no run summary for native/quick"); + }); + + test("safely relocates legacy absolute receipt paths with a moved run directory", () => { + const root = temporaryRoot(); + const budget = writeBudget(root); + const originalBase = join(root, "original-base"); + const originalCandidate = join(root, "original-candidate"); + const base = writeReceipt(originalBase, receipt(1)); + const candidate = writeReceipt(originalCandidate, receipt(1)); + writeRunSummary(originalBase, [base]); + writeRunSummary(originalCandidate, [candidate]); + const movedBase = join(root, "moved-base"); + const movedCandidate = join(root, "moved-candidate"); + renameSync(originalBase, movedBase); + renameSync(originalCandidate, movedCandidate); + + const compared = comparePaths({ + base: movedBase, + candidate: movedCandidate, + budgetPath: budget, + format: "json", + }); + expect(compared.result.status).toBe("pass"); + }); + + test("turns the native DrawList observation into a schema-valid receipt digest", () => { + const scenario = parseScenarioV1(JSON.parse(readFileSync(join(import.meta.dir, "../tools/perf/scenarios/boot.json"), "utf8"))); + const native: NativeRunResult = { + schemaVersion: 1, + kind: "pocketjs.perf.native-result", + status: "ok", + scenarioId: scenario.id, + executor: "native", + sourceRoot: join(import.meta.dir, ".."), + correctness: { + framebufferTraceHash: "1".repeat(64), + finalFramebufferHash: "2".repeat(64), + drawListHash: "fnv1a64:0123456789abcdef", + stateHash: "3".repeat(64), + effectHash: "4".repeat(64), + checkpoints: {}, + }, + measurement: { + bootWallTimeNs: 1, + phases: [{ name: "first-frame", startFrame: 0, endFrame: 1, wallTimeNs: 2 }], + finalFramebufferHash: "2".repeat(64), + finalDrawListHash: "fnv1a64:0123456789abcdef", + }, + diagnosticMetrics: { + "native.wall_time_ns": { value: 2, unit: "ns" }, + }, + exactMetrics: { + "artifact.bundle_bytes": { value: 1, unit: "bytes" }, + }, + unsupportedMetrics: [ + "guest.instructions", + "quickjs.live_bytes_after_gc", + ], + }; + const converted = nativeResultToReceipt(native, scenario, join(import.meta.dir, "..")); + expect(converted.status).toBe("valid"); + expect(converted.correctness?.drawListHash).toMatch(/^[a-f0-9]{64}$/); + }); +}); + +describe("perf local isolation", () => { + test("stages benchmark apps without admitting unrelated untracked files", async () => { + const repo = temporaryRoot("pocketjs-perf-local-fixture-"); + git(repo, "init", "-b", "main"); + git(repo, "config", "user.name", "Perf Test"); + git(repo, "config", "user.email", "perf@example.invalid"); + git(repo, "config", "commit.gpgsign", "false"); + mkdirSync(join(repo, "vendor", "base-dependency"), { recursive: true }); + mkdirSync(join(repo, "vendor", "candidate-dependency"), { recursive: true }); + writeFileSync( + join(repo, "vendor", "base-dependency", "package.json"), + `${JSON.stringify({ name: "fixture-dependency", version: "1.0.0" }, null, 2)}\n`, + ); + writeFileSync( + join(repo, "vendor", "candidate-dependency", "package.json"), + `${JSON.stringify({ name: "fixture-dependency", version: "2.0.0" }, null, 2)}\n`, + ); + writeFileSync(join(repo, "package.json"), localDependencyPackage("vendor/base-dependency")); + writeFileSync(join(repo, "bun.lock"), localDependencyLock("vendor/base-dependency")); + writeFileSync(join(repo, "value.txt"), "1\n"); + git(repo, "add", "value.txt", "package.json", "bun.lock", "vendor"); + git(repo, "commit", "-m", "fixture"); + writeFileSync(join(repo, "value.txt"), "2\n"); + writeFileSync(join(repo, "package.json"), localDependencyPackage("vendor/candidate-dependency")); + writeFileSync(join(repo, "bun.lock"), localDependencyLock("vendor/candidate-dependency")); + mkdirSync(join(repo, "notes")); + writeFileSync(join(repo, "notes", "untracked.md"), "must stay outside snapshots\n"); + mkdirSync(join(repo, "tools", "perf", "apps"), { recursive: true }); + writeFileSync( + join(repo, "tools", "perf", "apps", "fixture-main.tsx"), + "export const fixture = 'harness workload';\n", + ); + const budget = writeBudget(repo); + const before = git(repo, "worktree", "list", "--porcelain"); + const observedRoots: string[] = []; + const observedModules: Array<{ realPath: string; version: string }> = []; + + const result = await runLocal({ + base: "HEAD", + executors: ["native"], + suite: "quick", + repoRoot: repo, + harnessRoot: repo, + scenarioDir: repo, + budgetPath: budget, + format: "json", + maxEstimatedSeconds: 10, + }, { + async runExecutor(options): Promise { + const sourceRoot = options.sourceRoot!; + const outDir = options.outDir!; + observedRoots.push(sourceRoot); + expect(existsSync(join(sourceRoot, "notes", "untracked.md"))).toBe(false); + const nodeModules = join(sourceRoot, "node_modules"); + expect(lstatSync(nodeModules).isSymbolicLink()).toBe(false); + observedModules.push({ + realPath: realpathSync(nodeModules), + version: JSON.parse(readFileSync( + join(nodeModules, "fixture-dependency", "package.json"), + "utf8", + )).version, + }); + expect(readFileSync( + join(sourceRoot, "tools", "perf", "apps", "fixture-main.tsx"), + "utf8", + )).toContain("harness workload"); + const value = Number(readFileSync(join(sourceRoot, "value.txt"), "utf8").trim()); + const path = writeReceipt(outDir, receipt(value, git(sourceRoot, "rev-parse", "HEAD"))); + writeRunSummary(outDir, [path], { + executor: options.executor, + suite: options.suite, + sourceRoot, + outputDir: outDir, + }); + return { + schemaVersion: 1, + kind: "pocketjs.perf.run", + status: "valid", + executor: options.executor, + suite: options.suite, + sourceRoot, + outputDir: outDir, + receipts: [path], + invalidReasons: [], + }; + }, + }); + + expect(result.result.status).toBe("pass"); + expect(observedRoots).toHaveLength(2); + expect(observedModules.map((item) => item.version)).toEqual(["1.0.0", "2.0.0"]); + expect(observedModules[0]!.realPath).not.toBe(observedModules[1]!.realPath); + expect(observedRoots.every((path) => !existsSync(path))).toBe(true); + expect(git(repo, "worktree", "list", "--porcelain")).toBe(before); + expect(readFileSync(join(repo, "value.txt"), "utf8")).toBe("2\n"); + expect(readFileSync(join(repo, "notes", "untracked.md"), "utf8")).toContain("outside snapshots"); + }); + + test("returns structured invalid and cleans worktrees when a frozen install fails", async () => { + const repo = temporaryRoot("pocketjs-perf-local-install-failure-"); + git(repo, "init", "-b", "main"); + git(repo, "config", "user.name", "Perf Test"); + git(repo, "config", "user.email", "perf@example.invalid"); + git(repo, "config", "commit.gpgsign", "false"); + mkdirSync(join(repo, "vendor", "base-dependency"), { recursive: true }); + mkdirSync(join(repo, "vendor", "candidate-dependency"), { recursive: true }); + writeFileSync( + join(repo, "vendor", "base-dependency", "package.json"), + `${JSON.stringify({ name: "fixture-dependency", version: "1.0.0" })}\n`, + ); + writeFileSync( + join(repo, "vendor", "candidate-dependency", "package.json"), + `${JSON.stringify({ name: "fixture-dependency", version: "2.0.0" })}\n`, + ); + writeFileSync(join(repo, "package.json"), localDependencyPackage("vendor/base-dependency")); + writeFileSync(join(repo, "bun.lock"), localDependencyLock("vendor/base-dependency")); + git(repo, "add", "package.json", "bun.lock", "vendor"); + git(repo, "commit", "-m", "fixture"); + // Change only the manifest. A non-frozen install could silently rewrite + // the candidate lockfile; perf local must instead reject this snapshot. + writeFileSync(join(repo, "package.json"), localDependencyPackage("vendor/candidate-dependency")); + const before = git(repo, "worktree", "list", "--porcelain"); + let executorCalls = 0; + + const result = await runLocal({ + base: "HEAD", + executors: ["native"], + suite: "quick", + repoRoot: repo, + harnessRoot: repo, + scenarioDir: repo, + format: "json", + maxEstimatedSeconds: 10, + }, { + async runExecutor(): Promise { + executorCalls += 1; + throw new Error("executor must not run after dependency installation fails"); + }, + }); + + expect(result.result.kind).toBe("pocketjs.perf.local"); + expect(result.result.status).toBe("invalid"); + if (result.result.kind !== "pocketjs.perf.local") throw new Error("expected a local invalid result"); + expect(result.result.invalidReasons.join("\n")).toContain("candidate dependency install failed"); + expect(result.result.temporaryWorktreesCleaned).toBe(true); + expect(executorCalls).toBe(0); + expect(git(repo, "worktree", "list", "--porcelain")).toBe(before); + }); +}); diff --git a/tests/perf-comparator.test.ts b/tests/perf-comparator.test.ts new file mode 100644 index 00000000..e9aebe57 --- /dev/null +++ b/tests/perf-comparator.test.ts @@ -0,0 +1,662 @@ +import { describe, expect, test } from "bun:test"; +import { + buildRenderConfig, + DEFAULT_BUDGET_SET, + METRIC_CATALOG, + SchemaValidationError, + compareReceipts, + comparisonToJson, + comparisonToMarkdown, + parseBudgetSetV1, + parseComparisonV1, + parseInputTapeV1, + parseReceiptV1, + parseScenarioV1, + safeParseReceiptV1, + safeParseScenarioV1, + withHardLimits, + type BudgetSetV1, + type InputTapeV1, + type MetricSampleV1, + type ReceiptV1, + type ScenarioV1, +} from "../tools/perf/core/index.ts"; + +const HASH = { + content: "1".repeat(64), + tape: "2".repeat(64), + manifest: "9".repeat(64), + binary: "3".repeat(64), + framebuffer: "4".repeat(64), + drawList: "5".repeat(64), + state: "6".repeat(64), + effect: "7".repeat(64), +}; + +function exact(value: number, unit: "count" | "bytes" | "ns" = "count"): MetricSampleV1 { + return { kind: "exact", value, unit }; +} + +function sampled(samples: readonly number[], unit: "count" | "bytes" | "ns" = "count"): MetricSampleV1 { + return { kind: "sampled", samples, unit }; +} + +function receipt( + metrics: Readonly> = { + "guest.instructions": exact(100), + }, + options: { + revision?: string; + binary?: string; + executorId?: string; + executorProfile?: string; + executorFingerprint?: string; + buildProfile?: string; + rustc?: string; + scenarioId?: string; + gateMetrics?: readonly string[]; + unsupportedMetrics?: readonly string[]; + } = {}, +): ReceiptV1 { + return { + schemaVersion: 1, + kind: "pocketjs.perf.receipt", + createdAt: "2026-08-09T12:00:00.000Z", + status: "valid", + invalidReasons: [], + provenance: { + source: { + revision: options.revision ?? "base", + dirty: options.revision === "candidate", + contentHash: HASH.content, + }, + scenario: { + id: options.scenarioId ?? "startup-solid", + suite: "quick", + framework: "solid", + manifestHash: HASH.manifest, + inputTapeHash: HASH.tape, + }, + toolchain: { + rustc: options.rustc ?? "rustc 1.91.0", + cCompiler: "clang 21.0.0", + sysroot: "sysroot-sha256:fixture", + qemu: "11.0.3", + }, + build: { + target: "armv7-unknown-linux-gnueabihf", + profile: options.buildProfile ?? "release-perf", + rustFlags: ["-C", "target-feature=+thumb-mode"], + cFlags: ["-mthumb", "-march=armv7-a"], + linkerFlags: ["-mthumb"], + }, + executor: { + id: options.executorId ?? "qemu-armv7-thumb2", + version: "11.0.3", + profile: options.executorProfile ?? "armv7a-thumb2-vfpv3-d16-hardfloat", + fingerprint: options.executorFingerprint ?? "a".repeat(64), + }, + binary: { sha256: options.binary ?? HASH.binary }, + }, + correctness: { + framebufferHash: HASH.framebuffer, + drawListHash: HASH.drawList, + stateHash: HASH.state, + effectHash: HASH.effect, + }, + gateMetrics: options.gateMetrics ?? ["guest.instructions"], + unsupportedMetrics: options.unsupportedMetrics ?? [], + metrics, + }; +} + +function candidate( + metrics: Readonly>, + options: Parameters[1] = {}, +): ReceiptV1 { + return receipt(metrics, { + revision: "candidate", + binary: "8".repeat(64), + ...options, + }); +} + +function instructionBudget( + metricBudget: BudgetSetV1["metrics"][string] = { + warn: { relative: 0.05, absolute: 10 }, + regression: { relative: 0.1, absolute: 20 }, + }, +): BudgetSetV1 { + return { + schemaVersion: 1, + kind: "pocketjs.perf.budget-set", + id: "instruction-test-v1", + metrics: { "guest.instructions": metricBudget }, + }; +} + +const inputTape: InputTapeV1 = { + schemaVersion: 1, + kind: "pocketjs.perf.input-tape", + id: "all-input-kinds", + frames: 4, + tracks: [ + { kind: "button", control: "confirm", samples: [{ frame: 0, pressed: true }, { frame: 1, pressed: false }] }, + { kind: "analog", control: "primary-x", samples: [{ frame: 0, value: 0 }, { frame: 2, value: 0.5 }] }, + { kind: "touch", control: "primary", samples: [{ frame: 1, phase: "start", x: 12, y: 18 }, { frame: 3, phase: "end", x: 12, y: 18 }] }, + { kind: "relative-axis", control: "scroll-y", samples: [{ frame: 2, delta: 3 }] }, + { kind: "effect", effect: "tile-upload", samples: [{ frame: 3, value: { tile: 4 } }] }, + ], +}; + +const scenario: ScenarioV1 = { + schemaVersion: 1, + kind: "pocketjs.perf.scenario", + id: "all-input-scenario", + suite: "quick", + subject: { + id: "fixture", + family: "input", + framework: "core", + entry: "fixtures/input.ts", + }, + executorRequirements: ["native", "qemu-armv7-thumb2"], + frames: 4, + tape: inputTape, + phases: [ + { name: "setup", startFrame: 0, endFrame: 1, collect: false }, + { name: "measure", startFrame: 1, endFrame: 4, collect: true }, + ], + checkpoints: [{ frame: 3, capture: ["framebuffer", "drawList", "state", "effects"] }], + params: { repetitions: 1, labels: ["deterministic"] }, +}; + +describe("perf v1 schemas", () => { + test("strictly validates input tapes and scenarios", () => { + expect(parseInputTapeV1(inputTape)).toBe(inputTape); + expect(parseScenarioV1(scenario)).toBe(scenario); + + const extra = structuredClone(scenario) as any; + extra.subject.device = "vita"; + expect(() => parseScenarioV1(extra)).toThrow("unexpected property"); + + const unsorted = structuredClone(inputTape) as any; + unsorted.tracks[0].samples[1].frame = 0; + expect(() => parseInputTapeV1(unsorted)).toThrow("strictly increasing"); + + const wrongFrames = structuredClone(scenario) as any; + wrongFrames.frames = 5; + expect(() => parseScenarioV1(wrongFrames)).toThrow("must equal"); + + const diagnosticGate = structuredClone(scenario) as any; + diagnosticGate.params.gateMetrics = ["memory.current_bytes"]; + expect(() => parseScenarioV1(diagnosticGate)).toThrow("diagnostic metrics cannot be regression gates"); + + const duplicateGate = structuredClone(scenario) as any; + duplicateGate.params.gateMetrics = ["guest.instructions", "guest.instructions"]; + expect(() => parseScenarioV1(duplicateGate)).toThrow("duplicate gate metric"); + }); + + test("strictly resolves the shared viewport, density and render-scale contract", () => { + expect(buildRenderConfig(scenario.params)).toEqual({ + width: 480, + height: 272, + rasterDensity: 1, + renderScale: 1, + }); + + const custom = structuredClone(scenario) as any; + custom.params.viewport = { + width: 320, + height: 180, + rasterDensity: 3, + renderScale: 4, + }; + const parsed = parseScenarioV1(custom); + expect(buildRenderConfig(parsed.params)).toEqual(custom.params.viewport); + + for (const [field, value] of [ + ["width", 1.5], + ["width", 32_001], + ["height", 0], + ["rasterDensity", 256], + ["renderScale", 5], + ] as const) { + const invalid = structuredClone(scenario) as any; + invalid.params.viewport = { [field]: value }; + expect(() => parseScenarioV1(invalid), `${field}=${value}`).toThrow("expected an integer"); + } + + const unknown = structuredClone(scenario) as any; + unknown.params.viewport = { width: 320, deviceScale: 2 }; + const result = safeParseScenarioV1(unknown); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error.issues).toContainEqual({ + path: "/params/viewport/deviceScale", + message: "unexpected property", + }); + } + }); + + test("rejects wrong versions, unknown properties, non-finite values and unknown metrics", () => { + const wrongVersion = structuredClone(receipt()) as any; + wrongVersion.schemaVersion = 2; + expect(() => parseReceiptV1(wrongVersion)).toThrow('expected 1'); + + const extra = structuredClone(receipt()) as any; + extra.provenance.executor.device = "specific-hardware"; + const parsed = safeParseReceiptV1(extra); + expect(parsed.success).toBe(false); + if (!parsed.success) { + expect(parsed.error).toBeInstanceOf(SchemaValidationError); + expect(parsed.error.issues).toContainEqual({ + path: "/provenance/executor/device", + message: "unexpected property", + }); + } + + const infinity = structuredClone(receipt()) as any; + infinity.metrics["guest.instructions"].value = Number.POSITIVE_INFINITY; + expect(() => parseReceiptV1(infinity)).toThrow("finite number"); + + const unknownMetric = structuredClone(receipt()) as any; + unknownMetric.metrics["custom.metric"] = exact(1); + expect(() => parseReceiptV1(unknownMetric)).toThrow("unknown metric id"); + }); + + test("requires every valid receipt gate to be observed or explicitly unsupported", () => { + const missing = structuredClone(receipt()) as any; + missing.metrics = { "native.wall_time_ns": exact(1, "ns") }; + expect(() => parseReceiptV1(missing)).toThrow("neither observed nor explicitly unsupported"); + + missing.unsupportedMetrics = ["guest.instructions"]; + expect(parseReceiptV1(missing).unsupportedMetrics).toEqual(["guest.instructions"]); + + missing.metrics["guest.instructions"] = exact(1); + expect(() => parseReceiptV1(missing)).toThrow("both observed and unsupported"); + }); + + test("enforces valid/invalid receipt invariants and sampled observations", () => { + const invalid = structuredClone(receipt()) as any; + invalid.status = "invalid"; + invalid.invalidReasons = ["measurement marker was not closed"]; + invalid.correctness = null; + invalid.metrics = {}; + expect(parseReceiptV1(invalid).status).toBe("invalid"); + + invalid.invalidReasons = []; + expect(() => parseReceiptV1(invalid)).toThrow("must contain a reason"); + + const emptySamples = structuredClone(receipt({ + "guest.instructions": sampled([]), + })); + expect(() => parseReceiptV1(emptySamples)).toThrow("at least one observation"); + }); + + test("validates budgets and publishes the planned defaults", () => { + expect(parseBudgetSetV1(DEFAULT_BUDGET_SET)).toBe(DEFAULT_BUDGET_SET); + expect(DEFAULT_BUDGET_SET.metrics["guest.instructions"]).toMatchObject({ + warn: { relative: 0.005, absolute: 5_000 }, + regression: { relative: 0.01, absolute: 10_000 }, + }); + expect(DEFAULT_BUDGET_SET.metrics["guest.instruction_bytes"]).toMatchObject({ + warn: { relative: 0.005, absolute: 10 * 1024 }, + regression: { relative: 0.01, absolute: 20 * 1024 }, + }); + expect(DEFAULT_BUDGET_SET.metrics["guest.load_store_events"]).toMatchObject({ + warn: { relative: 0.01, absolute: 10_000 }, + regression: { relative: 0.02, absolute: 20_000 }, + }); + expect(DEFAULT_BUDGET_SET.metrics["guest.loads"]).toBeUndefined(); + expect(DEFAULT_BUDGET_SET.metrics["guest.stores"]).toBeUndefined(); + expect(DEFAULT_BUDGET_SET.metrics["memory.current_bytes"]).toBeUndefined(); + expect(DEFAULT_BUDGET_SET.metrics["memory.peak_bytes"]).toBeUndefined(); + expect(DEFAULT_BUDGET_SET.metrics["artifact.elf_text_rodata_bytes"]).toMatchObject({ + warn: { relative: 0.005, absolute: 2 * 1024 }, + regression: { relative: 0.01, absolute: 4 * 1024 }, + }); + expect(DEFAULT_BUDGET_SET.scenarios?.["vapor.todo.reactive-grid.v1"]?.["memory.allocations"]) + .toMatchObject({ hardMax: 0 }); + + const partialThreshold = structuredClone(instructionBudget()) as any; + delete partialThreshold.metrics["guest.instructions"].warn.absolute; + expect(() => parseBudgetSetV1(partialThreshold)).toThrow("required property is missing"); + + const backwards = instructionBudget({ + warn: { relative: 0.1, absolute: 20 }, + regression: { relative: 0.05, absolute: 10 }, + }); + expect(() => parseBudgetSetV1(backwards)).toThrow("greater than or equal"); + + for (const diagnostic of [ + "guest.thumb16_instructions", + "guest.thumb32_instructions", + "guest.loads", + "guest.stores", + "memory.current_bytes", + "memory.peak_bytes", + "native.wall_time_ns", + ]) { + const globalBudget = structuredClone(instructionBudget()) as any; + globalBudget.metrics[diagnostic] = { hardMax: 1 }; + expect(() => parseBudgetSetV1(globalBudget)).toThrow( + "diagnostic metrics cannot have regression budgets", + ); + + const scenarioBudget = structuredClone(instructionBudget()) as any; + scenarioBudget.scenarios = { "fixture.v1": { [diagnostic]: { hardMax: 1 } } }; + expect(() => parseBudgetSetV1(scenarioBudget)).toThrow( + "diagnostic metrics cannot have regression budgets", + ); + } + }); +}); + +describe("metric catalog", () => { + test("defines direction, kind and unit, including diagnostics", () => { + expect(METRIC_CATALOG["guest.instructions"]).toMatchObject({ + direction: "lower-is-better", + kind: "counter", + unit: "count", + diagnostic: false, + }); + expect(METRIC_CATALOG["guest.thumb16_instructions"].diagnostic).toBe(true); + expect(METRIC_CATALOG["guest.thumb32_instructions"].diagnostic).toBe(true); + expect(METRIC_CATALOG["guest.loads"].unit).toBe("count"); + expect(METRIC_CATALOG["guest.stores"].unit).toBe("count"); + expect(METRIC_CATALOG["guest.loads"].diagnostic).toBe(true); + expect(METRIC_CATALOG["guest.stores"].diagnostic).toBe(true); + expect(METRIC_CATALOG["memory.current_bytes"].diagnostic).toBe(true); + expect(METRIC_CATALOG["memory.peak_bytes"].diagnostic).toBe(true); + expect(METRIC_CATALOG["memory.allocations"].diagnostic).toBe(false); + expect(METRIC_CATALOG["native.wall_time_ns"]).toMatchObject({ unit: "ns", diagnostic: true }); + }); +}); + +describe("receipt comparison", () => { + test("requires relative and absolute thresholds together and treats equality as within budget", () => { + const budget = instructionBudget(); + expect(compareReceipts(receipt(), candidate({ "guest.instructions": exact(110) }), budget).status).toBe("pass"); + expect(compareReceipts(receipt(), candidate({ "guest.instructions": exact(111) }), budget).status).toBe("warn"); + expect(compareReceipts(receipt(), candidate({ "guest.instructions": exact(119) }), budget).status).toBe("warn"); + expect(compareReceipts(receipt(), candidate({ "guest.instructions": exact(120) }), budget).status).toBe("warn"); + expect(compareReceipts(receipt(), candidate({ "guest.instructions": exact(121) }), budget).status).toBe("regression"); + expect(compareReceipts(receipt(), candidate({ "guest.instructions": exact(80) }), budget).status).toBe("pass"); + }); + + test("uses the absolute half of conjunction when the baseline is zero", () => { + const base = receipt({ "guest.instructions": exact(0) }); + const budget = instructionBudget(); + const below = compareReceipts(base, candidate({ "guest.instructions": exact(9) }), budget); + expect(below.status).toBe("pass"); + expect(below.metrics[0]?.relativeDelta).toBeNull(); + expect(compareReceipts(base, candidate({ "guest.instructions": exact(10) }), budget).status).toBe("pass"); + expect(compareReceipts(base, candidate({ "guest.instructions": exact(11) }), budget).status).toBe("warn"); + expect(compareReceipts(base, candidate({ "guest.instructions": exact(20) }), budget).status).toBe("warn"); + expect(compareReceipts(base, candidate({ "guest.instructions": exact(21) }), budget).status).toBe("regression"); + }); + + test("applies hardMax and hardMin as strict absolute bounds", () => { + const maxBudget = instructionBudget({ hardMax: 100 }); + expect(compareReceipts(receipt(), candidate({ "guest.instructions": exact(100) }), maxBudget).status).toBe("pass"); + expect(compareReceipts(receipt(), candidate({ "guest.instructions": exact(101) }), maxBudget).status).toBe("regression"); + + const minBudget = instructionBudget({ hardMin: 100 }); + expect(compareReceipts(receipt(), candidate({ "guest.instructions": exact(100) }), minBudget).status).toBe("pass"); + expect(compareReceipts(receipt(), candidate({ "guest.instructions": exact(99) }), minBudget).status).toBe("regression"); + + const zeroAllocation = withHardLimits({ + schemaVersion: 1, + kind: "pocketjs.perf.budget-set", + id: "zero-allocation", + metrics: { "memory.allocations": { hardMax: 2 } }, + }, "memory.allocations", { hardMax: 0 }); + expect(zeroAllocation.metrics["memory.allocations"]?.hardMax).toBe(0); + + const vaporOptions = { + scenarioId: "vapor.todo.reactive-grid.v1#settle", + gateMetrics: ["memory.allocations"], + }; + const vaporBudget: BudgetSetV1 = { + schemaVersion: 1, + kind: "pocketjs.perf.budget-set", + id: "scenario-hard-max", + metrics: { "artifact.bundle_bytes": { hardMax: 1, executors: ["native"] } }, + scenarios: { + "vapor.todo.reactive-grid.v1": { + "memory.allocations": { hardMax: 0, executors: ["qemu-armv7-thumb2"] }, + }, + }, + }; + const zero = compareReceipts( + receipt({ "memory.allocations": exact(0) }, vaporOptions), + candidate({ "memory.allocations": exact(0) }, vaporOptions), + vaporBudget, + ); + expect(zero.status).toBe("pass"); + expect(zero.metrics.find((metric) => metric.id === "memory.allocations")?.budget?.hardMax).toBe(0); + const allocated = compareReceipts( + receipt({ "memory.allocations": exact(0) }, vaporOptions), + candidate({ "memory.allocations": exact(1) }, vaporOptions), + vaporBudget, + ); + expect(allocated.status).toBe("regression"); + }); + + test("marks missing and incompatible metric samples invalid", () => { + const budget = instructionBudget(); + const missing = compareReceipts( + receipt({ "guest.instructions": exact(100), "memory.current_bytes": exact(2, "bytes") }), + candidate({ "guest.instructions": exact(100) }), + budget, + ); + expect(missing.status).toBe("invalid"); + expect(missing.reasons.some((reason) => reason.code === "metric-missing")).toBe(true); + + const units = compareReceipts( + receipt({ "guest.instructions": exact(100, "count") }), + candidate({ "guest.instructions": exact(100, "bytes") }), + budget, + ); + expect(units.status).toBe("invalid"); + expect(units.reasons[0]?.code).toBe("unit-mismatch"); + + const catalogUnits = compareReceipts( + receipt({ "guest.instructions": exact(100, "bytes") }), + candidate({ "guest.instructions": exact(110, "bytes") }), + budget, + ); + expect(catalogUnits.status).toBe("invalid"); + expect(catalogUnits.reasons[0]?.code).toBe("catalog-unit-mismatch"); + }); + + test("invalidates a declared gate without an applicable budget", () => { + const noInstructionBudget: BudgetSetV1 = { + schemaVersion: 1, + kind: "pocketjs.perf.budget-set", + id: "unrelated-budget", + metrics: { "artifact.bundle_bytes": { hardMax: 1_000_000 } }, + }; + const comparison = compareReceipts( + receipt(), + candidate({ "guest.instructions": exact(100) }), + noInstructionBudget, + ); + expect(comparison.status).toBe("invalid"); + expect(comparison.reasons).toContainEqual(expect.objectContaining({ code: "budget-missing" })); + }); + + test("applies configured budgets to emitted metrics outside the required gate list", () => { + const base = receipt({ + "guest.instructions": exact(100), + "guest.instruction_bytes": exact(100_000, "bytes"), + }); + const next = candidate({ + "guest.instructions": exact(100), + "guest.instruction_bytes": exact(130_001, "bytes"), + }); + const comparison = compareReceipts(base, next); + expect(comparison.status).toBe("regression"); + expect(comparison.metrics.find((metric) => metric.id === "guest.instruction_bytes")) + .toMatchObject({ status: "regression" }); + }); + + test("keeps Native unsupported gates explicit and rejects an applicable gate budget", () => { + const nativeOptions = { + executorId: "native", + executorProfile: "host-diagnostic", + gateMetrics: ["guest.instructions"], + unsupportedMetrics: ["guest.instructions"], + }; + const base = receipt({ "native.wall_time_ns": exact(10, "ns") }, nativeOptions); + const next = candidate({ "native.wall_time_ns": exact(20, "ns") }, nativeOptions); + const qemuOnly = instructionBudget({ + warn: { relative: 0.05, absolute: 10 }, + regression: { relative: 0.1, absolute: 20 }, + executors: ["qemu-armv7-thumb2", "qemu-aarch64"], + }); + const diagnostic = compareReceipts(base, next, qemuOnly); + expect(diagnostic.status).toBe("pass"); + expect(diagnostic.unsupportedMetrics).toEqual(["guest.instructions"]); + expect(comparisonToMarkdown(diagnostic)).toContain("Unsupported gates for this executor"); + + const omitted: BudgetSetV1 = { + schemaVersion: 1, + kind: "pocketjs.perf.budget-set", + id: "omitted-native-gate", + metrics: { "artifact.bundle_bytes": { hardMax: 1_000_000 } }, + }; + const missingBudget = compareReceipts(base, next, omitted); + expect(missingBudget.status).toBe("invalid"); + expect(missingBudget.reasons).toContainEqual(expect.objectContaining({ + code: "budget-missing", + })); + + const required = compareReceipts(base, next, instructionBudget()); + expect(required.status).toBe("invalid"); + expect(required.reasons).toContainEqual(expect.objectContaining({ + code: "metric-support-mismatch", + })); + }); + + test("rejects executor, profile and toolchain mismatches but permits different source identities", () => { + const budget = instructionBudget(); + const base = receipt(); + const sourceChange = compareReceipts(base, candidate({ "guest.instructions": exact(100) }), budget); + expect(sourceChange.status).toBe("pass"); + expect(sourceChange.comparable).toBe(true); + + const executor = compareReceipts(base, candidate({ "guest.instructions": exact(100) }, { executorId: "qemu-aarch64" }), budget); + expect(executor.status).toBe("invalid"); + expect(executor.reasons.map((reason) => reason.path)).toContain("/provenance/executor/id"); + + const executorProfile = compareReceipts(base, candidate({ "guest.instructions": exact(100) }, { executorProfile: "different-profile" }), budget); + expect(executorProfile.status).toBe("invalid"); + expect(executorProfile.reasons.map((reason) => reason.path)).toContain("/provenance/executor/profile"); + + const executorFingerprint = compareReceipts(base, candidate( + { "guest.instructions": exact(100) }, + { executorFingerprint: "b".repeat(64) }, + ), budget); + expect(executorFingerprint.status).toBe("invalid"); + expect(executorFingerprint.reasons.map((reason) => reason.path)) + .toContain("/provenance/executor/fingerprint"); + + const buildProfile = compareReceipts(base, candidate({ "guest.instructions": exact(100) }, { buildProfile: "debug" }), budget); + expect(buildProfile.status).toBe("invalid"); + expect(buildProfile.reasons.map((reason) => reason.path)).toContain("/provenance/build/profile"); + + const toolchain = compareReceipts(base, candidate({ "guest.instructions": exact(100) }, { rustc: "rustc 1.92.0" }), budget); + expect(toolchain.status).toBe("invalid"); + expect(toolchain.reasons.map((reason) => reason.path)).toContain("/provenance/toolchain"); + }); + + test("rejects correctness changes and invalid execution receipts", () => { + const changed = structuredClone(candidate({ "guest.instructions": exact(100) })) as any; + changed.correctness.stateHash = "9".repeat(64); + const correctness = compareReceipts(receipt(), changed, instructionBudget()); + expect(correctness.status).toBe("invalid"); + expect(correctness.reasons[0]?.code).toBe("correctness-mismatch"); + + const invalid = structuredClone(changed) as any; + invalid.status = "invalid"; + invalid.invalidReasons = ["measurement marker order was invalid"]; + invalid.correctness = null; + const execution = compareReceipts(receipt(), invalid, instructionBudget()); + expect(execution.status).toBe("invalid"); + expect(execution.reasons.some((reason) => reason.code === "receipt-invalid")).toBe(true); + }); + + test("keeps paired raw samples and emits a deterministic bootstrap interval", () => { + const base = receipt({ "guest.instructions": sampled([100, 102, 98, 100]) }); + const next = candidate({ "guest.instructions": sampled([111, 112, 110, 111]) }); + const first = compareReceipts(base, next, instructionBudget()); + const second = compareReceipts(base, next, instructionBudget()); + expect(first.status).toBe("warn"); + expect(first.metrics[0]).toMatchObject({ + baseline: 100, + candidate: 111, + delta: 11, + sampleKind: "paired", + sampleCount: 4, + confidenceInterval: { + level: 0.95, + method: "paired-bootstrap", + iterations: 2_000, + }, + }); + expect(first.metrics[0]?.confidenceInterval).toEqual(second.metrics[0]?.confidenceInterval); + + const conclusive = compareReceipts( + receipt({ "guest.instructions": sampled([100, 100, 100, 100]) }), + candidate({ "guest.instructions": sampled([121, 121, 121, 121]) }), + instructionBudget(), + ); + expect(conclusive.status).toBe("regression"); + + const noisy = compareReceipts( + receipt({ "guest.instructions": sampled([100, 100, 100, 100]) }), + candidate({ "guest.instructions": sampled([80, 80, 162, 162]) }), + instructionBudget(), + ); + expect(noisy.metrics[0]).toMatchObject({ candidate: 121, status: "warn" }); + expect(noisy.metrics[0]?.reasons[0]?.message).toContain("not conclusive"); + + const countMismatch = compareReceipts( + base, + candidate({ "guest.instructions": sampled([100, 101]) }), + instructionBudget(), + ); + expect(countMismatch.status).toBe("invalid"); + expect(countMismatch.reasons[0]?.code).toBe("sample-count-mismatch"); + + const kindMismatch = compareReceipts( + receipt({ "guest.instructions": exact(100) }), + candidate({ "guest.instructions": sampled([100, 101]) }), + instructionBudget(), + ); + expect(kindMismatch.status).toBe("invalid"); + expect(kindMismatch.reasons[0]?.code).toBe("sample-kind-mismatch"); + }); + + test("serializes schema-valid JSON and readable Markdown", () => { + const comparison = compareReceipts( + receipt(), + candidate({ "guest.instructions": exact(121) }), + instructionBudget(), + ); + const json = comparisonToJson(comparison); + expect(json.endsWith("\n")).toBe(true); + expect(parseComparisonV1(JSON.parse(json))).toEqual(comparison); + + const markdown = comparisonToMarkdown(comparison); + expect(markdown).toContain("Status: **regression**"); + expect(markdown).toContain("| Guest instructions | 100 | 121 | +21 (+21.00%) | **regression** |"); + expect(markdown).toContain("`threshold-exceeded`"); + + const extra = structuredClone(comparison) as any; + extra.metrics[0].note = "not in v1"; + expect(() => parseComparisonV1(extra)).toThrow("unexpected property"); + }); +}); diff --git a/tests/perf-damage-executor.test.ts b/tests/perf-damage-executor.test.ts new file mode 100644 index 00000000..a31df663 --- /dev/null +++ b/tests/perf-damage-executor.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { parseScenarioV1 } from "../tools/perf/core/index.ts"; +import { + DAMAGE_FIXTURE_BINARY, + DAMAGE_FIXTURE_PACKAGE, + isDamageScenario, + materializeDamageFixture, + runNativeDamageScenario, +} from "../tools/perf/executors/damage.ts"; +import { parseNativeResult } from "../tools/perf/receipts/native-protocol.ts"; + +const ROOT = resolve(import.meta.dir, ".."); +const SCENARIO = parseScenarioV1(JSON.parse( + readFileSync(join(ROOT, "tools/perf/scenarios/damage.json"), "utf8"), +)); + +describe("core damage performance executor", () => { + test("stages the harness separately from the core revision under test", () => { + const temporary = mkdtempSync(join(tmpdir(), "pocketjs-damage-materialize-")); + try { + const fixture = materializeDamageFixture({ + sourceRoot: ROOT, + destination: temporary, + dependencyRoot: "/source", + }); + expect(fixture.packageName).toBe(DAMAGE_FIXTURE_PACKAGE); + expect(fixture.binaryName).toBe(DAMAGE_FIXTURE_BINARY); + const manifest = readFileSync(fixture.manifestPath, "utf8"); + expect(manifest).toContain('path = "/source/engine/core"'); + expect(manifest).not.toContain(`${ROOT}/engine/core`); + } finally { + rmSync(temporary, { recursive: true, force: true }); + } + }); + + test("runs all eight real DamagePlan paths twice with stable correctness", async () => { + expect(isDamageScenario(SCENARIO)).toBe(true); + const temporary = mkdtempSync(join(tmpdir(), "pocketjs-damage-native-")); + try { + const first = await runNativeDamageScenario(SCENARIO, { + sourceRoot: ROOT, + outDir: temporary, + }); + const second = await runNativeDamageScenario(SCENARIO, { + sourceRoot: ROOT, + outDir: temporary, + }); + expect(first.status).toBe("ok"); + expect(second.status).toBe("ok"); + if (first.status !== "ok" || second.status !== "ok") return; + expect(first.measurement.phases.map((phase) => phase.name)).toEqual([ + "single-small", + "corner-touch", + "overlap", + "eight-sparse", + "structural", + "clip-transform", + "texture-in-place", + "settle", + ]); + expect(first.correctness).toEqual(second.correctness); + expect(first.correctness.finalFramebufferHash) + .toBe(first.measurement.finalFramebufferHash); + expect(first.correctness.drawListHash) + .toBe(first.measurement.finalDrawListHash); + expect(first.diagnosticMetrics["native.measured_frames"].value).toBe(960); + expect(first.diagnosticMetrics["native.damage.eight-sparse.max_regions"].value).toBe(8); + expect(first.diagnosticMetrics["native.damage.texture-in-place.full_redraw_frames"].value) + .toBe(120); + expect(first.diagnosticMetrics["native.damage.settle.empty_frames"].value).toBe(119); + expect(parseNativeResult(first).success).toBe(true); + } finally { + rmSync(temporary, { recursive: true, force: true }); + } + }, 120_000); +}); diff --git a/tests/perf-qemu-executor.test.ts b/tests/perf-qemu-executor.test.ts new file mode 100644 index 00000000..7f4d25a6 --- /dev/null +++ b/tests/perf-qemu-executor.test.ts @@ -0,0 +1,264 @@ +import { describe, expect, test } from "bun:test"; +import { + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { parseScenarioV1 } from "../tools/perf/core/index.ts"; +import { + QEMU_ENTROPY_PROFILE, + qemuCleanupFallbackArgs, + qemuHarnessFingerprint, + qemuInvocationProfile, + qemuQuickJsReplayReasons, + qemuScenarioRenderContract, + snapshotGuestArtifacts, +} from "../tools/perf/executors/qemu.ts"; +import { + GUEST_OUTPUT_PREFIX, + scenarioPhaseId, +} from "../tools/perf/receipts/index.ts"; + +const ROOT = join(import.meta.dir, ".."); +const BOOT = parseScenarioV1(JSON.parse( + readFileSync(join(ROOT, "tools/perf/scenarios/boot.json"), "utf8"), +)); +const FRAMEBUFFER_TRACE = "3".repeat(64); +const DRAW_LIST = "fnv1a64:1111111111111111"; +const STATE = "fnv1a64:2222222222222222"; +const EFFECT = "fnv1a64:3333333333333333"; + +function guestLine(value: unknown): string { + return `${GUEST_OUTPUT_PREFIX}${JSON.stringify(value)}\n`; +} + +function quickJsGuestOutput(completeOverrides: Record = {}): string { + return [ + guestLine({ + schemaVersion: 1, + event: "phase", + scenarioId: BOOT.id, + phase: "first-frame", + phaseId: scenarioPhaseId(BOOT.id, "first-frame"), + iteration: 0, + allocCalls: 7, + allocatedBytes: 2_048, + currentBytes: 1_024, + peakBytes: 1_536, + quickjsLiveBytesAfterGc: 768, + drawListHash: DRAW_LIST, + }), + guestLine({ + schemaVersion: 1, + event: "complete", + scenarioId: BOOT.id, + suite: BOOT.suite, + framework: BOOT.subject.framework, + finalDrawListHash: DRAW_LIST, + finalStateHash: STATE, + effectHash: EFFECT, + ...completeOverrides, + }), + ].join(""); +} + +describe("QEMU render/build contract", () => { + test("confines the permission cleanup fallback to the disposable work directory", () => { + const directory = mkdtempSync(join(tmpdir(), "pocketjs-qemu-cleanup-")); + try { + const work = join(directory, ".qemu-work-fixture"); + mkdirSync(work, { mode: 0o700 }); + expect(qemuCleanupFallbackArgs("pocketjs-perf-qemu:11.0.3", work)).toEqual([ + "docker", "run", "--rm", + "--network", "none", + "--read-only", + "--cap-drop", "ALL", + "--cap-add", "DAC_OVERRIDE", + "--security-opt", "no-new-privileges", + "--mount", `type=bind,source=${realpathSync(work)},target=/work`, + "--entrypoint", "find", + "pocketjs-perf-qemu:11.0.3", + "/work", "-mindepth", "1", "-delete", + ]); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + test("separates pinned CPU and deterministic emulator arguments", () => { + expect(qemuInvocationProfile("qemu-armv7-thumb2")).toEqual({ + cpuArgs: ["-cpu", "cortex-a9,neon=off,vfp-d32=off"], + emulatorArgs: ["-seed", "1"], + entropyProfile: QEMU_ENTROPY_PROFILE, + }); + expect(qemuInvocationProfile("qemu-aarch64")).toEqual({ + cpuArgs: ["-cpu", "cortex-a53"], + emulatorArgs: ["-seed", "1"], + entropyProfile: QEMU_ENTROPY_PROFILE, + }); + expect(QEMU_ENTROPY_PROFILE).toBe("seed-1+guest-shim-v1"); + }); + + test("fingerprints the host Bun runtime and harness lockfile", () => { + const directory = mkdtempSync(join(tmpdir(), "pocketjs-qemu-fingerprint-")); + try { + writeFileSync(join(directory, "bun.lock"), "lock-v1"); + const first = qemuHarnessFingerprint( + directory, + "sha256:" + "1".repeat(64) + " linux/amd64 []", + "qemu-armv7-thumb2", + "Bun 1.3.14", + "darwin", + "arm64", + ); + const runtimeChanged = qemuHarnessFingerprint( + directory, + "sha256:" + "1".repeat(64) + " linux/amd64 []", + "qemu-armv7-thumb2", + "Bun 1.3.15", + "darwin", + "arm64", + ); + expect(runtimeChanged).not.toBe(first); + + const hostChanged = qemuHarnessFingerprint( + directory, + "sha256:" + "1".repeat(64) + " linux/amd64 []", + "qemu-armv7-thumb2", + "Bun 1.3.14", + "linux", + "x64", + ); + expect(hostChanged).not.toBe(first); + + writeFileSync(join(directory, "bun.lock"), "lock-v2"); + const lockChanged = qemuHarnessFingerprint( + directory, + "sha256:" + "1".repeat(64) + " linux/amd64 []", + "qemu-armv7-thumb2", + "Bun 1.3.14", + "darwin", + "arm64", + ); + expect(lockChanged).not.toBe(first); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + test("keeps framebuffer trace hashing in correctness replay and matches Native", () => { + const valid = qemuQuickJsReplayReasons( + BOOT, + quickJsGuestOutput({ framebufferTraceHash: FRAMEBUFFER_TRACE }), + quickJsGuestOutput(), + FRAMEBUFFER_TRACE, + ); + expect(valid.reasons).toEqual([]); + expect(valid.correctness.complete?.framebufferTraceHash).toBe(FRAMEBUFFER_TRACE); + expect(valid.measurement.complete?.framebufferTraceHash).toBeUndefined(); + + const missing = qemuQuickJsReplayReasons( + BOOT, + quickJsGuestOutput(), + quickJsGuestOutput(), + FRAMEBUFFER_TRACE, + ); + expect(missing.reasons.join(" ")).toContain( + "guest complete has no required framebufferTraceHash", + ); + + const malformed = qemuQuickJsReplayReasons( + BOOT, + quickJsGuestOutput({ framebufferTraceHash: "A".repeat(64) }), + quickJsGuestOutput(), + FRAMEBUFFER_TRACE, + ); + expect(malformed.reasons.join(" ")).toContain( + "framebufferTraceHash must be a lowercase SHA-256 digest", + ); + + const mismatch = qemuQuickJsReplayReasons( + BOOT, + quickJsGuestOutput({ framebufferTraceHash: "a".repeat(64) }), + quickJsGuestOutput(), + FRAMEBUFFER_TRACE, + ); + expect(mismatch.reasons).toContain( + "QEMU correctness framebuffer trace differs from Native/WASM correctness replay", + ); + + const measurementLeak = qemuQuickJsReplayReasons( + BOOT, + quickJsGuestOutput({ framebufferTraceHash: FRAMEBUFFER_TRACE }), + quickJsGuestOutput({ framebufferTraceHash: FRAMEBUFFER_TRACE }), + FRAMEBUFFER_TRACE, + ); + expect(measurementLeak.reasons.join(" ")).toContain( + "guest complete emitted correctness-only framebufferTraceHash", + ); + }); + + test("uses one strict config for the build density, cache key and framebuffer size", () => { + const scenario = parseScenarioV1({ + ...BOOT, + params: { + ...BOOT.params, + viewport: { + width: 320, + height: 180, + rasterDensity: 2, + renderScale: 3, + }, + }, + }); + const contract = qemuScenarioRenderContract(scenario); + expect(contract.densityArgument).toBe("--density=2"); + expect(contract.artifactCacheKey).toEndWith("\0density=2"); + expect(contract.framebufferByteLength).toBe(320 * 3 * 180 * 3 * 4); + + const otherDensity = parseScenarioV1({ + ...scenario, + params: { + ...scenario.params, + viewport: { + ...(scenario.params.viewport as Record), + rasterDensity: 3, + }, + }, + }); + expect(qemuScenarioRenderContract(otherDensity).artifactCacheKey) + .not.toBe(contract.artifactCacheKey); + + const otherEntry = parseScenarioV1({ + ...scenario, + subject: { ...scenario.subject, entry: `${scenario.subject.entry}-alternate` }, + }); + expect(qemuScenarioRenderContract(otherEntry).artifactCacheKey) + .not.toBe(contract.artifactCacheKey); + }); + + test("snapshots build outputs before another framework can overwrite dist", () => { + const directory = mkdtempSync(join(tmpdir(), "pocketjs-qemu-artifacts-")); + try { + const bundle = join(directory, "source.js"); + const pak = join(directory, "source.pak"); + writeFileSync(bundle, "variant-a"); + writeFileSync(pak, "pak-a"); + const frozen = snapshotGuestArtifacts(bundle, pak, join(directory, "work"), "variant-a"); + + writeFileSync(bundle, "variant-b"); + writeFileSync(pak, "pak-b"); + + expect(readFileSync(frozen.bundle, "utf8")).toBe("variant-a"); + expect(readFileSync(frozen.pak!, "utf8")).toBe("pak-a"); + expect(frozen.bundle.startsWith(join(directory, "work"))).toBe(true); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/perf-receipts.test.ts b/tests/perf-receipts.test.ts new file mode 100644 index 00000000..2f93ec04 --- /dev/null +++ b/tests/perf-receipts.test.ts @@ -0,0 +1,638 @@ +import { describe, expect, test } from "bun:test"; +import { + compareReceipts, + parseReceiptV1, + parseScenarioV1, + type ScenarioV1, +} from "../tools/perf/core/index.ts"; +import type { NativeOkResult } from "../tools/perf/runner/native.ts"; +import { + GUEST_OUTPUT_PREFIX, + QEMU_OUTPUT_PREFIX, + canonicalJson, + createNativeReceipt, + createQemuReceipts, + guestDigestToSha256, + parseGuestOutput, + parseQemuOutput, + scenarioPhaseId, + sha256Json, + type ReceiptEnvironmentV1, +} from "../tools/perf/receipts/index.ts"; + +const HASH = { + content: "1".repeat(64), + binary: "2".repeat(64), + framebufferTrace: "3".repeat(64), + framebufferFinal: "4".repeat(64), + state: "5".repeat(64), + effect: "6".repeat(64), + draw: "7".repeat(64), +}; +const FNV = { + phaseDraw: "fnv1a64:1111111111111111", + finalDraw: "fnv1a64:1111111111111111", + state: "fnv1a64:2222222222222222", + effect: "fnv1a64:3333333333333333", +}; + +const scenario: ScenarioV1 = parseScenarioV1({ + schemaVersion: 1, + kind: "pocketjs.perf.scenario", + id: "receipt-fixture", + suite: "quick", + subject: { + id: "fixture", + family: "guest-app", + framework: "solid", + entry: "fixture-main", + }, + executorRequirements: ["guest.frame"], + frames: 4, + tape: { + schemaVersion: 1, + kind: "pocketjs.perf.input-tape", + id: "receipt-fixture-tape", + frames: 4, + tracks: [], + }, + phases: [{ name: "steady", startFrame: 1, endFrame: 4, collect: true }], + checkpoints: [{ frame: 3, capture: ["framebuffer", "state", "effects"] }], + params: { + gateMetrics: [ + "artifact.bundle_bytes", + "guest.instructions", + "memory.allocated_bytes", + ], + }, +}); + +const provenance: ReceiptEnvironmentV1 = { + source: { revision: "fixture", dirty: false, contentHash: HASH.content }, + toolchain: { + rustc: "rustc 1.93.0", + cCompiler: "gcc 12.2.0", + sysroot: "debian-bookworm-20260803", + qemu: "11.0.3", + }, + build: { + target: "armv7-unknown-linux-gnueabihf", + profile: "release-perf", + rustFlags: ["-C", "target-feature=+thumb-mode"], + cFlags: ["-mthumb", "-march=armv7-a", "-mfpu=vfpv3-d16", "-mfloat-abi=hard"], + linkerFlags: [], + }, + executor: { + id: "qemu-armv7-thumb2", + version: "11.0.3", + profile: "linux-user-plugin-v1", + fingerprint: "8".repeat(64), + }, + binary: { sha256: HASH.binary }, +}; + +function guestPhase(overrides: Record = {}) { + return { + schemaVersion: 1, + event: "phase", + scenarioId: scenario.id, + phase: "steady", + phaseId: scenarioPhaseId(scenario.id, "steady"), + iteration: 0, + allocCalls: 7, + allocatedBytes: 2048, + currentBytes: 1024, + peakBytes: 1536, + quickjsLiveBytesAfterGc: 768, + drawListHash: FNV.phaseDraw, + ...overrides, + }; +} + +function guestComplete(overrides: Record = {}) { + return { + schemaVersion: 1, + event: "complete", + scenarioId: scenario.id, + suite: scenario.suite, + framework: scenario.subject.framework, + finalDrawListHash: FNV.finalDraw, + finalStateHash: FNV.state, + effectHash: FNV.effect, + ...overrides, + }; +} + +function qemuMeasurement(overrides: Record = {}) { + return { + schema: "pocketjs.perf.qemu", + version: 1, + event: "measurement", + plugin_api: 6, + qemu_version: "11.0.3", + target: "arm", + vcpu: 0, + phase_id: scenarioPhaseId(scenario.id, "steady"), + iteration: 0, + metrics: { + guest_insn_dispatched: 10_000, + guest_instruction_bytes: 24_000, + guest_insn_size_2: 8_000, + guest_insn_size_4: 2_000, + guest_load_events: 3_000, + guest_store_events: 1_000, + }, + ...overrides, + }; +} + +function qemuComplete(overrides: Record = {}) { + return { + schema: "pocketjs.perf.qemu", + version: 1, + event: "complete", + plugin_api: 6, + qemu_version: "11.0.3", + target: "arm", + measurements: 1, + ...overrides, + }; +} + +function line(prefix: string, value: unknown): string { + return `${prefix}${JSON.stringify(value)}\n`; +} + +function combinedOutput( + guestPhaseValue: unknown = guestPhase(), + qemuMeasurementValue: unknown = qemuMeasurement(), + guestCompleteValue: unknown = guestComplete(), +): string { + return [ + "untrusted program output\n", + line(GUEST_OUTPUT_PREFIX, guestPhaseValue), + line(QEMU_OUTPUT_PREFIX, qemuMeasurementValue), + line(GUEST_OUTPUT_PREFIX, guestCompleteValue), + line(QEMU_OUTPUT_PREFIX, qemuComplete()), + ].join(""); +} + +function correctnessOutput( + guestCompleteValue: unknown = guestComplete({ framebufferTraceHash: HASH.framebufferTrace }), +): string { + return [ + line(GUEST_OUTPUT_PREFIX, guestPhase()), + line(GUEST_OUTPUT_PREFIX, guestCompleteValue), + ].join(""); +} + +function nativeResult(overrides: Partial = {}): NativeOkResult { + return { + schemaVersion: 1, + kind: "pocketjs.perf.native-result", + status: "ok", + scenarioId: scenario.id, + executor: "native", + sourceRoot: "/tmp/pocketjs-fixture", + correctness: { + framebufferTraceHash: HASH.framebufferTrace, + finalFramebufferHash: HASH.framebufferFinal, + drawListHash: FNV.phaseDraw, + stateHash: HASH.state, + effectHash: HASH.effect, + checkpoints: { "3": { framebuffer: HASH.framebufferFinal, state: HASH.state } }, + }, + measurement: { + bootWallTimeNs: 1_000, + phases: [{ name: "steady", startFrame: 1, endFrame: 4, wallTimeNs: 9_000 }], + finalFramebufferHash: HASH.framebufferFinal, + finalDrawListHash: FNV.finalDraw, + }, + diagnosticMetrics: { + "native.boot_wall_time_ns": { value: 1_000, unit: "ns" }, + "native.measured_frames": { value: 3, unit: "count" }, + "native.phase.steady.wall_time_ns": { value: 9_000, unit: "ns" }, + "native.wall_time_ns": { value: 9_000, unit: "ns" }, + }, + exactMetrics: { + "artifact.bundle_bytes": { value: 4_096, unit: "bytes" }, + "artifact.pak_bytes": { value: 512, unit: "bytes" }, + }, + unsupportedMetrics: ["guest.instructions", "memory.allocated_bytes"], + ...overrides, + }; +} + +describe("perf receipt hashing", () => { + test("canonicalizes object keys recursively and hashes semantic JSON", () => { + expect(canonicalJson({ z: [3, { b: 2, a: 1 }], a: true })).toBe( + '{"a":true,"z":[3,{"a":1,"b":2}]}', + ); + expect(sha256Json({ b: 2, a: 1 })).toBe(sha256Json({ a: 1, b: 2 })); + expect(() => canonicalJson({ bad: undefined })).toThrow("cannot contain undefined"); + expect(guestDigestToSha256("draw-list", FNV.phaseDraw)).toMatch(/^[a-f0-9]{64}$/); + }); + + test("matches the guest's unsigned FNV-1a phase identifier", () => { + expect(scenarioPhaseId("receipt-fixture", "steady")).toBe(4_292_253_619); + }); +}); + +describe("strict guest and QEMU protocol parsing", () => { + test("accepts only complete, versioned protocol streams", () => { + const output = combinedOutput(); + const guest = parseGuestOutput(output); + const qemu = parseQemuOutput(output); + expect(guest.status).toBe("valid"); + expect(qemu.status).toBe("valid"); + expect(guest.phases).toHaveLength(1); + expect(qemu.measurements).toHaveLength(1); + }); + + test("strictly validates the optional correctness framebuffer trace digest", () => { + const valid = parseGuestOutput(correctnessOutput()); + expect(valid.status).toBe("valid"); + expect(valid.complete?.framebufferTraceHash).toBe(HASH.framebufferTrace); + + for (const framebufferTraceHash of ["a".repeat(63), "A".repeat(64), 42]) { + const invalid = parseGuestOutput(correctnessOutput(guestComplete({ framebufferTraceHash }))); + expect(invalid.status).toBe("invalid"); + expect(invalid.reasons.join(" ")).toContain( + "framebufferTraceHash must be a lowercase SHA-256 digest", + ); + } + }); + + test("rejects malformed JSON, unknown fields/events, and missing terminals", () => { + expect(parseGuestOutput(`${GUEST_OUTPUT_PREFIX}{broken\n`).status).toBe("invalid"); + const unknownField = parseGuestOutput( + line(GUEST_OUTPUT_PREFIX, guestPhase({ device: "vita" })) + + line(GUEST_OUTPUT_PREFIX, guestComplete()), + ); + expect(unknownField.status).toBe("invalid"); + expect(unknownField.reasons.join(" ")).toContain("unknown properties"); + + const unknownEvent = parseQemuOutput(line(QEMU_OUTPUT_PREFIX, { + schema: "pocketjs.perf.qemu", + version: 1, + event: "summary", + })); + expect(unknownEvent.status).toBe("invalid"); + expect(unknownEvent.reasons.join(" ")).toContain("unknown protocol event"); + + const noTerminal = parseQemuOutput(line(QEMU_OUTPUT_PREFIX, qemuMeasurement())); + expect(noTerminal.status).toBe("invalid"); + expect(noTerminal.reasons.join(" ")).toContain("exactly one"); + }); + + test("rejects duplicates, out-of-order terminals, wrong counts, and plugin errors", () => { + const duplicateGuest = parseGuestOutput( + line(GUEST_OUTPUT_PREFIX, guestPhase()) + + line(GUEST_OUTPUT_PREFIX, guestPhase()) + + line(GUEST_OUTPUT_PREFIX, guestComplete()), + ); + expect(duplicateGuest.status).toBe("invalid"); + expect(duplicateGuest.reasons.join(" ")).toContain("duplicate guest phase"); + + const afterTerminal = parseQemuOutput( + line(QEMU_OUTPUT_PREFIX, qemuComplete()) + + line(QEMU_OUTPUT_PREFIX, qemuMeasurement()), + ); + expect(afterTerminal.status).toBe("invalid"); + expect(afterTerminal.reasons.join(" ")).toContain("not the final"); + + const wrongCount = parseQemuOutput( + line(QEMU_OUTPUT_PREFIX, qemuMeasurement()) + + line(QEMU_OUTPUT_PREFIX, qemuComplete({ measurements: 2 })), + ); + expect(wrongCount.status).toBe("invalid"); + expect(wrongCount.reasons.join(" ")).toContain("does not match"); + + const pluginError = parseQemuOutput(line(QEMU_OUTPUT_PREFIX, { + schema: "pocketjs.perf.qemu", + version: 1, + event: "error", + plugin_api: 6, + qemu_version: "11.0.3", + target: "arm", + code: "missing_end", + measurements: 0, + })); + expect(pluginError.status).toBe("invalid"); + expect(pluginError.reasons).toContain("QEMU plugin reported missing_end"); + }); +}); + +describe("receipt factories", () => { + test("merges QEMU counters, guest memory, correctness, and artifact metrics per phase", () => { + const receipts = createQemuReceipts(scenario, combinedOutput(), { + provenance, + target: "arm", + correctnessGuestOutput: correctnessOutput(), + framebufferHash: HASH.framebufferTrace, + artifactMetrics: { + "artifact.bundle_bytes": 4096, + "artifact.pak_bytes": 512, + "artifact.elf_text_rodata_bytes": 80_000, + }, + createdAt: "2026-08-09T12:00:00.000Z", + }); + expect(receipts).toHaveLength(1); + const receipt = parseReceiptV1(receipts[0]); + expect(receipt.status).toBe("valid"); + expect(receipt.provenance.scenario.id).toBe("receipt-fixture#steady"); + expect(receipt.provenance.scenario.manifestHash).toBe(sha256Json(scenario)); + expect(receipt.provenance.scenario.inputTapeHash).toBe(sha256Json(scenario.tape)); + expect(receipt.gateMetrics).toEqual([ + "artifact.bundle_bytes", + "guest.instructions", + "memory.allocated_bytes", + ]); + expect(receipt.unsupportedMetrics).toEqual([]); + expect(receipt.metrics["guest.instructions"]).toEqual({ kind: "exact", value: 10_000, unit: "count" }); + expect(receipt.metrics["guest.load_store_events"]).toEqual({ kind: "exact", value: 4_000, unit: "count" }); + expect(receipt.metrics["memory.allocated_bytes"]).toEqual({ kind: "exact", value: 2048, unit: "bytes" }); + expect(receipt.metrics["quickjs.live_bytes_after_gc"]).toEqual({ kind: "exact", value: 768, unit: "bytes" }); + expect(receipt.metrics["artifact.elf_text_rodata_bytes"]).toEqual({ kind: "exact", value: 80_000, unit: "bytes" }); + if (receipt.status === "valid") { + expect(receipt.correctness.drawListHash).toBe(guestDigestToSha256("draw-list", FNV.phaseDraw)); + expect(receipt.correctness.framebufferHash).toBe(HASH.framebufferTrace); + } + }); + + test("sources guest-app framebuffer traces only from a valid correctness replay", () => { + const artifactMetrics = { "artifact.bundle_bytes": 4_096 } as const; + const withoutExpectedOracle = createQemuReceipts(scenario, combinedOutput(), { + provenance, + target: "arm", + correctnessGuestOutput: correctnessOutput(), + artifactMetrics, + })[0]!; + expect(withoutExpectedOracle.status).toBe("valid"); + expect(withoutExpectedOracle.correctness?.framebufferHash).toBe(HASH.framebufferTrace); + + const missingOutput = createQemuReceipts(scenario, combinedOutput(), { + provenance, + target: "arm", + framebufferHash: HASH.framebufferTrace, + artifactMetrics, + })[0]!; + expect(missingOutput.status).toBe("invalid"); + expect(missingOutput.invalidReasons.join(" ")).toContain( + "QEMU guest-app receipt has no correctness guest output", + ); + + const missingField = createQemuReceipts(scenario, combinedOutput(), { + provenance, + target: "arm", + correctnessGuestOutput: correctnessOutput(guestComplete()), + framebufferHash: HASH.framebufferTrace, + artifactMetrics, + })[0]!; + expect(missingField.status).toBe("invalid"); + expect(missingField.invalidReasons.join(" ")).toContain( + "guest complete has no required framebufferTraceHash", + ); + + const malformedField = createQemuReceipts(scenario, combinedOutput(), { + provenance, + target: "arm", + correctnessGuestOutput: correctnessOutput( + guestComplete({ framebufferTraceHash: "A".repeat(64) }), + ), + framebufferHash: HASH.framebufferTrace, + artifactMetrics, + })[0]!; + expect(malformedField.status).toBe("invalid"); + expect(malformedField.invalidReasons.join(" ")).toContain( + "framebufferTraceHash must be a lowercase SHA-256 digest", + ); + + const mismatch = createQemuReceipts(scenario, combinedOutput(), { + provenance, + target: "arm", + correctnessGuestOutput: correctnessOutput( + guestComplete({ framebufferTraceHash: "a".repeat(64) }), + ), + framebufferHash: HASH.framebufferTrace, + artifactMetrics, + })[0]!; + expect(mismatch.status).toBe("invalid"); + expect(mismatch.invalidReasons.join(" ")).toContain( + "QEMU correctness framebuffer trace differs from the independent correctness replay", + ); + + const measurementLeak = createQemuReceipts( + scenario, + combinedOutput( + guestPhase(), + qemuMeasurement(), + guestComplete({ framebufferTraceHash: HASH.framebufferTrace }), + ), + { + provenance, + target: "arm", + correctnessGuestOutput: correctnessOutput(), + framebufferHash: HASH.framebufferTrace, + artifactMetrics, + }, + )[0]!; + expect(measurementLeak.status).toBe("invalid"); + expect(measurementLeak.invalidReasons.join(" ")).toContain( + "guest complete emitted correctness-only framebufferTraceHash", + ); + }); + + test("detects injected loop, allocation, and bundle-padding regressions", () => { + const base = createQemuReceipts(scenario, combinedOutput(), { + provenance, + target: "arm", + correctnessGuestOutput: correctnessOutput(), + framebufferHash: HASH.framebufferTrace, + artifactMetrics: { "artifact.bundle_bytes": 4_096 }, + createdAt: "2026-08-09T12:00:00.000Z", + })[0]!; + const injectedMetrics = { + ...qemuMeasurement().metrics, + guest_insn_dispatched: 20_001, + }; + const injectedOutput = combinedOutput( + guestPhase({ allocatedBytes: 10_241 }), + qemuMeasurement({ metrics: injectedMetrics }), + ); + const candidate = createQemuReceipts(scenario, injectedOutput, { + provenance: { + ...provenance, + binary: { sha256: "9".repeat(64) }, + }, + target: "arm", + correctnessGuestOutput: correctnessOutput(), + framebufferHash: HASH.framebufferTrace, + artifactMetrics: { "artifact.bundle_bytes": 8_193 }, + createdAt: "2026-08-09T12:00:00.000Z", + })[0]!; + const comparison = compareReceipts(base, candidate); + expect(comparison.status).toBe("regression"); + for (const metricId of [ + "guest.instructions", + "memory.allocated_bytes", + "artifact.bundle_bytes", + ]) { + expect(comparison.metrics.find((metric) => metric.id === metricId)?.status).toBe("regression"); + } + }); + + test("turns missing, unknown, and mismatched phase records into invalid receipts", () => { + const missingQemu = [ + line(GUEST_OUTPUT_PREFIX, guestPhase()), + line(GUEST_OUTPUT_PREFIX, guestComplete()), + line(QEMU_OUTPUT_PREFIX, qemuComplete({ measurements: 0 })), + ].join(""); + const missingReceipt = createQemuReceipts(scenario, missingQemu, { + provenance, + target: "arm", + correctnessGuestOutput: correctnessOutput(), + framebufferHash: HASH.framebufferTrace, + })[0]; + expect(parseReceiptV1(missingReceipt).status).toBe("invalid"); + expect(missingReceipt.invalidReasons.join(" ")).toContain("missing QEMU measurement steady"); + + const wrongId = scenarioPhaseId(scenario.id, "another-phase"); + const mismatchReceipt = createQemuReceipts( + scenario, + combinedOutput(guestPhase({ phaseId: wrongId }), qemuMeasurement({ phase_id: wrongId })), + { + provenance, + target: "arm", + correctnessGuestOutput: correctnessOutput(), + framebufferHash: HASH.framebufferTrace, + }, + )[0]; + expect(mismatchReceipt.status).toBe("invalid"); + expect(mismatchReceipt.invalidReasons.join(" ")).toContain("phaseId mismatch"); + + const unknownReceipt = createQemuReceipts( + scenario, + combinedOutput(guestPhase({ phase: "unknown" }), qemuMeasurement()), + { + provenance, + target: "arm", + correctnessGuestOutput: correctnessOutput(), + framebufferHash: HASH.framebufferTrace, + }, + )[0]; + expect(unknownReceipt.status).toBe("invalid"); + expect(unknownReceipt.invalidReasons.join(" ")).toContain("phase order mismatch"); + }); + + test("requires QEMU gate observations and exact Native unsupported declarations", () => { + const gated = parseScenarioV1({ + ...scenario, + params: { gateMetrics: ["artifact.elf_text_rodata_bytes"] }, + }); + const qemuReceipt = createQemuReceipts(gated, combinedOutput(), { + provenance, + target: "arm", + correctnessGuestOutput: correctnessOutput(), + framebufferHash: HASH.framebufferTrace, + })[0]!; + expect(qemuReceipt.status).toBe("invalid"); + expect(qemuReceipt.invalidReasons).toContain( + "required gate metric artifact.elf_text_rodata_bytes is missing", + ); + + const nativeGated = parseScenarioV1({ + ...scenario, + params: { gateMetrics: ["guest.instructions"] }, + }); + const nativeEnvironment = { + ...provenance, + toolchain: { rustc: "host", cCompiler: "host", sysroot: "host" }, + build: { target: "wasm32-host", profile: "release", rustFlags: [], cFlags: [], linkerFlags: [] }, + executor: { + id: "native", + version: "bun 1.3.14", + profile: "wasm-sim", + fingerprint: "8".repeat(64), + }, + }; + const explicit = createNativeReceipt(nativeGated, nativeResult({ + unsupportedMetrics: ["guest.instructions"], + }), { provenance: nativeEnvironment }); + expect(explicit.status).toBe("valid"); + expect(explicit.unsupportedMetrics).toEqual(["guest.instructions"]); + + const silent = createNativeReceipt(nativeGated, nativeResult({ unsupportedMetrics: [] }), { + provenance: nativeEnvironment, + }); + expect(silent.status).toBe("invalid"); + expect(silent.invalidReasons.join(" ")).toContain("missing without an explicit native unsupported declaration"); + + const noGates = parseScenarioV1({ ...scenario, params: {} }); + const extra = createNativeReceipt(noGates, nativeResult({ + unsupportedMetrics: ["guest.instructions"], + }), { provenance: nativeEnvironment }); + expect(extra.status).toBe("invalid"); + expect(extra.invalidReasons.join(" ")).toContain("marked non-gate metric"); + }); + + test("maps native correctness explicitly and refuses to invent a missing draw-list hash", () => { + const missingDraw = structuredClone(nativeResult()) as any; + delete missingDraw.correctness.drawListHash; + const noDraw = createNativeReceipt(scenario, missingDraw, { + provenance: { + ...provenance, + toolchain: { rustc: "host", cCompiler: "host", sysroot: "host" }, + build: { target: "wasm32-host", profile: "release", rustFlags: [], cFlags: [], linkerFlags: [] }, + executor: { + id: "native", + version: "bun 1.3.14", + profile: "wasm-sim", + fingerprint: "8".repeat(64), + }, + }, + createdAt: "2026-08-09T12:00:00.000Z", + }); + expect(parseReceiptV1(noDraw).status).toBe("invalid"); + expect(noDraw.invalidReasons.join(" ")).toContain("native.correctness.drawListHash is missing"); + expect(noDraw.correctness).toBeNull(); + + const withDraw = createNativeReceipt(scenario, nativeResult(), { + provenance: { + ...provenance, + toolchain: { rustc: "host", cCompiler: "host", sysroot: "host" }, + build: { target: "wasm32-host", profile: "release", rustFlags: [], cFlags: [], linkerFlags: [] }, + executor: { + id: "native", + version: "bun 1.3.14", + profile: "wasm-sim", + fingerprint: "8".repeat(64), + }, + }, + createdAt: "2026-08-09T12:00:00.000Z", + }); + expect(parseReceiptV1(withDraw).status).toBe("valid"); + expect(withDraw.metrics["native.wall_time_ns"]).toEqual({ kind: "exact", value: 9_000, unit: "ns" }); + expect(withDraw.metrics["artifact.bundle_bytes"]).toEqual({ kind: "exact", value: 4_096, unit: "bytes" }); + if (withDraw.status === "valid") { + expect(withDraw.correctness).toEqual({ + framebufferHash: HASH.framebufferTrace, + drawListHash: guestDigestToSha256("draw-list", FNV.phaseDraw), + stateHash: HASH.state, + effectHash: HASH.effect, + }); + } + }); + + test("emits schema-valid invalid receipts for malformed native execution output", () => { + const malformed = structuredClone(nativeResult()) as unknown as Record; + malformed.extra = true; + const receipt = createNativeReceipt(scenario, malformed, { + provenance, + artifactMetrics: { "artifact.bundle_bytes": 10 }, + createdAt: "2026-08-09T12:00:00.000Z", + }); + expect(parseReceiptV1(receipt).status).toBe("invalid"); + expect(receipt.invalidReasons.join(" ")).toContain("native.extra is unknown"); + }); +}); diff --git a/tests/perf-runner.test.ts b/tests/perf-runner.test.ts new file mode 100644 index 00000000..38092731 --- /dev/null +++ b/tests/perf-runner.test.ts @@ -0,0 +1,480 @@ +import { describe, expect, test } from "bun:test"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + INITIAL_KEYED_ROWS, + keyedDelete, + keyedInsert, + keyedReorder, +} from "../tools/perf/apps/keyed-list-model.ts"; +import { + artifactBuildVariantKey, + buildRenderConfig, + DEFAULT_BUDGET_SET, + isMetricId, + parseInputTapeV1, + parseScenarioV1, +} from "../tools/perf/core/index.ts"; +import { expandInputTape } from "../tools/perf/runner/input.ts"; +import { + devtoolsTapeToInputTape, + goldenSpecToInputTape, + ppssppScriptToInputTape, + vaporTodoToInputTape, +} from "../tools/perf/runner/legacy-input.ts"; +import { + runNativeQuick, + type NativeBootAdapter, + type NativeRunResult, + type NativeSimWorld, +} from "../tools/perf/runner/native.ts"; +import { + estimatedSuiteSeconds, + expandSuiteFrameworks, + loadScenarioSuite, + runNativeSuite, +} from "../tools/perf/runner/suite.ts"; + +const SCENARIO_DIR = join(import.meta.dir, "..", "tools", "perf", "scenarios"); + +function scenario(name: string) { + return parseScenarioV1( + JSON.parse(readFileSync(join(SCENARIO_DIR, `${name}.json`), "utf8")), + ); +} + +describe("performance scenario catalog", () => { + test("strictly validates the complete v1 scenario matrix", () => { + const names = readdirSync(SCENARIO_DIR) + .filter((name) => name.endsWith(".json")) + .map((name) => name.slice(0, -5)) + .sort(); + expect(names).toEqual([ + "boot", + "damage", + "deepzoom", + "fixed-text", + "idle", + "list", + "style", + "timer", + "touch", + "vapor", + ]); + for (const name of names) { + const parsed = scenario(name); + expect(parsed.frames).toBe(parsed.tape.frames); + expect(parsed.params.gateMetrics).toBeArray(); + for (const metric of parsed.params.gateMetrics as readonly string[]) { + expect(isMetricId(metric), `${name}: ${metric}`).toBe(true); + expect(metric, `${name}: aggregate load/store gate`).not.toBe("guest.loads"); + expect(metric, `${name}: aggregate load/store gate`).not.toBe("guest.stores"); + expect(metric, `${name}: current memory is diagnostic`).not.toBe("memory.current_bytes"); + expect(metric, `${name}: peak memory is diagnostic`).not.toBe("memory.peak_bytes"); + } + expect(parsed.executorRequirements, `${name}: DrawList correctness capability`) + .toContain("correctness.draw-list"); + expect(buildRenderConfig(parsed.params)).toEqual({ + width: 480, + height: 272, + rasterDensity: 1, + renderScale: 1, + }); + } + }); + + test("keys built app artifacts by their resolved raster density", () => { + const base = scenario("boot"); + const highDensity = parseScenarioV1({ + ...base, + params: { + ...base.params, + viewport: { rasterDensity: 2 }, + }, + }); + expect(artifactBuildVariantKey(base)).toEndWith("\0density=1"); + expect(artifactBuildVariantKey(highDensity)).toEndWith("\0density=2"); + expect(artifactBuildVariantKey(highDensity)).not.toBe(artifactBuildVariantKey(base)); + }); + + test("keeps all ten approved workloads in the quick suite", () => { + expect([ + "boot", + "idle", + "fixed-text", + "list", + "style", + "timer", + "damage", + "touch", + "vapor", + "deepzoom", + ].map((name) => scenario(name).suite)).toEqual(new Array(10).fill("quick")); + }); + + test("gives every required gate an applicable QEMU budget", () => { + for (const name of readdirSync(SCENARIO_DIR) + .filter((entry) => entry.endsWith(".json")) + .map((entry) => entry.slice(0, -5))) { + const parsed = scenario(name); + for (const metric of parsed.params.gateMetrics as readonly string[]) { + const scenarioBudget = DEFAULT_BUDGET_SET.scenarios?.[parsed.id]?.[metric]; + const globalBudget = DEFAULT_BUDGET_SET.metrics[metric]; + const budget = scenarioBudget ?? globalBudget; + expect(budget, `${parsed.id}: ${metric}`).toBeDefined(); + for (const executor of ["qemu-armv7-thumb2", "qemu-aarch64"]) { + expect( + budget?.executors === undefined || budget.executors.includes(executor), + `${parsed.id}: ${metric} must apply to ${executor}`, + ).toBe(true); + } + } + } + }); + + test("maps framework matrix entries to their real application artifacts", () => { + const expanded = expandSuiteFrameworks([scenario("boot")]); + expect(expanded.map((item) => [item.subject.framework, item.subject.id, item.subject.entry])) + .toEqual([ + ["solid", "hero-main", "hero-main"], + ["vue-vapor", "hero-vue-vapor-main", "hero-vue-vapor-main.vue-vapor"], + ["octane", "hero-main", "hero-main.octane"], + ]); + }); + + test("drives real keyed insert, reorder, and delete operations", () => { + const keyed = scenario("list"); + expect([keyed.subject.id, keyed.subject.entry]).toEqual([ + "tools/perf/apps/list-fixture-main.tsx", + "list-fixture-main", + ]); + expect(keyed.phases.map((phase) => [phase.name, phase.collect])).toEqual([ + ["warmup", false], + ["keyed-insert", true], + ["keyed-reorder", true], + ["keyed-delete", true], + ["steady", true], + ]); + expect(keyed.tape.tracks.map((track) => track.kind === "button" ? track.control : track.kind)) + .toEqual(["quaternary", "secondary", "primary"]); + expect(keyed.checkpoints.map((checkpoint) => checkpoint.frame)).toEqual([59, 89, 119, 179]); + for (const checkpoint of keyed.checkpoints) expect(checkpoint.capture).toContain("drawList"); + const fixtureSource = readFileSync( + join(import.meta.dir, "..", "tools", "perf", "apps", "list-fixture-main.tsx"), + "utf8", + ); + expect(fixtureSource).toContain(""); + expect(fixtureSource).toContain("onButtonPress(BTN.SQUARE"); + expect(fixtureSource).toContain("onButtonPress(BTN.TRIANGLE"); + expect(fixtureSource).toContain("onButtonPress(BTN.CIRCLE"); + + const initial = [...INITIAL_KEYED_ROWS]; + const inserted = keyedInsert(initial); + expect(inserted.map((item) => item.id)).toEqual([ + "alpha", "inserted", "bravo", "charlie", "delta", + ]); + expect(inserted[0]).toBe(initial[0]); + expect(inserted[2]).toBe(initial[1]); + + const reordered = keyedReorder(inserted); + expect(reordered.map((item) => item.id)).toEqual([ + "inserted", "bravo", "charlie", "delta", "alpha", + ]); + expect(reordered[4]).toBe(initial[0]); + expect(reordered[1]).toBe(initial[1]); + + const deleted = keyedDelete(reordered); + expect(deleted.map((item) => item.id)).toEqual(["bravo", "charlie", "delta", "alpha"]); + expect(deleted[0]).toBe(initial[1]); + expect(deleted[3]).toBe(initial[0]); + }); + + test("uses a genuinely static idle application", () => { + const idle = scenario("idle"); + expect([idle.id, idle.subject.id, idle.subject.entry]).toEqual([ + "guest.fixture.idle-600.v1", + "tools/perf/apps/idle-fixture-main.tsx", + "idle-fixture-main", + ]); + expect(idle.tape.tracks).toEqual([]); + for (const checkpoint of idle.checkpoints) { + expect(checkpoint.capture).toContain("framebuffer"); + expect(checkpoint.capture).toContain("drawList"); + } + const source = readFileSync( + join(import.meta.dir, "..", "tools", "perf", "apps", "idle-fixture-main.tsx"), + "utf8", + ); + expect(source).toContain('class="h-[144] flex-row gap-4"'); + expect(source).toContain('class="w-[216] h-[144]'); + expect(source).not.toMatch(/\bitems-center\b|\bjustify-center\b/); + expect(source).not.toMatch(/\bonFrame\s*\(/); + expect(source).not.toMatch(/\bcreateSpriteAnimation\s*\(/); + expect(source).not.toMatch(/<(?:Sprite|Image)\b/); + expect(source).not.toMatch(/\b(?:animate|spring)\s*\(/); + }); + + test("does not claim final state capture for the core damage adapter", () => { + const damage = scenario("damage"); + expect(damage.executorRequirements).not.toContain("correctness.state-final"); + expect(damage.checkpoints.flatMap((checkpoint) => checkpoint.capture)).not.toContain("state"); + }); + + test("keeps the local quick suite within an explicit estimated-time budget", async () => { + const quick = loadScenarioSuite("quick", SCENARIO_DIR); + expect(quick).toHaveLength(10); + expect(estimatedSuiteSeconds(quick)).toBe(80); + expect(estimatedSuiteSeconds(expandSuiteFrameworks(quick))).toBe(88); + expect(estimatedSuiteSeconds(expandSuiteFrameworks(quick))).toBeLessThan(1500); + expect( + runNativeSuite("quick", { + sourceRoot: "/tmp/unused", + scenarioDir: SCENARIO_DIR, + maxEstimatedSeconds: 87, + bootAdapter: { async boot() { throw new Error("must not boot"); } }, + }), + ).rejects.toThrow(/estimate 88s exceeds the 87s limit/); + }); +}); + +describe("hardware-neutral input adapter", () => { + test("lowers logical controls only at the guest ABI boundary", () => { + const tape = parseInputTapeV1({ + schemaVersion: 1, + kind: "pocketjs.perf.input-tape", + id: "adapter-contract", + frames: 6, + tracks: [ + { + kind: "button", + control: "primary", + samples: [ + { frame: 1, pressed: true }, + { frame: 3, pressed: false }, + ], + }, + { + kind: "analog", + control: "x", + samples: [ + { frame: 2, value: -1 }, + { frame: 4, value: 0 }, + ], + }, + { + kind: "touch", + control: "contact-0", + samples: [ + { frame: 2, phase: "start", x: 240, y: 90 }, + { frame: 3, phase: "move", x: 241, y: 91 }, + { frame: 4, phase: "end", x: 241, y: 91 }, + ], + }, + { + kind: "relative-axis", + control: "primary", + samples: [{ frame: 3, delta: -45000 }], + }, + { + kind: "effect", + effect: "probe", + samples: [{ frame: 5, value: { ok: true } }], + }, + ], + }); + const frames = expandInputTape(tape); + + expect(frames[0]).toMatchObject({ buttons: 0, analog: 0x8080, touches: undefined }); + expect(frames[1].buttons).toBe(0x2000); // logical primary -> guest CIRCLE + expect(frames[2].analog).toBe(0x0080); // target-neutral x=-1 -> guest raw x=0 + expect(frames[2].touches).toEqual([((90 << 9) | 240) >>> 0]); + expect(frames[3].relativeAxes).toEqual([{ control: "primary", delta: -45000 }]); + expect(frames[4].buttons).toBe(0); + expect(frames[4].analog).toBe(0x8080); + expect(frames[4].touches).toBeUndefined(); + expect(frames[5].effects).toEqual([{ effect: "probe", value: { ok: true } }]); + }); + + test("freezes GoldenSpec, DevTools, PPSSPP and Vapor inputs into one shape", () => { + const golden = goldenSpecToInputTape("golden", { + frames: 4, + input: (frame) => frame === 1 ? 0x2000 : 0, + touch: (frame) => frame >= 1 && frame < 3 ? [{ id: 0, x: 20 + frame, y: 30 }] : [], + }); + expect(golden.tracks.find((track) => track.kind === "button")).toEqual({ + kind: "button", + control: "primary", + samples: [ + { frame: 1, pressed: true }, + { frame: 2, pressed: false }, + ], + }); + expect(golden.tracks.find((track) => track.kind === "touch")?.samples).toEqual([ + { frame: 1, phase: "start", x: 21, y: 30 }, + { frame: 2, phase: "move", x: 22, y: 30 }, + { frame: 3, phase: "end", x: 22, y: 30 }, + ]); + + const devtools = devtoolsTapeToInputTape("devtools", { + frames: 4, + masks: [[0, 4]], + analog: [[0x8080, 2], [0x0080, 1], [0x8080, 1]], + touch: [[1, [((30 << 9) | 20) >>> 0]]], + startFrame: 0, + }); + const devFrames = expandInputTape(devtools); + expect(devFrames[2].analog).toBe(0x0080); + expect(devFrames[1].touches).toEqual([((30 << 9) | 20) >>> 0]); + expect(devFrames[2].touches).toBeUndefined(); + + const ppsspp = expandInputTape( + ppssppScriptToInputTape("ppsspp", 4, "0:0,1:0x40,3:0"), + ); + expect(ppsspp.map((frame) => frame.buttons)).toEqual([0, 0x40, 0x40, 0]); + + const vapor = vaporTodoToInputTape("vapor", [7, 0], { bootFrames: 1, spacing: 2 }); + expect(vapor.frames).toBe(5); + expect(expandInputTape(vapor).map((frame) => frame.buttons)).toEqual([ + 0, + 0x40, + 0, + 0x2000, + 0, + ]); + }); + + test("rejects wrapped recorder tapes instead of approximate replay", () => { + expect(() => devtoolsTapeToInputTape("wrapped", { + frames: 1, + masks: [[0, 1]], + startFrame: 99, + })).toThrow(/startFrame must be 0/); + }); +}); + +describe("native quick runner", () => { + test("separates correctness and measurement replays", async () => { + const worlds: FakeWorld[] = []; + const idleScenario = scenario("idle"); + const adapter: NativeBootAdapter = { + async boot(sourceRoot, parsedScenario) { + expect(sourceRoot).toEndWith("candidate-source"); + expect(parsedScenario.subject.entry).toBe("idle-fixture-main"); + const world = new FakeWorld(); + worlds.push(world); + return world; + }, + }; + + const result = await runNativeQuick(idleScenario, { + sourceRoot: "/tmp/candidate-source", + bootAdapter: adapter, + }); + expect(result.status).toBe("ok"); + if (result.status !== "ok") return; + expect(worlds).toHaveLength(2); + expect(worlds[0].treeReads).toBe(1); + expect(worlds[1].treeReads).toBe(0); + expect(worlds[0].jobDrains).toBe(idleScenario.frames); + expect(worlds[1].jobDrains).toBe(idleScenario.frames); + expect(result.correctness.finalFramebufferHash) + .toBe(result.measurement.finalFramebufferHash); + expect(result.correctness.drawListHash) + .toBe(result.measurement.finalDrawListHash); + expect(result.measurement.phases.map((phase) => phase.name)).toEqual(["idle"]); + expect(result.diagnosticMetrics["native.measured_frames"].value).toBe(600); + expect(result.unsupportedMetrics).toContain("guest.instructions"); + expect(result.correctness.drawListHash).toStartWith("fnv1a64:"); + }); + + test("returns structured unsupported instead of placeholder metrics", async () => { + const result = await runNativeQuick(scenario("damage"), { + sourceRoot: "/tmp/unused", + bootAdapter: { async boot() { throw new Error("must not boot"); } }, + }); + expect(result.status).toBe("unsupported"); + if (result.status !== "unsupported") return; + expect(result.reasons.some((reason) => reason.includes("core-lab"))).toBe(true); + expect(result.reasons.some((reason) => reason.includes("fixture.core.damage"))).toBe(true); + expect(Object.hasOwn(result, "metrics")).toBe(false); + }); + + test("dispatches damage and Vapor through their specialized suite adapters", async () => { + const calls: string[] = []; + const unsupported = async ( + parsedScenario: ReturnType, + options: { readonly sourceRoot: string; readonly harnessRoot: string }, + ): Promise => { + calls.push(`${parsedScenario.subject.family}:${parsedScenario.id}`); + expect(options).toMatchObject({ + sourceRoot: "/tmp/candidate-source", + harnessRoot: "/tmp/perf-harness", + }); + return { + schemaVersion: 1, + kind: "pocketjs.perf.native-result", + status: "unsupported", + scenarioId: parsedScenario.id, + executor: "native", + reasons: ["fixture adapter disabled by this unit test"], + }; + }; + const result = await runNativeSuite("quick", { + sourceRoot: "/tmp/candidate-source", + harnessRoot: "/tmp/perf-harness", + scenarioDir: SCENARIO_DIR, + maxEstimatedSeconds: 1500, + bootAdapter: { async boot() { return new FakeWorld(); } }, + suiteAdapters: { damage: unsupported, vapor: unsupported }, + }); + expect(result.results).toHaveLength(12); + expect(result.results.filter((item) => item.status === "ok")).toHaveLength(10); + expect(result.results + .filter((item) => item.status === "unsupported") + .map((item) => item.scenarioId) + .sort()).toEqual(["core.damage-cases.v1", "vapor.todo.reactive-grid.v1"]); + expect(calls).toEqual([ + "core-lab:core.damage-cases.v1", + "vapor:vapor.todo.reactive-grid.v1", + ]); + }); +}); + +class FakeWorld implements NativeSimWorld { + readonly ticksPerFrame = 1; + readonly effects: unknown[] = []; + treeReads = 0; + jobDrains = 0; + private value = 0; + + frame(buttons: number, analog = 0x8080, touches?: readonly number[]): void { + this.value = Math.imul(this.value ^ buttons ^ analog ^ (touches?.[0] ?? 0), 16777619) >>> 0; + } + + async drainJobs(): Promise { + this.jobDrains++; + await Promise.resolve(); + } + + tick(): void { + this.value = (this.value + 1) >>> 0; + } + + render(): Uint8Array { + return new Uint8Array([ + this.value & 0xff, + (this.value >>> 8) & 0xff, + (this.value >>> 16) & 0xff, + (this.value >>> 24) & 0xff, + ]); + } + + drawHash(): string { + return `fnv1a64:${this.value.toString(16).padStart(16, "0")}`; + } + + getTree(): unknown { + this.treeReads++; + return { value: this.value }; + } +} diff --git a/tests/perf-vapor-executor.test.ts b/tests/perf-vapor-executor.test.ts new file mode 100644 index 00000000..18efcaf1 --- /dev/null +++ b/tests/perf-vapor-executor.test.ts @@ -0,0 +1,609 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { parseScenarioV1 } from "../tools/perf/core/index.ts"; +import { scenarioPhaseId } from "../tools/perf/receipts/hash.ts"; +import { + GUEST_OUTPUT_PREFIX, + QEMU_OUTPUT_PREFIX, +} from "../tools/perf/receipts/protocol.ts"; +import { + prepareVaporQemuFixture, + runNativeVaporScenario, + runVaporScenario, + vaporGuestStateParityReasons, + vaporQemuReplayReasons, + VAPOR_QEMU_BUILD_SPECS, +} from "../tools/perf/executors/vapor.ts"; + +const ROOT = join(import.meta.dir, ".."); +const SCENARIO = parseScenarioV1( + JSON.parse(readFileSync(join(ROOT, "tools/perf/scenarios/vapor.json"), "utf8")), +); + +const VAPOR_STATE_OUTPUT_PREFIX = "POCKETJS_PERF_VAPOR "; +const PHASE_DRAW_HASHES = [ + "fnv1a64:1111111111111111", + "fnv1a64:2222222222222222", + "fnv1a64:3333333333333333", +] as const; +const STATE_HASHES = [ + "fnv1a64:aaaaaaaaaaaaaaaa", + "fnv1a64:bbbbbbbbbbbbbbbb", + "fnv1a64:cccccccccccccccc", +] as const; +const EFFECT_HASH = "fnv1a64:dddddddddddddddd"; +const FRAMEBUFFER_TRACE_HASH = "0123456789abcdef".repeat(4); + +function protocolLine(prefix: string, value: unknown): string { + return `${prefix}${JSON.stringify(value)}\n`; +} + +function generatedCPhase(index: number, overrides: Record = {}): Record { + const phase = SCENARIO.phases.filter((candidate) => candidate.collect)[index]!; + return { + schemaVersion: 1, + event: "phase", + scenarioId: SCENARIO.id, + phase: phase.name, + phaseId: scenarioPhaseId(SCENARIO.id, phase.name), + iteration: 0, + allocCalls: 0, + allocatedBytes: 0, + currentBytes: 0, + peakBytes: 0, + quickjsLiveBytesAfterGc: 0, + drawListHash: PHASE_DRAW_HASHES[index], + ...overrides, + }; +} + +function generatedCCheckpoint(index: number, overrides: Record = {}): Record { + return { + schemaVersion: 1, + event: "state-checkpoint", + scenarioId: SCENARIO.id, + frame: SCENARIO.checkpoints[index]!.frame, + stateHash: STATE_HASHES[index], + ...overrides, + }; +} + +function generatedCComplete( + includeFramebufferTrace: boolean, + overrides: Record = {}, +): Record { + return { + schemaVersion: 1, + event: "complete", + scenarioId: SCENARIO.id, + suite: SCENARIO.suite, + framework: SCENARIO.subject.framework, + finalDrawListHash: PHASE_DRAW_HASHES.at(-1), + finalStateHash: STATE_HASHES.at(-1), + effectHash: EFFECT_HASH, + ...(includeFramebufferTrace ? { framebufferTraceHash: FRAMEBUFFER_TRACE_HASH } : {}), + ...overrides, + }; +} + +function qemuMeasurement(index: number, overrides: Record = {}): Record { + const phase = SCENARIO.phases.filter((candidate) => candidate.collect)[index]!; + return { + schema: "pocketjs.perf.qemu", + version: 1, + event: "measurement", + plugin_api: 6, + qemu_version: "11.0.3", + target: "arm", + vcpu: 0, + phase_id: scenarioPhaseId(SCENARIO.id, phase.name), + iteration: 0, + metrics: { + guest_insn_dispatched: 100 + index, + guest_instruction_bytes: 200 + index, + guest_insn_size_2: 50 + index, + guest_insn_size_4: 50, + guest_load_events: 30 + index, + guest_store_events: 20 + index, + }, + ...overrides, + }; +} + +interface GeneratedCReplayOptions { + readonly plugin?: boolean; + readonly phaseOverrides?: Readonly>>; + readonly checkpointOverrides?: Readonly>>; + readonly completeOverrides?: Record; + readonly qemuOverrides?: Readonly>>; + readonly omitPhase?: number; + readonly omitCheckpoint?: number; +} + +function generatedCReplayOutput(options: GeneratedCReplayOptions = {}): string { + const phases = SCENARIO.phases.filter((phase) => phase.collect); + const lines: string[] = []; + phases.forEach((_phase, index) => { + if (options.omitPhase !== index) { + lines.push(protocolLine( + GUEST_OUTPUT_PREFIX, + generatedCPhase(index, options.phaseOverrides?.[index]), + )); + } + if (options.omitCheckpoint !== index) { + lines.push(protocolLine( + VAPOR_STATE_OUTPUT_PREFIX, + generatedCCheckpoint(index, options.checkpointOverrides?.[index]), + )); + } + }); + lines.push(protocolLine( + GUEST_OUTPUT_PREFIX, + generatedCComplete(!options.plugin, options.completeOverrides), + )); + if (options.plugin) { + phases.forEach((_phase, index) => { + if (options.omitPhase !== index) { + lines.push(protocolLine( + QEMU_OUTPUT_PREFIX, + qemuMeasurement(index, options.qemuOverrides?.[index]), + )); + } + }); + lines.push(protocolLine(QEMU_OUTPUT_PREFIX, { + schema: "pocketjs.perf.qemu", + version: 1, + event: "complete", + plugin_api: 6, + qemu_version: "11.0.3", + target: "arm", + measurements: phases.length - (options.omitPhase === undefined ? 0 : 1), + })); + } + return lines.join(""); +} + +describe("Vapor performance executor", () => { + test("adapts two independent oracle replays to the common native protocol", async () => { + const result = await runNativeVaporScenario(SCENARIO, { + sourceRoot: ROOT, + harnessRoot: ROOT, + }); + expect(result.status).toBe("ok"); + if (result.status !== "ok") return; + + expect(result.correctness.finalFramebufferHash).toBe(result.measurement.finalFramebufferHash); + expect(result.correctness.drawListHash).toBe(result.measurement.finalDrawListHash); + expect(result.measurement.phases.map((phase) => phase.name)).toEqual([ + "idle", + "reactive", + "settle", + ]); + expect(Object.keys(result.correctness.checkpoints)).toEqual(["119", "359", "719"]); + expect(result.correctness.checkpoints["719"]?.state).toMatch(/^[a-f0-9]{64}$/); + expect(result.exactMetrics).toEqual({}); + expect(result.unsupportedMetrics).toEqual([ + "guest.instructions", + "memory.allocations", + "memory.allocated_bytes", + "artifact.elf_text_rodata_bytes", + ]); + }); + + test("replays the hardware-neutral tape on the real Vue Vapor oracle deterministically", async () => { + const options = { + scenario: SCENARIO, + executor: "native" as const, + sourceRoot: ROOT, + harnessRoot: ROOT, + }; + const first = await runVaporScenario(options); + const second = await runVaporScenario(options); + expect(first.status).toBe("ok"); + expect(second.status).toBe("ok"); + if (first.status !== "ok" || first.executor !== "native" || + second.status !== "ok" || second.executor !== "native") return; + + expect(first.framebufferHash).toBe(second.framebufferHash); + expect(first.finalDrawListHash).toBe(second.finalDrawListHash); + expect(first.stateHash).toBe(second.stateHash); + expect(first.effectHash).toBe(second.effectHash); + expect(first.finalStateDigest).toBe(second.finalStateDigest); + expect(first.checkpointStateDigests).toEqual(second.checkpointStateDigests); + expect(first.phaseDrawListHashes).toEqual(second.phaseDrawListHashes); + expect(first.axisEventsDelivered).toBe(3); + expect(first.axisEventsObserved).toBe(3); + expect(first.compiledRelativeAxesUsed).toEqual([0]); + expect(first.target).toBe("playdate"); + expect(first.finalDrawListHash).toBe("fnv1a64:e8fe72fac6607a31"); + expect(first.framebufferHash).toBe("c6bd7006790065646053ee94c67d61b902e96f3f9f642dc3586c8ec4772d2ca5"); + expect(Object.keys(first.phaseDrawListHashes)).toEqual(["idle", "reactive", "settle"]); + expect(Object.keys(first.checkpointStateDigests)).toEqual(["119", "359", "719"]); + expect(first.checkpointStateDigests["719"]).toBe(first.finalStateDigest); + expect(first.phaseDrawListHashes.idle).not.toBe(first.phaseDrawListHashes.reactive); + }); + + test("isolates concurrent correctness and measurement oracle replays", async () => { + const results = await Promise.all(Array.from({ length: 4 }, () => + runNativeVaporScenario(SCENARIO, { + sourceRoot: ROOT, + harnessRoot: ROOT, + }))); + + for (const result of results) expect(result.status).toBe("ok"); + const successful = results.filter((result) => result.status === "ok"); + expect(successful).toHaveLength(4); + expect(new Set(successful.map((result) => result.correctness.finalFramebufferHash)).size).toBe(1); + expect(new Set(successful.map((result) => result.correctness.drawListHash)).size).toBe(1); + for (const result of successful) { + expect(result.correctness.finalFramebufferHash).toBe(result.measurement.finalFramebufferHash); + expect(result.correctness.drawListHash).toBe(result.measurement.finalDrawListHash); + } + }); + + test("generates a freestanding, allocation-free Linux guest with pinned ARM flags", async () => { + const outDir = mkdtempSync(join(tmpdir(), "pocketjs-vapor-fixture-test-")); + try { + const fixture = await prepareVaporQemuFixture({ + scenario: SCENARIO, + executor: "qemu-armv7-thumb2", + sourceRoot: ROOT, + harnessRoot: ROOT, + outDir, + }); + const harness = readFileSync(fixture.guestHarness, "utf8"); + const header = readFileSync(fixture.runtimeHeader, "utf8"); + const generated = readFileSync(fixture.generatedApp, "utf8"); + + expect(generated).toContain("void app_on_axis_delta(u8 axis, s32 delta)"); + expect(generated).toContain("static s32 g_axisEvents;"); + expect(generated).toContain("vp_axis_handler_0"); + expect(generated).toContain("g_axisEvents + 1"); + expect(harness).toContain("app_on_axis_delta(event->control, event->value)"); + expect(harness).toContain("\\\"allocCalls\\\":0"); + expect(harness).toContain("POCKETJS_PERF_VAPOR"); + expect(harness).toContain("if (len != 56u)"); + expect(harness).toContain("u8 n = state.bytes[16]"); + expect(harness).toContain("for (j = 0; j < n; j++)"); + const stateHashSource = harness.slice( + harness.indexOf("static perf_u64 state_hash(void)"), + harness.indexOf("static perf_u64 events_hash(void)"), + ); + expect(stateHashSource).not.toContain("for (i = 0; i < len; i++)"); + expect(harness).not.toMatch(/\b(?:malloc|calloc|realloc|free)\s*\(/); + expect(harness).toContain("static u8 sha256_self_test(void)"); + expect(harness).toContain("framebufferTraceHash"); + expect(harness.match(/sha256_update\(&perf_framebuffer_trace/g)).toHaveLength(2); + expect(harness).toContain("if (!sha256_self_test()) return 78;"); + expect(harness).toContain('stack[2], "--correctness"'); + expect(harness).toContain("else if (argc != 1u)"); + const entrySource = harness.slice(harness.indexOf("static __attribute__((used, noreturn, noinline)) void perf_start")); + const architectureStart = entrySource.slice(entrySource.indexOf("#if defined(__aarch64__)")); + const aarch64Start = architectureStart.slice(0, architectureStart.indexOf("#else")); + const armStart = architectureStart.slice( + architectureStart.indexOf("#else"), + architectureStart.indexOf("#endif"), + ); + expect(aarch64Start).not.toMatch(/__attribute__\s*\(\(naked/); + expect(aarch64Start).not.toContain("void _start(void)"); + expect(aarch64Start).toMatch(/^__asm__\s*\(/m); + expect(aarch64Start).toContain(".global _start"); + expect(aarch64Start).toContain(".type _start"); + expect(aarch64Start).toMatch(/mov x0, sp[\s\S]*?b perf_start/); + // Keep the already calibrated ARM/Thumb entry byte-for-byte stable. + expect(armStart).toContain("__attribute__((naked, noreturn)) void _start(void)"); + expect(armStart).toMatch(/mov r0, sp[\s\S]*?b perf_start/); + const perfMain = harness.indexOf("static int perf_main(void)"); + const markerEnd = harness.indexOf("pocketjs_perf_end(phase->id, 0)", perfMain); + const phaseDiagnostic = harness.indexOf("emit_phase(phase)", markerEnd); + const stateDiagnostic = harness.indexOf("emit_state_checkpoint(frame)", markerEnd); + expect(perfMain).toBeGreaterThanOrEqual(0); + expect(markerEnd).toBeGreaterThan(perfMain); + expect(phaseDiagnostic).toBeGreaterThan(markerEnd); + expect(stateDiagnostic).toBeGreaterThan(markerEnd); + expect(header).toContain("typedef uint32_t u32;"); + expect(header).toContain("typedef int32_t s32;"); + expect(fixture.build.cFlags).toEqual(VAPOR_QEMU_BUILD_SPECS["qemu-armv7-thumb2"].cFlags); + expect(fixture.build.cFlags).toContain("-mthumb"); + expect(fixture.build.cFlags).not.toContain("-mfpu=neon"); + expect(fixture.build.cpuArgs).toEqual([ + "-cpu", + "cortex-a9,neon=off,vfp-d32=off", + ]); + expect(fixture.build.emulatorArgs).toEqual(["-seed", "1"]); + expect(VAPOR_QEMU_BUILD_SPECS["qemu-aarch64"].cpuArgs).toEqual([ + "-cpu", + "cortex-a53", + ]); + expect(VAPOR_QEMU_BUILD_SPECS["qemu-aarch64"].emulatorArgs).toEqual(["-seed", "1"]); + expect(fixture.build.linkerFlags).toContain("-nostdlib"); + } finally { + rmSync(outDir, { recursive: true, force: true }); + } + }); + + test("accepts identical generated-C correctness and measurement replays with plugin output only on measurement", () => { + const replay = vaporQemuReplayReasons( + SCENARIO, + generatedCReplayOutput(), + generatedCReplayOutput({ plugin: true }), + ); + + expect(replay.reasons).toEqual([]); + expect(replay.correctness.status).toBe("valid"); + expect(replay.measurement.status).toBe("valid"); + expect(replay.qemu.status).toBe("valid"); + expect(replay.correctness.phases.map((phase) => phase.phase)).toEqual([ + "idle", + "reactive", + "settle", + ]); + expect(replay.correctness.complete?.finalStateHash).toBe(STATE_HASHES.at(-1)); + expect(replay.correctness.complete?.framebufferTraceHash).toBe(FRAMEBUFFER_TRACE_HASH); + expect(replay.measurement.complete?.effectHash).toBe(EFFECT_HASH); + expect(replay.measurement.complete?.framebufferTraceHash).toBeUndefined(); + }); + + test("requires a plugin-free correctness replay and a complete measurement plugin stream", () => { + const correctnessWithPlugin = vaporQemuReplayReasons( + SCENARIO, + generatedCReplayOutput({ plugin: true }), + generatedCReplayOutput({ plugin: true }), + ); + expect(correctnessWithPlugin.reasons.join("\n")).toContain( + "correctness replay emitted QEMU plugin records", + ); + + const measurementWithoutPlugin = vaporQemuReplayReasons( + SCENARIO, + generatedCReplayOutput(), + generatedCReplayOutput(), + ); + expect(measurementWithoutPlugin.reasons.join("\n")).toContain( + "measurement: no QEMU protocol records", + ); + }); + + test("requires a correctness framebuffer trace and forbids hashing it in measurement", () => { + const missing = vaporQemuReplayReasons( + SCENARIO, + generatedCReplayOutput({ + completeOverrides: { framebufferTraceHash: undefined }, + }), + generatedCReplayOutput({ plugin: true }), + ); + expect(missing.reasons.join("\n")).toContain( + "correctness: guest complete has no required framebufferTraceHash", + ); + + const leaked = vaporQemuReplayReasons( + SCENARIO, + generatedCReplayOutput(), + generatedCReplayOutput({ + plugin: true, + completeOverrides: { framebufferTraceHash: FRAMEBUFFER_TRACE_HASH }, + }), + ); + expect(leaked.reasons.join("\n")).toContain( + "measurement: guest complete emitted correctness-only framebufferTraceHash", + ); + }); + + test("strictly matches phase identity and phase output between generated-C replays", () => { + const identityDrift = vaporQemuReplayReasons( + SCENARIO, + generatedCReplayOutput(), + generatedCReplayOutput({ + plugin: true, + phaseOverrides: { + 1: { phase: "reactive-drift", iteration: 1 }, + }, + }), + ); + expect(identityDrift.reasons.join("\n")).toContain( + "correctness/measurement phase 1 identity differs", + ); + + const outputDrift = vaporQemuReplayReasons( + SCENARIO, + generatedCReplayOutput(), + generatedCReplayOutput({ + plugin: true, + phaseOverrides: { + 1: { drawListHash: "fnv1a64:eeeeeeeeeeeeeeee" }, + }, + }), + ); + expect(outputDrift.reasons.join("\n")).toContain( + "correctness/measurement DrawList differs after phase reactive", + ); + + const bothIncomplete = vaporQemuReplayReasons( + SCENARIO, + generatedCReplayOutput({ omitPhase: 1 }), + generatedCReplayOutput({ plugin: true, omitPhase: 1 }), + ); + expect(bothIncomplete.reasons.join("\n")).toContain( + "correctness emitted 2 phases; expected 3", + ); + expect(bothIncomplete.reasons.join("\n")).toContain( + "measurement emitted 2 phases; expected 3", + ); + + const sharedIdentityDrift = vaporQemuReplayReasons( + SCENARIO, + generatedCReplayOutput({ + phaseOverrides: { 1: { phase: "shared-drift", phaseId: 1234 } }, + }), + generatedCReplayOutput({ + plugin: true, + phaseOverrides: { 1: { phase: "shared-drift", phaseId: 1234 } }, + qemuOverrides: { 1: { phase_id: 1234 } }, + }), + ); + expect(sharedIdentityDrift.reasons.join("\n")).toContain( + "correctness phase 1 identity differs from scenario", + ); + expect(sharedIdentityDrift.reasons.join("\n")).toContain( + "measurement phase 1 identity differs from scenario", + ); + }); + + test("strictly matches checkpoint identity and state output between generated-C replays", () => { + const missing = vaporQemuReplayReasons( + SCENARIO, + generatedCReplayOutput(), + generatedCReplayOutput({ plugin: true, omitCheckpoint: 1 }), + ); + expect(missing.reasons.join("\n")).toContain( + "correctness and measurement emitted different state checkpoint counts", + ); + + const identityDrift = vaporQemuReplayReasons( + SCENARIO, + generatedCReplayOutput(), + generatedCReplayOutput({ + plugin: true, + checkpointOverrides: { 1: { frame: 358 } }, + }), + ); + expect(identityDrift.reasons.join("\n")).toContain( + "correctness/measurement state checkpoint 1 identity differs", + ); + + const stateDrift = vaporQemuReplayReasons( + SCENARIO, + generatedCReplayOutput(), + generatedCReplayOutput({ + plugin: true, + checkpointOverrides: { 1: { stateHash: "fnv1a64:eeeeeeeeeeeeeeee" } }, + }), + ); + expect(stateDrift.reasons.join("\n")).toContain( + "correctness/measurement state differs at checkpoint 359", + ); + + const bothIncomplete = vaporQemuReplayReasons( + SCENARIO, + generatedCReplayOutput({ omitCheckpoint: 1 }), + generatedCReplayOutput({ plugin: true, omitCheckpoint: 1 }), + ); + expect(bothIncomplete.reasons.join("\n")).toContain( + "correctness emitted 2 state checkpoints; expected 3", + ); + expect(bothIncomplete.reasons.join("\n")).toContain( + "measurement emitted 2 state checkpoints; expected 3", + ); + }); + + test("strictly matches final generated-C draw, state, effects and measurement identities", () => { + const finalDrift = vaporQemuReplayReasons( + SCENARIO, + generatedCReplayOutput(), + generatedCReplayOutput({ + plugin: true, + completeOverrides: { + suite: "other-suite", + finalDrawListHash: "fnv1a64:eeeeeeeeeeeeeeee", + finalStateHash: "fnv1a64:ffffffffffffffff", + effectHash: "fnv1a64:9999999999999999", + }, + }), + ); + expect(finalDrift.reasons).toEqual(expect.arrayContaining([ + "correctness/measurement complete identity differs", + "correctness/measurement final DrawList differs", + "correctness/measurement final state differs", + "correctness/measurement effects differ", + ])); + + const qemuIdentityDrift = vaporQemuReplayReasons( + SCENARIO, + generatedCReplayOutput(), + generatedCReplayOutput({ + plugin: true, + qemuOverrides: { 1: { phase_id: 1234, iteration: 1 } }, + }), + ); + expect(qemuIdentityDrift.reasons.join("\n")).toContain( + "measurement QEMU phase 1 identity differs from guest phase", + ); + }); + + test("rejects invalid relative-axis samples instead of coercing them", async () => { + const scenario = parseScenarioV1({ + ...SCENARIO, + tape: { + ...SCENARIO.tape, + tracks: [{ + kind: "relative-axis", + control: "primary", + samples: [{ frame: 1, delta: 0 }], + }], + }, + }); + const result = await runVaporScenario({ + scenario, + executor: "native", + sourceRoot: ROOT, + harnessRoot: ROOT, + }); + expect(result.status).toBe("invalid"); + if (result.status === "invalid") expect(result.reasons.join("\n")).toContain("non-zero signed 32-bit integer"); + }); + + test("strictly validates generated-C final and checkpoint state against the oracle", () => { + const hashes = { + "119": "fnv1a64:0000000000000119", + "359": "fnv1a64:0000000000000359", + "719": "fnv1a64:0000000000000719", + }; + const output = Object.entries(hashes).map(([frame, stateHash]) => + `POCKETJS_PERF_VAPOR ${JSON.stringify({ + schemaVersion: 1, + event: "state-checkpoint", + scenarioId: SCENARIO.id, + frame: Number(frame), + stateHash, + })}` + ).join("\n"); + const native = { + finalStateDigest: hashes["719"], + checkpointStateDigests: hashes, + }; + + expect(vaporGuestStateParityReasons(SCENARIO, output, hashes["719"], native)).toEqual([]); + + const drifted = output.replace(hashes["359"], "fnv1a64:ffffffffffffffff"); + expect(vaporGuestStateParityReasons(SCENARIO, drifted, "fnv1a64:eeeeeeeeeeeeeeee", native)) + .toEqual(expect.arrayContaining([ + "generated-C state differs from Vue Vapor oracle at checkpoint 359", + "generated-C final state differs from Vue Vapor oracle", + "generated-C final state differs between checkpoint and complete records", + ])); + expect(vaporGuestStateParityReasons(SCENARIO, output.split("\n").slice(1).join("\n"), hashes["719"], native)) + .toEqual(expect.arrayContaining([ + "generated-C emitted 2 state checkpoints; expected 3", + ])); + }); + + test("declares every input and correctness capability exercised by the Vapor scenario", () => { + expect(SCENARIO.executorRequirements).toEqual([ + "fixture.vapor.generated-c", + "input.buttons", + "input.relative-axis", + "correctness.draw-list", + "correctness.effects", + "correctness.framebuffer", + "correctness.state-final", + ]); + expect(SCENARIO.params.gateMetrics).toEqual([ + "guest.instructions", + "memory.allocations", + "memory.allocated_bytes", + "artifact.elf_text_rodata_bytes", + ]); + }); +}); diff --git a/tools/perf.ts b/tools/perf.ts new file mode 100644 index 00000000..84ffe81a --- /dev/null +++ b/tools/perf.ts @@ -0,0 +1,4 @@ +#!/usr/bin/env bun +import { runPerfCli } from "./perf/cli/main.ts"; + +if (import.meta.main) process.exitCode = await runPerfCli(process.argv.slice(2)); diff --git a/tools/perf/apps/idle-fixture-main.tsx b/tools/perf/apps/idle-fixture-main.tsx new file mode 100644 index 00000000..bcc027c1 --- /dev/null +++ b/tools/perf/apps/idle-fixture-main.tsx @@ -0,0 +1,33 @@ +import { mount } from "@pocketjs/framework"; +import { Text, View } from "@pocketjs/framework/components"; + +function IdleFixture() { + return ( + + + STATIC DASHBOARD + IDLE + + + + RUNTIME + QuickJS + No timers + No animation + + + RENDERER + UiSurface + Stable tree + Stable DrawList + + + + 480 x 272 + NO INPUT + + + ); +} + +mount(() => ); diff --git a/tools/perf/apps/keyed-list-model.ts b/tools/perf/apps/keyed-list-model.ts new file mode 100644 index 00000000..3411d498 --- /dev/null +++ b/tools/perf/apps/keyed-list-model.ts @@ -0,0 +1,57 @@ +export interface KeyedRow { + readonly id: string; + readonly label: string; + readonly detail: string; + readonly swatchClass: string; +} + +export const INITIAL_KEYED_ROWS: readonly KeyedRow[] = Object.freeze([ + { + id: "alpha", + label: "ALPHA", + detail: "retained row 01", + swatchClass: "w-3 h-3 bg-blue-500", + }, + { + id: "bravo", + label: "BRAVO", + detail: "retained row 02", + swatchClass: "w-3 h-3 bg-emerald-500", + }, + { + id: "charlie", + label: "CHARLIE", + detail: "retained row 03", + swatchClass: "w-3 h-3 bg-amber-500", + }, + { + id: "delta", + label: "DELTA", + detail: "retained row 04", + swatchClass: "w-3 h-3 bg-cyan-500", + }, +]); + +export const INSERTED_KEYED_ROW: KeyedRow = Object.freeze({ + id: "inserted", + label: "INSERTED", + detail: "new keyed row", + swatchClass: "w-3 h-3 bg-rose-500", +}); + +/** Insert one object without replacing any retained row object. */ +export function keyedInsert(rows: readonly KeyedRow[]): KeyedRow[] { + if (rows.some((row) => row.id === INSERTED_KEYED_ROW.id)) return [...rows]; + return [rows[0]!, INSERTED_KEYED_ROW, ...rows.slice(1)]; +} + +/** Rotate the same objects so Solid's For moves its retained row nodes. */ +export function keyedReorder(rows: readonly KeyedRow[]): KeyedRow[] { + if (rows.length < 2) return [...rows]; + return [...rows.slice(1), rows[0]!]; +} + +/** Delete the inserted object without replacing any surviving row object. */ +export function keyedDelete(rows: readonly KeyedRow[]): KeyedRow[] { + return rows.filter((row) => row.id !== INSERTED_KEYED_ROW.id); +} diff --git a/tools/perf/apps/list-fixture-main.tsx b/tools/perf/apps/list-fixture-main.tsx new file mode 100644 index 00000000..616daeb0 --- /dev/null +++ b/tools/perf/apps/list-fixture-main.tsx @@ -0,0 +1,51 @@ +import { createSignal, For } from "solid-js"; +import { mount } from "@pocketjs/framework"; +import { Text, View } from "@pocketjs/framework/components"; +import { onButtonPress } from "@pocketjs/framework/lifecycle"; +import { BTN } from "@pocketjs/framework/input"; +import { + INITIAL_KEYED_ROWS, + keyedDelete, + keyedInsert, + keyedReorder, +} from "./keyed-list-model.ts"; + +function KeyedListFixture() { + const [rows, setRows] = createSignal([...INITIAL_KEYED_ROWS]); + const [operation, setOperation] = createSignal("READY"); + + onButtonPress(BTN.SQUARE, () => { + setRows(keyedInsert); + setOperation("INSERT"); + }); + onButtonPress(BTN.TRIANGLE, () => { + setRows(keyedReorder); + setOperation("REORDER"); + }); + onButtonPress(BTN.CIRCLE, () => { + setRows(keyedDelete); + setOperation("DELETE"); + }); + + return ( + + + KEYED LIST + {operation()} + + + + {(row) => ( + + + {row.label} + {row.detail} + + )} + + + + ); +} + +mount(() => ); diff --git a/tools/perf/cli/args.ts b/tools/perf/cli/args.ts new file mode 100644 index 00000000..a84e04f6 --- /dev/null +++ b/tools/perf/cli/args.ts @@ -0,0 +1,203 @@ +import { EXECUTOR_IDS, type ExecutorId } from "./types.ts"; + +export class UsageError extends Error { + constructor(message: string) { + super(message); + this.name = "UsageError"; + } +} + +export type OutputFormat = "json" | "markdown"; + +export type PerfCommand = + | { readonly command: "help" } + | { readonly command: "doctor"; readonly format: "json" | "text" } + | { + readonly command: "run"; + readonly executor: ExecutorId; + readonly suite: string; + readonly sourceRoot?: string; + readonly scenarioDir?: string; + readonly outDir?: string; + readonly maxEstimatedSeconds: number; + } + | { + readonly command: "compare"; + readonly base: string; + readonly candidate: string; + readonly budget?: string; + readonly format: OutputFormat; + readonly out?: string; + } + | { + readonly command: "local"; + readonly base: string; + readonly executors: readonly ExecutorId[]; + readonly suite: string; + readonly scenarioDir?: string; + readonly budget?: string; + readonly format: OutputFormat; + readonly out?: string; + readonly maxEstimatedSeconds: number; + }; + +export const HELP = `PocketJS local performance regression runner + +Usage: + bun perf doctor [--json] + bun perf run --executor --suite [options] + bun perf compare --base --candidate [options] + bun perf local --base [options] + +Commands: + doctor Check local executor prerequisites without installing or building anything. + run Run one suite and write versioned receipts. + compare Compare one receipt pair, or matching receipts in two directories. + local Compare a git baseline with the tracked current worktree in isolated worktrees. + +Run options: + --source-root Source checkout to measure (default: current repository). + --scenario-dir Scenario manifests (default: tools/perf/scenarios). + --out-dir Receipt directory (default: a new system temp directory). + --max-estimated-seconds Suite estimate ceiling (default: 1500). + +Compare options: + --budget Versioned budget JSON (default: built-in quick budget). + --format Output format (default: json). + --out Also write the comparison report to this path. + +Local options: + --executor Executor to run (repeatable; default: all). + --suite Suite name (default: quick). + --scenario-dir, --budget, --format, --out and --max-estimated-seconds are also accepted. + +Exit status: 0 pass/warn, 1 regression, 2 invalid usage, prerequisites, or receipts. +`; + +interface ParsedFlags { + readonly values: ReadonlyMap; + readonly booleans: ReadonlySet; +} + +function flags(args: readonly string[], booleanNames: readonly string[] = []): ParsedFlags { + const booleans = new Set(); + const values = new Map(); + const booleanSet = new Set(booleanNames); + for (let index = 0; index < args.length; index += 1) { + const token = args[index]!; + if (!token.startsWith("--")) throw new UsageError(`unexpected positional argument ${JSON.stringify(token)}`); + const equal = token.indexOf("="); + const name = equal >= 0 ? token.slice(2, equal) : token.slice(2); + if (booleanSet.has(name)) { + if (equal >= 0) throw new UsageError(`--${name} does not take a value`); + booleans.add(name); + continue; + } + const value = equal >= 0 ? token.slice(equal + 1) : args[++index]; + if (value === undefined || value.startsWith("--")) throw new UsageError(`--${name} requires a value`); + const existing = values.get(name) ?? []; + values.set(name, [...existing, value]); + } + return { values, booleans }; +} + +function rejectUnknown(parsed: ParsedFlags, allowedValues: readonly string[], allowedBooleans: readonly string[] = []): void { + const allowedValueSet = new Set(allowedValues); + const allowedBooleanSet = new Set(allowedBooleans); + for (const name of parsed.values.keys()) { + if (!allowedValueSet.has(name)) throw new UsageError(`unknown option --${name}`); + } + for (const name of parsed.booleans) { + if (!allowedBooleanSet.has(name)) throw new UsageError(`unknown option --${name}`); + } +} + +function one(parsed: ParsedFlags, name: string, required = false): string | undefined { + const found = parsed.values.get(name) ?? []; + if (found.length > 1) throw new UsageError(`--${name} may only be supplied once`); + if (required && found.length === 0) throw new UsageError(`missing required option --${name}`); + return found[0]; +} + +function executor(value: string): ExecutorId { + if ((EXECUTOR_IDS as readonly string[]).includes(value)) return value as ExecutorId; + throw new UsageError(`unknown executor ${JSON.stringify(value)}`); +} + +function format(value: string | undefined): OutputFormat { + if (value === undefined || value === "json") return "json"; + if (value === "markdown") return "markdown"; + throw new UsageError(`--format must be json or markdown`); +} + +function seconds(value: string | undefined): number { + if (value === undefined) return 1_500; + const result = Number(value); + if (!Number.isFinite(result) || result <= 0) { + throw new UsageError(`--max-estimated-seconds must be a positive number`); + } + return result; +} + +export function parsePerfCommand(args: readonly string[]): PerfCommand { + if (args.length === 0 || args[0] === "help" || args[0] === "--help" || args[0] === "-h") { + return { command: "help" }; + } + const command = args[0]!; + const rest = args.slice(1); + if (rest.includes("--help") || rest.includes("-h")) return { command: "help" }; + + if (command === "doctor") { + const parsed = flags(rest, ["json"]); + rejectUnknown(parsed, [], ["json"]); + return { command, format: parsed.booleans.has("json") ? "json" : "text" }; + } + if (command === "run") { + const parsed = flags(rest); + rejectUnknown(parsed, ["executor", "suite", "source-root", "scenario-dir", "out-dir", "max-estimated-seconds"]); + return { + command, + executor: executor(one(parsed, "executor", true)!), + suite: one(parsed, "suite", true)!, + sourceRoot: one(parsed, "source-root"), + scenarioDir: one(parsed, "scenario-dir"), + outDir: one(parsed, "out-dir"), + maxEstimatedSeconds: seconds(one(parsed, "max-estimated-seconds")), + }; + } + if (command === "compare") { + const parsed = flags(rest); + rejectUnknown(parsed, ["base", "candidate", "budget", "format", "out"]); + return { + command, + base: one(parsed, "base", true)!, + candidate: one(parsed, "candidate", true)!, + budget: one(parsed, "budget"), + format: format(one(parsed, "format")), + out: one(parsed, "out"), + }; + } + if (command === "local") { + const parsed = flags(rest); + rejectUnknown(parsed, ["base", "executor", "suite", "scenario-dir", "budget", "format", "out", "max-estimated-seconds"]); + const rawExecutors = parsed.values.get("executor") ?? ["all"]; + const executors = rawExecutors.includes("all") + ? EXECUTOR_IDS + : rawExecutors.map(executor); + if (rawExecutors.includes("all") && rawExecutors.length > 1) { + throw new UsageError(`--executor all cannot be combined with another executor`); + } + return { + command, + base: one(parsed, "base", true)!, + executors: [...new Set(executors)], + suite: one(parsed, "suite") ?? "quick", + scenarioDir: one(parsed, "scenario-dir"), + budget: one(parsed, "budget"), + format: format(one(parsed, "format")), + out: one(parsed, "out"), + maxEstimatedSeconds: seconds(one(parsed, "max-estimated-seconds")), + }; + } + throw new UsageError(`unknown command ${JSON.stringify(command)}`); +} diff --git a/tools/perf/cli/compare-paths.ts b/tools/perf/cli/compare-paths.ts new file mode 100644 index 00000000..24056f00 --- /dev/null +++ b/tools/perf/cli/compare-paths.ts @@ -0,0 +1,457 @@ +import { existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, statSync, writeFileSync } from "node:fs"; +import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { + DEFAULT_BUDGET_SET, + compareReceipts, + comparisonToMarkdown, + parseBudgetSetV1, + type BudgetSetV1, + type ComparisonStatus, + type ComparisonV1, + type ReceiptV1, +} from "../core/index.ts"; +import type { OutputFormat } from "./args.ts"; +import { readReceipt } from "./receipts.ts"; +import { + EXECUTOR_IDS, + type ComparisonSetEntryV1, + type ComparisonSetV1, + type ExecutorId, + type PerfRunSummaryV1, +} from "./types.ts"; + +function statusRank(status: ComparisonStatus): number { + return { pass: 0, warn: 1, regression: 2, invalid: 3 }[status]; +} + +function worst(statuses: readonly ComparisonStatus[]): ComparisonStatus { + return statuses.reduce( + (current, status) => statusRank(status) > statusRank(current) ? status : current, + "pass", + ); +} + +function receiptKey(receipt: ReceiptV1): string { + const { scenario, executor, build } = receipt.provenance; + return [executor.id, executor.profile, build.target, scenario.suite, scenario.id, scenario.framework].join("/"); +} + +function findFiles(root: string, matches: (name: string) => boolean): string[] { + const result: string[] = []; + const visit = (directory: string): void => { + for (const entry of readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) { + const path = join(directory, entry.name); + if (entry.isDirectory()) visit(path); + else if (entry.isFile() && matches(entry.name)) result.push(path); + } + }; + visit(root); + return result; +} + +function findReceiptFiles(root: string): string[] { + return findFiles(root, (name) => name.endsWith(".receipt.json")); +} + +function findRunSummaryFiles(root: string): string[] { + return findFiles(root, (name) => name === "run.json"); +} + +function parseRunSummary(value: unknown): PerfRunSummaryV1 { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error("run summary must be an object"); + } + const record = value as Record; + const expected = [ + "schemaVersion", + "kind", + "status", + "executor", + "suite", + "sourceRoot", + "outputDir", + "receipts", + "invalidReasons", + ]; + const missing = expected.filter((key) => !(key in record)); + const unknown = Object.keys(record).filter((key) => !expected.includes(key)); + if (missing.length > 0 || unknown.length > 0) { + const reasons = [ + ...missing.length > 0 ? [`missing fields: ${missing.join(", ")}`] : [], + ...unknown.length > 0 ? [`unknown fields: ${unknown.sort().join(", ")}`] : [], + ]; + throw new Error(`run summary has invalid fields (${reasons.join("; ")})`); + } + if (record.schemaVersion !== 1) throw new Error("run summary schemaVersion must be 1"); + if (record.kind !== "pocketjs.perf.run") throw new Error("run summary kind must be pocketjs.perf.run"); + if (record.status !== "valid" && record.status !== "invalid") { + throw new Error("run summary status must be valid or invalid"); + } + if (typeof record.executor !== "string" || !EXECUTOR_IDS.includes(record.executor as ExecutorId)) { + throw new Error(`run summary executor is unsupported: ${JSON.stringify(record.executor)}`); + } + for (const field of ["suite", "sourceRoot", "outputDir"] as const) { + if (typeof record[field] !== "string" || record[field].length === 0) { + throw new Error(`run summary ${field} must be a non-empty string`); + } + } + if (!Array.isArray(record.receipts) || record.receipts.some((path) => typeof path !== "string" || path.length === 0)) { + throw new Error("run summary receipts must be an array of non-empty strings"); + } + if (!Array.isArray(record.invalidReasons) || record.invalidReasons.some((reason) => typeof reason !== "string")) { + throw new Error("run summary invalidReasons must be an array of strings"); + } + if (record.status === "valid" && record.invalidReasons.length > 0) { + throw new Error("a valid run summary must not contain invalidReasons"); + } + if (record.status === "invalid" && record.invalidReasons.length === 0) { + throw new Error("an invalid run summary must contain at least one invalidReason"); + } + return { + schemaVersion: 1, + kind: "pocketjs.perf.run", + status: record.status, + executor: record.executor as ExecutorId, + suite: record.suite as string, + sourceRoot: record.sourceRoot as string, + outputDir: record.outputDir as string, + receipts: record.receipts as string[], + invalidReasons: record.invalidReasons as string[], + }; +} + +interface DirectoryIssue { + readonly path: string | null; + readonly reason: string; +} + +interface RunSummaryRecord { + readonly path: string; + readonly summary: PerfRunSummaryV1; +} + +interface DirectoryContents { + readonly receipts: Map; + readonly receiptByRealPath: Map; + readonly runSummaryFiles: readonly string[]; + readonly summaries: readonly RunSummaryRecord[]; + readonly issues: DirectoryIssue[]; +} + +function containedRelativePath(path: string): boolean { + return path.length > 0 && path !== ".." && !path.startsWith(`..${sep}`) && !isAbsolute(path); +} + +function listedReceiptCandidates(runPath: string, summary: PerfRunSummaryV1, listed: string): string[] { + if (!isAbsolute(listed)) return [resolve(dirname(runPath), listed)]; + const result: string[] = []; + // Version 1 originally wrote absolute paths. If the complete run directory + // was moved, relocate only the suffix that was safely below its recorded + // outputDir; never reinterpret a path that escaped that directory. + if (isAbsolute(summary.outputDir)) { + const local = relative(resolve(summary.outputDir), resolve(listed)); + if (containedRelativePath(local)) result.push(resolve(dirname(runPath), local)); + } + result.push(resolve(listed)); + return [...new Set(result)]; +} + +function scanDirectory(root: string): DirectoryContents { + const receipts = new Map(); + const receiptByRealPath = new Map(); + const issues: DirectoryIssue[] = []; + const receiptFiles = findReceiptFiles(root); + for (const path of receiptFiles) { + let realPath: string; + try { + realPath = realpathSync(path); + } catch (error) { + issues.push({ path, reason: `cannot resolve receipt: ${error instanceof Error ? error.message : String(error)}` }); + continue; + } + try { + const receipt = readReceipt(path); + const item = { path, receipt }; + receiptByRealPath.set(realPath, item); + const key = receiptKey(receipt); + const previous = receipts.get(key); + if (previous) { + issues.push({ + path, + reason: `duplicate receipt identity ${JSON.stringify(key)} also appears in ${relative(root, previous.path)}`, + }); + continue; + } + receipts.set(key, item); + } catch (error) { + receiptByRealPath.set(realPath, null); + issues.push({ path, reason: `malformed receipt: ${error instanceof Error ? error.message : String(error)}` }); + } + } + if (receiptFiles.length === 0) issues.push({ path: null, reason: `no *.receipt.json files found in ${root}` }); + + const runSummaryFiles = findRunSummaryFiles(root); + const summaries: RunSummaryRecord[] = []; + for (const path of runSummaryFiles) { + try { + const summary = parseRunSummary(JSON.parse(readFileSync(path, "utf8"))); + summaries.push({ path, summary }); + } catch (error) { + issues.push({ path, reason: `malformed run summary: ${error instanceof Error ? error.message : String(error)}` }); + } + } + return { receipts, receiptByRealPath, runSummaryFiles, summaries, issues }; +} + +function validateRunSummaries(root: string, contents: DirectoryContents): Map { + const identities = new Map(); + const listedReceipts = new Map(); + const suites = new Set(); + for (const item of contents.summaries) { + const { path, summary } = item; + suites.add(summary.suite); + const identity = `${summary.executor}/${summary.suite}`; + const previous = identities.get(identity); + if (previous) { + contents.issues.push({ + path, + reason: `duplicate run summary for executor/suite ${JSON.stringify(identity)}; first appears in ${relative(root, previous.path)}`, + }); + } else { + identities.set(identity, item); + } + if (summary.status !== "valid") { + contents.issues.push({ + path, + reason: `run summary status is invalid: ${summary.invalidReasons.join("; ")}`, + }); + } + if (summary.receipts.length === 0) { + contents.issues.push({ path, reason: "run summary lists no receipts" }); + } + for (const listed of summary.receipts) { + const candidates = listedReceiptCandidates(path, summary, listed); + let realPath: string | null = null; + let foundFileOutsideTree = false; + let inspectionFailure: string | null = null; + for (const listedPath of candidates) { + if (!existsSync(listedPath)) continue; + try { + if (!statSync(listedPath).isFile()) { + inspectionFailure = `listed receipt is not a file: ${listed}`; + continue; + } + const candidateRealPath = realpathSync(listedPath); + if (!contents.receiptByRealPath.has(candidateRealPath)) { + foundFileOutsideTree = true; + continue; + } + realPath = candidateRealPath; + break; + } catch (error) { + inspectionFailure = `cannot inspect listed receipt ${listed}: ${error instanceof Error ? error.message : String(error)}`; + } + } + if (realPath === null) { + contents.issues.push({ + path, + reason: inspectionFailure ?? (foundFileOutsideTree + ? `listed receipt is not a *.receipt.json file inside the compared directory: ${listed}` + : `listed receipt does not exist: ${listed}`), + }); + continue; + } + const previousOwner = listedReceipts.get(realPath); + if (previousOwner) { + contents.issues.push({ + path, + reason: `receipt ${listed} is listed more than once; first listed by ${relative(root, previousOwner)}`, + }); + } else { + listedReceipts.set(realPath, path); + } + const parsed = contents.receiptByRealPath.get(realPath); + if (!parsed) continue; + if (parsed.receipt.provenance.executor.id !== summary.executor) { + contents.issues.push({ + path: parsed.path, + reason: `receipt executor ${parsed.receipt.provenance.executor.id} does not match run summary executor ${summary.executor}`, + }); + } + if (parsed.receipt.provenance.scenario.suite !== summary.suite) { + contents.issues.push({ + path: parsed.path, + reason: `receipt suite ${JSON.stringify(parsed.receipt.provenance.scenario.suite)} does not match run summary suite ${JSON.stringify(summary.suite)}`, + }); + } + if (parsed.receipt.status !== "valid") { + contents.issues.push({ path: parsed.path, reason: "a valid run summary lists an invalid receipt" }); + } + } + } + if (suites.size > 1) { + contents.issues.push({ + path: null, + reason: `run summaries disagree on suite: ${[...suites].sort().join(", ")}`, + }); + } + for (const [realPath, parsed] of contents.receiptByRealPath) { + if (!listedReceipts.has(realPath)) { + contents.issues.push({ + path: parsed?.path ?? realPath, + reason: "receipt is not listed by any run summary", + }); + } + } + return identities; +} + +function displayedPath(root: string, path: string | null): string | null { + if (path === null) return null; + const result = relative(root, path); + return result.length > 0 && !result.startsWith("..") ? result : path; +} + +function issueEntries( + side: "baseline" | "candidate", + root: string, + issues: readonly DirectoryIssue[], +): ComparisonSetEntryV1[] { + return issues.map((issue, index) => ({ + key: `!validation/${side}/${String(index + 1).padStart(4, "0")}`, + status: "invalid", + basePath: side === "baseline" ? displayedPath(root, issue.path) : null, + candidatePath: side === "candidate" ? displayedPath(root, issue.path) : null, + comparison: null, + reason: issue.reason, + })); +} + +function compareDirectories(baseRoot: string, candidateRoot: string, budget: BudgetSetV1): ComparisonSetV1 { + const base = scanDirectory(baseRoot); + const candidate = scanDirectory(candidateRoot); + if (base.runSummaryFiles.length === 0) { + base.issues.push({ path: null, reason: "baseline directory has no run.json summary" }); + } + if (candidate.runSummaryFiles.length === 0) { + candidate.issues.push({ path: null, reason: "candidate directory has no run.json summary" }); + } + const baseRuns = validateRunSummaries(baseRoot, base); + const candidateRuns = validateRunSummaries(candidateRoot, candidate); + for (const identity of [...new Set([...baseRuns.keys(), ...candidateRuns.keys()])].sort()) { + if (!baseRuns.has(identity)) { + base.issues.push({ path: null, reason: `baseline directory has no run summary for ${identity}` }); + } + if (!candidateRuns.has(identity)) { + candidate.issues.push({ path: null, reason: `candidate directory has no run summary for ${identity}` }); + } + } + + const keys = [...new Set([...base.receipts.keys(), ...candidate.receipts.keys()])].sort(); + const receiptEntries: ComparisonSetEntryV1[] = keys.map((key) => { + const left = base.receipts.get(key); + const right = candidate.receipts.get(key); + if (!left || !right) { + const missing = left ? "candidate" : "baseline"; + return { + key, + status: "invalid", + basePath: left ? relative(baseRoot, left.path) : null, + candidatePath: right ? relative(candidateRoot, right.path) : null, + comparison: null, + reason: `${missing} directory has no matching receipt`, + }; + } + const comparison = compareReceipts(left.receipt, right.receipt, budget); + return { + key, + status: comparison.status, + basePath: relative(baseRoot, left.path), + candidatePath: relative(candidateRoot, right.path), + comparison, + reason: null, + }; + }); + const entries = [ + ...issueEntries("baseline", baseRoot, base.issues), + ...issueEntries("candidate", candidateRoot, candidate.issues), + ...receiptEntries, + ]; + const status = worst(entries.map((entry) => entry.status)); + return { + schemaVersion: 1, + kind: "pocketjs.perf.comparison-set", + status, + comparable: entries.length > 0 && entries.every((entry) => entry.comparison?.comparable === true), + entries, + }; +} + +function comparisonSetToMarkdown(result: ComparisonSetV1): string { + const lines = [ + "# PocketJS performance comparison set", + "", + `Status: **${result.status}**`, + "", + "| Receipt | Status | Base | Candidate |", + "| --- | --- | --- | --- |", + ...result.entries.map((entry) => + `| \`${entry.key}\` | **${entry.status}** | ${entry.basePath ? `\`${entry.basePath}\`` : "—"} | ${entry.candidatePath ? `\`${entry.candidatePath}\`` : "—"} |`), + ]; + for (const entry of result.entries) { + if (entry.reason) lines.push("", `- \`${entry.key}\`: ${entry.reason}`); + if (entry.comparison && entry.comparison.status !== "pass") { + lines.push("", `## ${entry.key}`, "", comparisonToMarkdown(entry.comparison).trim()); + } + } + return `${lines.join("\n")}\n`; +} + +export type PathComparison = ComparisonV1 | ComparisonSetV1; + +export interface ComparePathsOptions { + readonly base: string; + readonly candidate: string; + readonly budgetPath?: string; + readonly format: OutputFormat; + readonly out?: string; +} + +export function loadBudget(path?: string): BudgetSetV1 { + return path + ? parseBudgetSetV1(JSON.parse(readFileSync(resolve(path), "utf8"))) + : DEFAULT_BUDGET_SET; +} + +export function comparePaths(options: ComparePathsOptions): { result: PathComparison; rendered: string } { + const base = resolve(options.base); + const candidate = resolve(options.candidate); + if (!existsSync(base)) throw new Error(`baseline path does not exist: ${base}`); + if (!existsSync(candidate)) throw new Error(`candidate path does not exist: ${candidate}`); + const baseDirectory = statSync(base).isDirectory(); + const candidateDirectory = statSync(candidate).isDirectory(); + if (baseDirectory !== candidateDirectory) { + throw new Error("--base and --candidate must both be receipt files or both be receipt directories"); + } + const budget = loadBudget(options.budgetPath); + const result = baseDirectory + ? compareDirectories(base, candidate, budget) + : compareReceipts(readReceipt(base), readReceipt(candidate), budget); + const rendered = options.format === "markdown" + ? result.kind === "pocketjs.perf.comparison-set" + ? comparisonSetToMarkdown(result) + : comparisonToMarkdown(result) + : `${JSON.stringify(result, null, 2)}\n`; + if (options.out) { + const output = resolve(options.out); + mkdirSync(dirname(output), { recursive: true }); + writeFileSync(output, rendered); + } + return { result, rendered }; +} + +export function comparisonExitCode(result: PathComparison): number { + if (result.status === "regression") return 1; + if (result.status === "invalid") return 2; + return 0; +} diff --git a/tools/perf/cli/doctor.ts b/tools/perf/cli/doctor.ts new file mode 100644 index 00000000..a882d250 --- /dev/null +++ b/tools/perf/cli/doctor.ts @@ -0,0 +1,143 @@ +import { existsSync, readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { runCommand } from "./process.ts"; +import { EXECUTOR_IDS, type DoctorCheckV1, type DoctorResultV1, type ExecutorId } from "./types.ts"; + +export const HARNESS_ROOT = resolve(new URL("../../..", import.meta.url).pathname); + +export function qemuBridgePath(harnessRoot = HARNESS_ROOT): string | null { + for (const relative of [ + "tools/perf/executors/qemu.ts", + "tools/perf/qemu/bridge.ts", + "tools/perf/qemu/runner.ts", + ]) { + const path = join(harnessRoot, relative); + if (existsSync(path)) return path; + } + return null; +} + +function executableCheck( + id: string, + argv: readonly string[], + executors: readonly ExecutorId[], + cwd: string, +): DoctorCheckV1 { + const result = runCommand(argv, { cwd }); + const stdout = new TextDecoder().decode(result.stdout).trim(); + const stderr = new TextDecoder().decode(result.stderr).trim(); + const failure = (stderr || stdout).split(/\r?\n/, 1)[0] || `${argv[0]} is unavailable`; + return { + id, + status: result.exitCode === 0 ? "ok" : "missing", + detail: result.exitCode === 0 ? (stdout || stderr).split(/\r?\n/, 1)[0]! : failure, + executors, + }; +} + +function fileCheck(id: string, path: string, executors: readonly ExecutorId[]): DoctorCheckV1 { + return { + id, + status: existsSync(path) ? "ok" : "missing", + detail: existsSync(path) ? path : `missing ${path}`, + executors, + }; +} + +function outputContainsCheck( + id: string, + argv: readonly string[], + expected: string, + executors: readonly ExecutorId[], + cwd: string, +): DoctorCheckV1 { + const result = runCommand(argv, { cwd }); + const output = `${new TextDecoder().decode(result.stdout)}\n${new TextDecoder().decode(result.stderr)}`; + const ok = result.exitCode === 0 && output.split(/\r?\n/).includes(expected); + return { + id, + status: ok ? "ok" : "missing", + detail: ok ? expected : `missing ${expected}`, + executors, + }; +} + +export function runDoctor(harnessRoot = HARNESS_ROOT): DoctorResultV1 { + const qemuExecutors = ["qemu-armv7-thumb2", "qemu-aarch64"] as const; + const all = EXECUTOR_IDS; + const checks: DoctorCheckV1[] = [ + executableCheck("bun", [process.execPath, "--version"], all, harnessRoot), + executableCheck("git", ["git", "--version"], all, harnessRoot), + executableCheck("rustc", ["rustc", "--version"], all, harnessRoot), + executableCheck("cargo", ["cargo", "--version"], all, harnessRoot), + outputContainsCheck( + "wasm-target", + ["rustup", "target", "list", "--installed"], + "wasm32-unknown-unknown", + all, + harnessRoot, + ), + fileCheck("workspace-dependencies", join(harnessRoot, "node_modules"), all), + fileCheck("native-perf-host", join(harnessRoot, "tools/perf/runner/native-world.ts"), all), + fileCheck("wasm-host-binding", join(harnessRoot, "hosts/web/wasm-ops.js"), all), + fileCheck("build-driver", join(harnessRoot, "tools/build.ts"), all), + fileCheck("scenario-schema", join(harnessRoot, "tools/perf/core/schema.ts"), all), + executableCheck("docker-cli", ["docker", "--version"], qemuExecutors, harnessRoot), + executableCheck("docker-daemon", ["docker", "info", "--format", "{{.ServerVersion}}"], qemuExecutors, harnessRoot), + fileCheck("qemu-plugin", join(harnessRoot, "tools/perf/qemu/perf_counter.c"), qemuExecutors), + fileCheck("qemu-container", join(harnessRoot, "tools/perf/qemu/Dockerfile"), qemuExecutors), + fileCheck("qemu-runner-container", join(harnessRoot, "tools/perf/qemu/Dockerfile.runner"), qemuExecutors), + executableCheck( + "qemu-runner-image", + ["docker", "image", "inspect", "pocketjs-perf-qemu:11.0.3", "--format", "{{.Id}}"], + qemuExecutors, + harnessRoot, + ), + { + id: "qemu-version-pin", + status: existsSync(join(harnessRoot, "tools/perf/qemu/Dockerfile")) && + readFileSync(join(harnessRoot, "tools/perf/qemu/Dockerfile"), "utf8").includes("11.0.3") + ? "ok" + : "mismatch", + detail: "QEMU linux-user and plugin must be built from pinned 11.0.3 sources", + executors: qemuExecutors, + }, + { + id: "qemu-run-bridge", + status: qemuBridgePath(harnessRoot) ? "ok" : "missing", + detail: qemuBridgePath(harnessRoot) ?? "missing tools/perf/executors/qemu.ts", + executors: qemuExecutors, + }, + ]; + const executorResult = (executor: ExecutorId): DoctorResultV1["executors"][ExecutorId] => { + const failed = checks.filter((check) => check.executors.includes(executor) && check.status !== "ok"); + return { + ready: failed.length === 0, + reasons: failed.map((check) => `${check.id}: ${check.detail}`), + }; + }; + const executorResults: DoctorResultV1["executors"] = { + native: executorResult("native"), + "qemu-armv7-thumb2": executorResult("qemu-armv7-thumb2"), + "qemu-aarch64": executorResult("qemu-aarch64"), + }; + return { + schemaVersion: 1, + kind: "pocketjs.perf.doctor", + status: Object.values(executorResults).every((entry) => entry.ready) ? "ok" : "missing", + checks, + executors: executorResults, + }; +} + +export function doctorToText(result: DoctorResultV1): string { + const lines = ["PocketJS performance doctor", ""]; + for (const check of result.checks) { + lines.push(`${check.status === "ok" ? "ok" : check.status}: ${check.id} — ${check.detail}`); + } + lines.push(""); + for (const executor of EXECUTOR_IDS) { + lines.push(`${executor}: ${result.executors[executor].ready ? "ready" : "not ready"}`); + } + return `${lines.join("\n")}\n`; +} diff --git a/tools/perf/cli/executors.ts b/tools/perf/cli/executors.ts new file mode 100644 index 00000000..2fd01633 --- /dev/null +++ b/tools/perf/cli/executors.ts @@ -0,0 +1,294 @@ +import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { isAbsolute, join, relative, resolve, sep } from "node:path"; +import { tmpdir } from "node:os"; +import { pathToFileURL } from "node:url"; +import { + artifactBuildVariantKey, + buildRenderConfig, + parseReceiptV1, + type ReceiptV1, + type ScenarioV1, +} from "../core/index.ts"; +import { loadScenarioSuite, estimatedSuiteSeconds, expandSuiteFrameworks } from "../runner/suite.ts"; +import type { NativeRunResult } from "../runner/native.ts"; +import { isDamageScenario } from "../executors/damage.ts"; +import { NATIVE_RUN_OUTPUT_PREFIX, parseNativeResult } from "../receipts/native-protocol.ts"; +import { HARNESS_ROOT, qemuBridgePath } from "./doctor.ts"; +import { runCommand } from "./process.ts"; +import { nativeResultToReceipt, writeReceipt } from "./receipts.ts"; +import type { ExecutorId, PerfRunSummaryV1, QemuBridge } from "./types.ts"; + +export interface RunExecutorOptions { + readonly executor: ExecutorId; + readonly suite: string; + readonly sourceRoot?: string; + readonly harnessRoot?: string; + readonly scenarioDir?: string; + readonly outDir?: string; + readonly maxEstimatedSeconds: number; +} + +function prepareOutput(path?: string): string { + const result = path + ? resolve(path) + : mkdtempSync(join(tmpdir(), "pocketjs-perf-run-")); + mkdirSync(result, { recursive: true }); + return result; +} + +function writeSummary(summary: PerfRunSummaryV1): PerfRunSummaryV1 { + mkdirSync(summary.outputDir, { recursive: true }); + const outputDir = resolve(summary.outputDir); + const receipts = summary.receipts.map((path) => { + const absolute = isAbsolute(path) ? resolve(path) : resolve(outputDir, path); + const local = relative(outputDir, absolute); + return local.length > 0 && local !== ".." && !local.startsWith(`..${sep}`) && !isAbsolute(local) + ? local + : path; + }); + const portable = { ...summary, outputDir, receipts }; + writeFileSync(join(outputDir, "run.json"), `${JSON.stringify(portable, null, 2)}\n`); + return portable; +} + +function invalidSummary(options: Required> & { + sourceRoot: string; + outDir: string; +}, reasons: readonly string[]): PerfRunSummaryV1 { + return writeSummary({ + schemaVersion: 1, + kind: "pocketjs.perf.run", + status: "invalid", + executor: options.executor, + suite: options.suite, + sourceRoot: options.sourceRoot, + outputDir: options.outDir, + receipts: [], + invalidReasons: reasons, + }); +} + +function buildGuestApp( + sourceRoot: string, + scenario: ScenarioV1, +): string | null { + if (scenario.subject.family !== "guest-app") return null; + const render = buildRenderConfig(scenario.params); + const outDir = join(sourceRoot, "dist"); + const argv = [ + process.execPath, + join(sourceRoot, "tools/build.ts"), + scenario.subject.id, + `--framework=${scenario.subject.framework}`, + `--density=${render.rasterDensity}`, + `--outdir=${outDir}`, + ]; + const build = runCommand(argv, { cwd: sourceRoot }); + if (build.exitCode === 0) return null; + const detail = new TextDecoder().decode(build.stderr).trim(); + return `${scenario.id}: build failed (${build.exitCode})${detail ? `: ${detail}` : ""}`; +} + +function buildNativeRuntime(sourceRoot: string): string | null { + const build = runCommand([process.execPath, join(sourceRoot, "tools/wasm.ts")], { + cwd: sourceRoot, + }); + if (build.exitCode === 0) return null; + const detail = new TextDecoder().decode(build.stderr).trim(); + return `WASM runtime build failed (${build.exitCode})${detail ? `: ${detail}` : ""}`; +} + +function runIsolatedNativeScenario( + scenario: ScenarioV1, + options: { sourceRoot: string; harnessRoot: string; rawOutDir: string }, +): { result: NativeRunResult; scenarioOutDir: string } { + const safe = `${scenario.id}.${scenario.subject.framework}`.replace(/[^a-zA-Z0-9._-]+/g, "-"); + const scenarioOutDir = join(options.rawOutDir, safe); + const scenarioPath = join(scenarioOutDir, "scenario.json"); + mkdirSync(scenarioOutDir, { recursive: true }); + writeFileSync(scenarioPath, `${JSON.stringify(scenario, null, 2)}\n`); + const command = runCommand([ + process.execPath, + join(options.harnessRoot, "tools/perf/runner/native-cli.ts"), + scenarioPath, + "--source-root", options.sourceRoot, + "--harness-root", options.harnessRoot, + "--out-dir", scenarioOutDir, + ], { cwd: options.harnessRoot }); + const stdout = new TextDecoder().decode(command.stdout); + const records = stdout.split(/\r?\n/) + .filter((line) => line.startsWith(NATIVE_RUN_OUTPUT_PREFIX)); + if (records.length !== 1) { + const detail = new TextDecoder().decode(command.stderr).trim(); + throw new Error( + `${scenario.id}: isolated Native runner emitted ${records.length} result records` + + `${detail ? `: ${detail}` : ""}`, + ); + } + let value: unknown; + try { + value = JSON.parse(records[0]!.slice(NATIVE_RUN_OUTPUT_PREFIX.length)); + } catch (error) { + throw new Error(`${scenario.id}: isolated Native result is not JSON: ${String(error)}`); + } + const parsed = parseNativeResult(value); + if (!parsed.success) { + throw new Error(`${scenario.id}: ${parsed.reasons.join("; ")}`); + } + if (command.exitCode !== 0 && !(command.exitCode === 2 && parsed.data.status === "unsupported")) { + const detail = new TextDecoder().decode(command.stderr).trim(); + throw new Error(`${scenario.id}: isolated Native runner failed (${command.exitCode})${detail ? `: ${detail}` : ""}`); + } + return { result: parsed.data, scenarioOutDir }; +} + +async function runNative(options: RunExecutorOptions & { + sourceRoot: string; + scenarioDir: string; + outDir: string; +}): Promise { + let scenarios: ScenarioV1[]; + try { + scenarios = expandSuiteFrameworks(loadScenarioSuite(options.suite, options.scenarioDir)); + if (scenarios.length === 0) return invalidSummary(options, [`no scenarios found for suite ${JSON.stringify(options.suite)}`]); + const estimate = estimatedSuiteSeconds(scenarios); + if (estimate > options.maxEstimatedSeconds) { + return invalidSummary(options, [ + `${options.suite} suite estimate ${estimate}s exceeds the ${options.maxEstimatedSeconds}s limit`, + ]); + } + } catch (error) { + return invalidSummary(options, [error instanceof Error ? error.message : String(error)]); + } + + try { + const receipts: string[] = []; + const invalidReasons: string[] = []; + if (scenarios.some((scenario) => scenario.subject.family === "guest-app")) { + const runtimeFailure = buildNativeRuntime(options.sourceRoot); + if (runtimeFailure) return invalidSummary(options, [runtimeFailure]); + } + const builtVariant = new Map(); + // Build immediately before each replay. Framework variants may share an + // output name, so prebuilding the complete matrix would make the final + // variant silently replace earlier bundles. + for (const scenario of scenarios) { + const variant = artifactBuildVariantKey(scenario); + const alreadyBuilt = builtVariant.get(scenario.subject.entry) === variant; + const buildFailure = alreadyBuilt ? null : buildGuestApp(options.sourceRoot, scenario); + if (buildFailure) { + invalidReasons.push(buildFailure); + continue; + } + if (scenario.subject.family === "guest-app") builtVariant.set(scenario.subject.entry, variant); + const rawOutDir = join(options.outDir, "raw"); + const isolated = runIsolatedNativeScenario(scenario, { + sourceRoot: options.sourceRoot, + harnessRoot: options.harnessRoot ?? HARNESS_ROOT, + rawOutDir, + }); + const nativeResult = isolated.result; + const extraArtifacts = isDamageScenario(scenario) + ? [join(isolated.scenarioOutDir, "damage-fixture", "target", "release", "pocketjs-perf-damage")] + : []; + const receipt = nativeResultToReceipt( + nativeResult, + scenario, + options.sourceRoot, + extraArtifacts, + ); + receipts.push(writeReceipt(options.outDir, receipt)); + if (receipt.status === "invalid") invalidReasons.push(...receipt.invalidReasons.map((reason) => `${scenario.id}: ${reason}`)); + } + return writeSummary({ + schemaVersion: 1, + kind: "pocketjs.perf.run", + status: invalidReasons.length === 0 ? "valid" : "invalid", + executor: "native", + suite: options.suite, + sourceRoot: options.sourceRoot, + outputDir: options.outDir, + receipts, + invalidReasons, + }); + } catch (error) { + return invalidSummary(options, [error instanceof Error ? error.message : String(error)]); + } +} + +function qemuReceipts(value: unknown): { receipts: readonly ReceiptV1[]; invalidReasons: readonly string[] } | null { + if (Array.isArray(value)) return { receipts: value.map(parseReceiptV1), invalidReasons: [] }; + if (typeof value !== "object" || value === null) return null; + const record = value as Record; + if (!Array.isArray(record.receipts)) return null; + return { + receipts: record.receipts.map(parseReceiptV1), + invalidReasons: Array.isArray(record.invalidReasons) + ? record.invalidReasons.map((reason) => String(reason)) + : [], + }; +} + +async function runQemu(options: RunExecutorOptions & { + sourceRoot: string; + harnessRoot: string; + scenarioDir: string; + outDir: string; +}): Promise { + const bridgePath = qemuBridgePath(options.harnessRoot); + if (!bridgePath) { + return invalidSummary(options, [ + "QEMU executor bridge is unavailable; expected tools/perf/executors/qemu.ts", + ]); + } + try { + const module = await import(pathToFileURL(bridgePath).href) as Partial; + if (typeof module.runQemuSuite !== "function") { + return invalidSummary(options, [`${bridgePath} does not export runQemuSuite(options)`]); + } + const bridgeResult = await module.runQemuSuite({ + executor: options.executor as Exclude, + suite: options.suite, + sourceRoot: options.sourceRoot, + harnessRoot: options.harnessRoot, + scenarioDir: options.scenarioDir, + outDir: options.outDir, + maxEstimatedSeconds: options.maxEstimatedSeconds, + }); + if (typeof bridgeResult === "object" && bridgeResult !== null && + "kind" in bridgeResult && bridgeResult.kind === "pocketjs.perf.run") { + return writeSummary(bridgeResult as PerfRunSummaryV1); + } + const normalized = qemuReceipts(bridgeResult); + if (!normalized) return invalidSummary(options, ["QEMU bridge returned an unsupported result"]); + const paths = normalized.receipts.map((receipt) => writeReceipt(options.outDir, receipt)); + const receiptReasons = normalized.receipts.flatMap((receipt) => + receipt.status === "invalid" ? receipt.invalidReasons : []); + const invalidReasons = [...normalized.invalidReasons, ...receiptReasons]; + return writeSummary({ + schemaVersion: 1, + kind: "pocketjs.perf.run", + status: invalidReasons.length === 0 ? "valid" : "invalid", + executor: options.executor, + suite: options.suite, + sourceRoot: options.sourceRoot, + outputDir: options.outDir, + receipts: paths, + invalidReasons, + }); + } catch (error) { + return invalidSummary(options, [error instanceof Error ? error.message : String(error)]); + } +} + +export async function runExecutor(options: RunExecutorOptions): Promise { + const harnessRoot = resolve(options.harnessRoot ?? HARNESS_ROOT); + const sourceRoot = resolve(options.sourceRoot ?? harnessRoot); + const scenarioDir = resolve(options.scenarioDir ?? join(harnessRoot, "tools/perf/scenarios")); + const outDir = prepareOutput(options.outDir); + if (!existsSync(sourceRoot)) return invalidSummary({ ...options, sourceRoot, outDir }, [`source root does not exist: ${sourceRoot}`]); + if (!existsSync(scenarioDir)) return invalidSummary({ ...options, sourceRoot, outDir }, [`scenario directory does not exist: ${scenarioDir}`]); + return options.executor === "native" + ? runNative({ ...options, sourceRoot, scenarioDir, outDir }) + : runQemu({ ...options, sourceRoot, harnessRoot, scenarioDir, outDir }); +} diff --git a/tools/perf/cli/local.ts b/tools/perf/cli/local.ts new file mode 100644 index 00000000..075c3d3e --- /dev/null +++ b/tools/perf/cli/local.ts @@ -0,0 +1,223 @@ +import { cpSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import type { OutputFormat } from "./args.ts"; +import { comparePaths, type PathComparison } from "./compare-paths.ts"; +import { runExecutor, type RunExecutorOptions } from "./executors.ts"; +import { HARNESS_ROOT } from "./doctor.ts"; +import { commandText, runCommand } from "./process.ts"; +import type { ExecutorId, LocalInvalidResultV1, PerfRunSummaryV1 } from "./types.ts"; + +export interface LocalOptions { + readonly base: string; + readonly executors: readonly ExecutorId[]; + readonly suite: string; + readonly repoRoot?: string; + readonly harnessRoot?: string; + readonly scenarioDir?: string; + readonly budgetPath?: string; + readonly format: OutputFormat; + readonly out?: string; + readonly maxEstimatedSeconds: number; +} + +export interface LocalDependencies { + readonly runExecutor?: (options: RunExecutorOptions) => Promise; +} + +export interface LocalSuccess { + readonly result: PathComparison; + readonly rendered: string; +} + +export interface LocalInvalid { + readonly result: LocalInvalidResultV1; + readonly rendered: string; +} + +function git(argv: readonly string[], cwd: string): void { + const result = runCommand(["git", ...argv], { cwd }); + if (result.exitCode !== 0) { + const detail = new TextDecoder().decode(result.stderr).trim(); + throw new Error(`git ${argv.join(" ")} failed (${result.exitCode})${detail ? `: ${detail}` : ""}`); + } +} + +function addWorktree(repoRoot: string, path: string, revision: string): void { + git(["worktree", "add", "--detach", path, revision], repoRoot); +} + +function overlayTrackedChanges(repoRoot: string, candidateRoot: string): void { + // `git diff HEAD` includes index and working-tree changes to tracked files. + // It deliberately excludes every untracked file, including local research + // notes. The perf harness continues running from harnessRoot instead. + const patch = runCommand(["git", "diff", "--binary", "--no-ext-diff", "HEAD", "--"], { cwd: repoRoot }); + if (patch.exitCode !== 0) throw new Error("failed to capture tracked candidate changes"); + if (patch.stdout.byteLength === 0) return; + const applied = runCommand(["git", "apply", "--binary", "--whitespace=nowarn", "-"], { + cwd: candidateRoot, + stdin: patch.stdout, + }); + if (applied.exitCode !== 0) { + const detail = new TextDecoder().decode(applied.stderr).trim(); + throw new Error(`failed to overlay tracked candidate changes${detail ? `: ${detail}` : ""}`); + } +} + +function installWorkspaceDependencies(sourceRoot: string, label: "baseline" | "candidate"): void { + const installed = runCommand([process.execPath, "install", "--frozen-lockfile"], { cwd: sourceRoot }); + if (installed.exitCode === 0) return; + const stderr = new TextDecoder().decode(installed.stderr).trim(); + const stdout = new TextDecoder().decode(installed.stdout).trim(); + const detail = stderr || stdout; + throw new Error( + `${label} dependency install failed (${installed.exitCode})${detail ? `: ${detail}` : ""}`, + ); +} + +function stageBenchmarkApps(harnessRoot: string, sourceRoot: string): void { + const fixtures = join(harnessRoot, "tools", "perf", "apps"); + if (!existsSync(fixtures)) return; + const destination = join(sourceRoot, "tools", "perf", "apps"); + mkdirSync(destination, { recursive: true }); + // These sources define the workload, like scenarios and input tapes. Both + // revisions compile the same harness app against their own framework/core. + cpSync(fixtures, destination, { recursive: true, force: true }); +} + +function safeRemoveTempRoot(path: string): void { + const expectedPrefix = join(tmpdir(), "pocketjs-perf-local-"); + const absolute = resolve(path); + if (!absolute.startsWith(expectedPrefix)) { + throw new Error(`refusing to remove unexpected local perf path: ${absolute}`); + } + rmSync(absolute, { recursive: true, force: true }); +} + +function invalidResult(options: LocalOptions, reasons: readonly string[]): LocalInvalidResultV1 { + return { + schemaVersion: 1, + kind: "pocketjs.perf.local", + status: "invalid", + baseRef: options.base, + suite: options.suite, + executors: options.executors, + invalidReasons: reasons, + temporaryWorktreesCleaned: true, + }; +} + +function writeOptionalReport(path: string | undefined, rendered: string): void { + if (!path) return; + const output = resolve(path); + mkdirSync(dirname(output), { recursive: true }); + writeFileSync(output, rendered); +} + +export async function runLocal( + options: LocalOptions, + dependencies: LocalDependencies = {}, +): Promise { + const harnessRoot = resolve(options.harnessRoot ?? HARNESS_ROOT); + const scenarioDir = resolve(options.scenarioDir ?? join(harnessRoot, "tools/perf/scenarios")); + const execute = dependencies.runExecutor ?? runExecutor; + + // Resolve before registering a worktree, so an invalid user ref has no side effects. + let repoRoot: string; + let baseCommit: string; + let candidateCommit: string; + try { + repoRoot = resolve(options.repoRoot ?? commandText(["git", "rev-parse", "--show-toplevel"], harnessRoot)); + baseCommit = commandText(["git", "rev-parse", "--verify", `${options.base}^{commit}`], repoRoot); + candidateCommit = commandText(["git", "rev-parse", "--verify", "HEAD^{commit}"], repoRoot); + } catch (error) { + const result = invalidResult(options, [error instanceof Error ? error.message : String(error)]); + const rendered = `${JSON.stringify(result, null, 2)}\n`; + writeOptionalReport(options.out, rendered); + return { result, rendered }; + } + const tempRoot = mkdtempSync(join(tmpdir(), "pocketjs-perf-local-")); + const baselineRoot = join(tempRoot, "baseline-source"); + const candidateRoot = join(tempRoot, "candidate-source"); + const baseReceipts = join(tempRoot, "receipts", "base"); + const candidateReceipts = join(tempRoot, "receipts", "candidate"); + let baselineAdded = false; + let candidateAdded = false; + let outcome: LocalSuccess | LocalInvalid; + + try { + addWorktree(repoRoot, baselineRoot, baseCommit); + baselineAdded = true; + addWorktree(repoRoot, candidateRoot, candidateCommit); + candidateAdded = true; + overlayTrackedChanges(repoRoot, candidateRoot); + stageBenchmarkApps(harnessRoot, baselineRoot); + stageBenchmarkApps(harnessRoot, candidateRoot); + // Bun's global download cache remains shared, but dependency resolution and + // node_modules are derived independently from each source snapshot's own + // package manifest and frozen lockfile. + installWorkspaceDependencies(baselineRoot, "baseline"); + installWorkspaceDependencies(candidateRoot, "candidate"); + + const invalidReasons: string[] = []; + for (const executor of options.executors) { + const common = { + executor, + suite: options.suite, + harnessRoot, + scenarioDir, + maxEstimatedSeconds: options.maxEstimatedSeconds, + } as const; + const baseline = await execute({ + ...common, + sourceRoot: baselineRoot, + outDir: join(baseReceipts, executor), + }); + const candidate = await execute({ + ...common, + sourceRoot: candidateRoot, + outDir: join(candidateReceipts, executor), + }); + if (baseline.status === "invalid") { + invalidReasons.push(...baseline.invalidReasons.map((reason) => `${executor} baseline: ${reason}`)); + } + if (candidate.status === "invalid") { + invalidReasons.push(...candidate.invalidReasons.map((reason) => `${executor} candidate: ${reason}`)); + } + } + if (invalidReasons.length > 0) { + const result = invalidResult(options, invalidReasons); + const rendered = `${JSON.stringify(result, null, 2)}\n`; + writeOptionalReport(options.out, rendered); + outcome = { result, rendered }; + } else { + outcome = comparePaths({ + base: baseReceipts, + candidate: candidateReceipts, + budgetPath: options.budgetPath, + format: options.format, + out: options.out, + }); + } + } catch (error) { + const result = invalidResult(options, [error instanceof Error ? error.message : String(error)]); + const rendered = `${JSON.stringify(result, null, 2)}\n`; + writeOptionalReport(options.out, rendered); + outcome = { result, rendered }; + } finally { + // Remove registrations before deleting their directories. Both commands + // target only worktrees created under this invocation's mkdtemp root. + if (candidateAdded) runCommand(["git", "worktree", "remove", "--force", candidateRoot], { cwd: repoRoot }); + if (baselineAdded) runCommand(["git", "worktree", "remove", "--force", baselineRoot], { cwd: repoRoot }); + try { + safeRemoveTempRoot(tempRoot); + } finally { + // macOS may register /var/... worktrees under their /private/var/... + // canonical path. If `worktree remove` misses that alias, deleting the + // known mkdtemp root makes it safely prunable; expire it immediately so + // `perf local` never leaves stale registrations behind. + runCommand(["git", "worktree", "prune", "--expire", "now"], { cwd: repoRoot }); + } + } + return outcome!; +} diff --git a/tools/perf/cli/main.ts b/tools/perf/cli/main.ts new file mode 100644 index 00000000..db7c804c --- /dev/null +++ b/tools/perf/cli/main.ts @@ -0,0 +1,73 @@ +import { parsePerfCommand, HELP, UsageError } from "./args.ts"; +import { comparePaths, comparisonExitCode } from "./compare-paths.ts"; +import { doctorToText, runDoctor } from "./doctor.ts"; +import { runExecutor } from "./executors.ts"; +import { runLocal } from "./local.ts"; + +export interface CliIo { + readonly stdout: (value: string) => void; + readonly stderr: (value: string) => void; +} + +const DEFAULT_IO: CliIo = { + stdout(value) { process.stdout.write(value); }, + stderr(value) { process.stderr.write(value); }, +}; + +export async function runPerfCli(args: readonly string[], io: CliIo = DEFAULT_IO): Promise { + try { + const parsed = parsePerfCommand(args); + if (parsed.command === "help") { + io.stdout(HELP); + return 0; + } + if (parsed.command === "doctor") { + const result = runDoctor(); + io.stdout(parsed.format === "json" ? `${JSON.stringify(result, null, 2)}\n` : doctorToText(result)); + return result.status === "ok" ? 0 : 2; + } + if (parsed.command === "run") { + const result = await runExecutor({ + executor: parsed.executor, + suite: parsed.suite, + sourceRoot: parsed.sourceRoot, + scenarioDir: parsed.scenarioDir, + outDir: parsed.outDir, + maxEstimatedSeconds: parsed.maxEstimatedSeconds, + }); + io.stdout(`${JSON.stringify(result, null, 2)}\n`); + return result.status === "valid" ? 0 : 2; + } + if (parsed.command === "compare") { + const { result, rendered } = comparePaths({ + base: parsed.base, + candidate: parsed.candidate, + budgetPath: parsed.budget, + format: parsed.format, + out: parsed.out, + }); + io.stdout(rendered); + return comparisonExitCode(result); + } + const { result, rendered } = await runLocal({ + base: parsed.base, + executors: parsed.executors, + suite: parsed.suite, + scenarioDir: parsed.scenarioDir, + budgetPath: parsed.budget, + format: parsed.format, + out: parsed.out, + maxEstimatedSeconds: parsed.maxEstimatedSeconds, + }); + io.stdout(rendered); + if (result.kind === "pocketjs.perf.local") return 2; + return comparisonExitCode(result); + } catch (error) { + if (error instanceof UsageError) { + io.stderr(`perf: ${error.message}\n\n${HELP}`); + return 2; + } + io.stderr(`perf: ${error instanceof Error ? error.message : String(error)}\n`); + return 2; + } +} diff --git a/tools/perf/cli/process.ts b/tools/perf/cli/process.ts new file mode 100644 index 00000000..3090ae6b --- /dev/null +++ b/tools/perf/cli/process.ts @@ -0,0 +1,55 @@ +import { createHash } from "node:crypto"; + +export interface CommandResult { + readonly exitCode: number; + readonly stdout: Uint8Array; + readonly stderr: Uint8Array; +} + +export function runCommand( + argv: readonly string[], + options: { readonly cwd: string; readonly stdin?: Uint8Array } , +): CommandResult { + const child = Bun.spawnSync(argv as string[], { + cwd: options.cwd, + stdin: options.stdin, + stdout: "pipe", + stderr: "pipe", + }); + return { + exitCode: child.exitCode, + stdout: child.stdout, + stderr: child.stderr, + }; +} + +export function commandText( + argv: readonly string[], + cwd: string, + options: { readonly allowFailure?: boolean } = {}, +): string { + const result = runCommand(argv, { cwd }); + if (result.exitCode !== 0 && !options.allowFailure) { + const detail = new TextDecoder().decode(result.stderr).trim(); + throw new Error(`${argv.join(" ")} failed (${result.exitCode})${detail ? `: ${detail}` : ""}`); + } + return new TextDecoder().decode(result.stdout).trim(); +} + +export function sha256(value: string | Uint8Array): string { + return createHash("sha256").update(value).digest("hex"); +} + +export function canonicalJson(value: unknown): string { + if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value); + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new Error("cannot encode a non-finite JSON number"); + return JSON.stringify(value); + } + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object") { + const record = value as Record; + return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(",")}}`; + } + throw new Error(`cannot encode ${typeof value} as canonical JSON`); +} diff --git a/tools/perf/cli/receipts.ts b/tools/perf/cli/receipts.ts new file mode 100644 index 00000000..01817d2b --- /dev/null +++ b/tools/perf/cli/receipts.ts @@ -0,0 +1,158 @@ +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { + parseReceiptV1, + type ReceiptProvenanceV1, + type ReceiptV1, + type ScenarioV1, +} from "../core/index.ts"; +import { createNativeReceipt } from "../receipts/index.ts"; +import type { NativeRunResult } from "../runner/native.ts"; +import { canonicalJson, commandText, runCommand, sha256 } from "./process.ts"; + +const BENCHMARK_ROOT = resolve(new URL("../../..", import.meta.url).pathname); + +function firstLine(value: string): string { + return value.split(/\r?\n/, 1)[0]?.trim() || "unavailable"; +} + +function commandVersion(argv: readonly string[], cwd: string): string { + const result = runCommand(argv, { cwd }); + return result.exitCode === 0 + ? firstLine(new TextDecoder().decode(result.stdout) || new TextDecoder().decode(result.stderr)) + : "unavailable"; +} + +function sourceProvenance(sourceRoot: string): ReceiptProvenanceV1["source"] { + const revision = commandText(["git", "rev-parse", "HEAD"], sourceRoot); + const diff = runCommand(["git", "diff", "--binary", "--no-ext-diff", "HEAD", "--"], { cwd: sourceRoot }); + if (diff.exitCode !== 0) throw new Error(`cannot inspect tracked source changes in ${sourceRoot}`); + const dirty = diff.stdout.byteLength > 0; + const content = createHash("sha256").update(revision).update("\0").update(diff.stdout).digest("hex"); + return { revision, dirty, contentHash: content }; +} + +function artifactHash( + sourceRoot: string, + scenario: ScenarioV1, + extraArtifacts: readonly string[] = [], +): string { + const hash = createHash("sha256"); + let count = 0; + const artifacts: readonly (readonly [string, string])[] = [ + ...scenario.subject.family === "guest-app" ? [ + ["wasm", join(sourceRoot, "hosts/web/pocketjs.wasm")] as const, + ["js", join(sourceRoot, "dist", `${scenario.subject.entry}.js`)] as const, + ["pak", join(sourceRoot, "dist", `${scenario.subject.entry}.pak`)] as const, + ] : [], + ...scenario.subject.family === "vapor" ? [ + ["vapor-source", join(sourceRoot, scenario.subject.entry)] as const, + ] : [], + ...extraArtifacts.map((path, index) => [`extra-${index}`, path] as const), + ]; + for (const [label, path] of artifacts) { + if (!existsSync(path)) continue; + const bytes = readFileSync(path); + hash.update(label).update("\0").update(String(bytes.byteLength)).update("\0").update(bytes); + count += 1; + } + if (count === 0) hash.update("pocketjs.perf.no-native-artifact.v1"); + return hash.digest("hex"); +} + +function nativeExecutorFingerprint(): string { + const hash = createHash("sha256").update("pocketjs.native-perf-host.v1\0"); + for (const relative of [ + "tools/perf/core/render-config.ts", + "tools/perf/runner/native.ts", + "tools/perf/runner/native-cli.ts", + "tools/perf/runner/native-world.ts", + "tools/perf/runner/input.ts", + "tools/perf/apps/idle-fixture-main.tsx", + "tools/perf/apps/list-fixture-main.tsx", + "tools/perf/apps/keyed-list-model.ts", + "tools/perf/executors/damage.ts", + "tools/perf/executors/vapor.ts", + "tools/perf/damage-fixture/src/main.rs", + "tools/perf/damage-fixture/Cargo.lock", + "tools/perf/receipts/factory.ts", + "tools/perf/receipts/hash.ts", + "tools/perf/receipts/native-protocol.ts", + "hosts/web/wasm-ops.js", + "framework/src/touch.ts", + "bun.lock", + ]) { + const bytes = readFileSync(join(BENCHMARK_ROOT, relative)); + hash.update(relative).update("\0").update(String(bytes.byteLength)).update("\0").update(bytes); + } + return hash.digest("hex"); +} + +export function nativeProvenance( + sourceRootInput: string, + scenario: ScenarioV1, + extraArtifacts: readonly string[] = [], +): ReceiptProvenanceV1 { + const sourceRoot = resolve(sourceRootInput); + return { + source: sourceProvenance(sourceRoot), + scenario: { + id: scenario.id, + suite: scenario.suite, + framework: scenario.subject.framework, + manifestHash: sha256(canonicalJson(scenario)), + inputTapeHash: sha256(canonicalJson(scenario.tape)), + }, + toolchain: { + rustc: commandVersion(["rustc", "--version"], sourceRoot), + cCompiler: commandVersion(["cc", "--version"], sourceRoot), + sysroot: commandText(["rustc", "--print", "sysroot"], sourceRoot, { allowFailure: true }) || "unavailable", + }, + build: { + target: `${process.arch}-${process.platform}`, + profile: "native-sim", + rustFlags: [], + cFlags: [], + linkerFlags: [], + }, + executor: { + id: "native", + version: `bun ${Bun.version}`, + profile: "host-diagnostic", + fingerprint: nativeExecutorFingerprint(), + }, + binary: { sha256: artifactHash(sourceRoot, scenario, extraArtifacts) }, + }; +} + +export function nativeResultToReceipt( + result: NativeRunResult, + scenario: ScenarioV1, + sourceRoot: string, + extraArtifacts: readonly string[] = [], +): ReceiptV1 { + const complete = nativeProvenance(sourceRoot, scenario, extraArtifacts); + const { scenario: _scenario, ...environment } = complete; + return createNativeReceipt(scenario, result, { + provenance: environment, + }); +} + +export function receiptFileName(receipt: ReceiptV1): string { + const safe = `${receipt.provenance.scenario.id}.${receipt.provenance.scenario.framework}.${receipt.provenance.executor.id}` + .replace(/[^a-zA-Z0-9._-]+/g, "-"); + return `${safe}.receipt.json`; +} + +export function writeReceipt(outDir: string, receipt: ReceiptV1): string { + mkdirSync(outDir, { recursive: true }); + const path = join(outDir, receiptFileName(receipt)); + writeFileSync(path, `${JSON.stringify(parseReceiptV1(receipt), null, 2)}\n`); + return path; +} + +export function readReceipt(path: string): ReceiptV1 { + if (!statSync(path).isFile()) throw new Error(`receipt is not a file: ${path}`); + return parseReceiptV1(JSON.parse(readFileSync(path, "utf8"))); +} diff --git a/tools/perf/cli/types.ts b/tools/perf/cli/types.ts new file mode 100644 index 00000000..8b6a3962 --- /dev/null +++ b/tools/perf/cli/types.ts @@ -0,0 +1,86 @@ +import type { ComparisonStatus, ComparisonV1, ReceiptV1 } from "../core/index.ts"; + +export const EXECUTOR_IDS = [ + "native", + "qemu-armv7-thumb2", + "qemu-aarch64", +] as const; + +export type ExecutorId = (typeof EXECUTOR_IDS)[number]; + +export interface PerfRunSummaryV1 { + readonly schemaVersion: 1; + readonly kind: "pocketjs.perf.run"; + readonly status: "valid" | "invalid"; + readonly executor: ExecutorId; + readonly suite: string; + readonly sourceRoot: string; + readonly outputDir: string; + readonly receipts: readonly string[]; + readonly invalidReasons: readonly string[]; +} + +export interface ComparisonSetEntryV1 { + readonly key: string; + readonly status: ComparisonStatus; + readonly basePath: string | null; + readonly candidatePath: string | null; + readonly comparison: ComparisonV1 | null; + readonly reason: string | null; +} + +export interface ComparisonSetV1 { + readonly schemaVersion: 1; + readonly kind: "pocketjs.perf.comparison-set"; + readonly status: ComparisonStatus; + readonly comparable: boolean; + readonly entries: readonly ComparisonSetEntryV1[]; +} + +export interface DoctorCheckV1 { + readonly id: string; + readonly status: "ok" | "missing" | "mismatch"; + readonly detail: string; + readonly executors: readonly ExecutorId[]; +} + +export interface DoctorResultV1 { + readonly schemaVersion: 1; + readonly kind: "pocketjs.perf.doctor"; + readonly status: "ok" | "missing"; + readonly checks: readonly DoctorCheckV1[]; + readonly executors: Readonly>; +} + +export interface LocalInvalidResultV1 { + readonly schemaVersion: 1; + readonly kind: "pocketjs.perf.local"; + readonly status: "invalid"; + readonly baseRef: string; + readonly suite: string; + readonly executors: readonly ExecutorId[]; + readonly invalidReasons: readonly string[]; + readonly temporaryWorktreesCleaned: true; +} + +export interface QemuBridgeOptions { + readonly executor: Exclude; + readonly suite: string; + readonly sourceRoot: string; + /** The current checkout owns the versioned protocol and guest harness. */ + readonly harnessRoot: string; + readonly scenarioDir: string; + readonly outDir: string; + readonly maxEstimatedSeconds: number; +} + +export interface QemuBridge { + runQemuSuite(options: QemuBridgeOptions): Promise< + | PerfRunSummaryV1 + | readonly ReceiptV1[] + | { readonly receipts: readonly ReceiptV1[]; readonly invalidReasons?: readonly string[] } + >; +} diff --git a/tools/perf/core/budgets.ts b/tools/perf/core/budgets.ts new file mode 100644 index 00000000..24a1d01e --- /dev/null +++ b/tools/perf/core/budgets.ts @@ -0,0 +1,69 @@ +import type { BudgetSetV1, MetricBudgetV1 } from "./types.ts"; + +const QEMU_EXECUTORS = ["qemu-armv7-thumb2", "qemu-aarch64"] as const; + +export const DEFAULT_BUDGET_SET: BudgetSetV1 = Object.freeze({ + schemaVersion: 1, + kind: "pocketjs.perf.budget-set", + id: "pocketjs-quick-v1", + metrics: Object.freeze({ + "guest.instructions": { + warn: { relative: 0.005, absolute: 5_000 }, + regression: { relative: 0.01, absolute: 10_000 }, + executors: QEMU_EXECUTORS, + }, + "guest.instruction_bytes": { + warn: { relative: 0.005, absolute: 10 * 1024 }, + regression: { relative: 0.01, absolute: 20 * 1024 }, + executors: QEMU_EXECUTORS, + }, + "guest.load_store_events": { + warn: { relative: 0.01, absolute: 10_000 }, + regression: { relative: 0.02, absolute: 20_000 }, + executors: QEMU_EXECUTORS, + }, + "memory.allocated_bytes": { + warn: { relative: 0.01, absolute: 4 * 1024 }, + regression: { relative: 0.02, absolute: 8 * 1024 }, + executors: QEMU_EXECUTORS, + }, + "quickjs.live_bytes_after_gc": { + warn: { relative: 0.01, absolute: 32 * 1024 }, + regression: { relative: 0.02, absolute: 64 * 1024 }, + executors: QEMU_EXECUTORS, + }, + "artifact.bundle_bytes": { + warn: { relative: 0.01, absolute: 2 * 1024 }, + regression: { relative: 0.03, absolute: 4 * 1024 }, + }, + "artifact.elf_text_rodata_bytes": { + warn: { relative: 0.005, absolute: 2 * 1024 }, + regression: { relative: 0.01, absolute: 4 * 1024 }, + executors: QEMU_EXECUTORS, + }, + }), + scenarios: Object.freeze({ + "vapor.todo.reactive-grid.v1": Object.freeze({ + "memory.allocations": { + hardMax: 0, + executors: QEMU_EXECUTORS, + }, + }), + }), +}); + +/** Return a copied budget set with an explicit absolute bound for one metric. */ +export function withHardLimits( + budgetSet: BudgetSetV1, + metricId: string, + limits: Pick, +): BudgetSetV1 { + const existing = budgetSet.metrics[metricId] ?? {}; + return { + ...budgetSet, + metrics: { + ...budgetSet.metrics, + [metricId]: { ...existing, ...limits }, + }, + }; +} diff --git a/tools/perf/core/catalog.ts b/tools/perf/core/catalog.ts new file mode 100644 index 00000000..30b1a29d --- /dev/null +++ b/tools/perf/core/catalog.ts @@ -0,0 +1,154 @@ +import type { JsonValue, MetricDefinition } from "./types.ts"; + +export const METRIC_CATALOG = { + "guest.instructions": { + id: "guest.instructions", + label: "Guest instructions", + direction: "lower-is-better", + kind: "counter", + unit: "count", + diagnostic: false, + }, + "guest.instruction_bytes": { + id: "guest.instruction_bytes", + label: "Dynamic instruction bytes", + direction: "lower-is-better", + kind: "counter", + unit: "bytes", + diagnostic: false, + }, + "guest.thumb16_instructions": { + id: "guest.thumb16_instructions", + label: "16-bit Thumb instructions", + direction: "lower-is-better", + kind: "counter", + unit: "count", + diagnostic: true, + }, + "guest.thumb32_instructions": { + id: "guest.thumb32_instructions", + label: "32-bit Thumb instructions", + direction: "lower-is-better", + kind: "counter", + unit: "count", + diagnostic: true, + }, + "guest.load_store_events": { + id: "guest.load_store_events", + label: "Guest load/store events", + direction: "lower-is-better", + kind: "counter", + unit: "count", + diagnostic: false, + }, + "guest.loads": { + id: "guest.loads", + label: "Guest loads", + direction: "lower-is-better", + kind: "counter", + unit: "count", + diagnostic: true, + }, + "guest.stores": { + id: "guest.stores", + label: "Guest stores", + direction: "lower-is-better", + kind: "counter", + unit: "count", + diagnostic: true, + }, + "memory.allocations": { + id: "memory.allocations", + label: "Allocations", + direction: "lower-is-better", + kind: "counter", + unit: "count", + diagnostic: false, + }, + "memory.allocated_bytes": { + id: "memory.allocated_bytes", + label: "Allocated bytes", + direction: "lower-is-better", + kind: "counter", + unit: "bytes", + diagnostic: false, + }, + "memory.current_bytes": { + id: "memory.current_bytes", + label: "Current allocated bytes", + direction: "lower-is-better", + kind: "gauge", + unit: "bytes", + diagnostic: true, + }, + "memory.peak_bytes": { + id: "memory.peak_bytes", + label: "Peak bytes above phase baseline", + direction: "lower-is-better", + kind: "gauge", + unit: "bytes", + diagnostic: true, + }, + "quickjs.live_bytes_after_gc": { + id: "quickjs.live_bytes_after_gc", + label: "QuickJS live bytes after GC", + direction: "lower-is-better", + kind: "gauge", + unit: "bytes", + diagnostic: false, + }, + "artifact.bundle_bytes": { + id: "artifact.bundle_bytes", + label: "Bundle size", + direction: "lower-is-better", + kind: "gauge", + unit: "bytes", + diagnostic: false, + }, + "artifact.pak_bytes": { + id: "artifact.pak_bytes", + label: "PAK size", + direction: "lower-is-better", + kind: "gauge", + unit: "bytes", + diagnostic: true, + }, + "artifact.elf_text_rodata_bytes": { + id: "artifact.elf_text_rodata_bytes", + label: "ELF .text + .rodata", + direction: "lower-is-better", + kind: "gauge", + unit: "bytes", + diagnostic: false, + }, + "native.wall_time_ns": { + id: "native.wall_time_ns", + label: "Native wall time", + direction: "lower-is-better", + kind: "gauge", + unit: "ns", + diagnostic: true, + }, +} as const satisfies Record; + +export type MetricId = keyof typeof METRIC_CATALOG; + +export const METRIC_IDS = Object.freeze( + Object.keys(METRIC_CATALOG) as MetricId[], +); + +export function isMetricId(value: string): value is MetricId { + return Object.hasOwn(METRIC_CATALOG, value); +} + +export function metricDefinition(id: MetricId): MetricDefinition { + return METRIC_CATALOG[id]; +} + +/** Scenario schemas validate this field before runners or factories consume it. */ +export function gateMetricIds( + params: Readonly>, +): readonly MetricId[] { + const configured = params.gateMetrics; + return Array.isArray(configured) ? configured as MetricId[] : []; +} diff --git a/tools/perf/core/compare.ts b/tools/perf/core/compare.ts new file mode 100644 index 00000000..48c1a417 --- /dev/null +++ b/tools/perf/core/compare.ts @@ -0,0 +1,571 @@ +import { DEFAULT_BUDGET_SET } from "./budgets.ts"; +import { isMetricId, METRIC_CATALOG, METRIC_IDS, type MetricId } from "./catalog.ts"; +import { + parseBudgetSetV1, + parseComparisonV1, + parseReceiptV1, +} from "./schema.ts"; +import type { + BudgetSetV1, + ComparisonReasonV1, + ComparisonStatus, + ComparisonV1, + MetricBudgetV1, + MetricComparisonV1, + MetricDirection, + MetricSampleV1, + ReceiptV1, + RelativeAbsoluteThresholdV1, +} from "./types.ts"; + +const BOOTSTRAP_SEED = 0x5eedc0de; +const BOOTSTRAP_ITERATIONS = 2_000; + +function valuesEqual(left: unknown, right: unknown): boolean { + if (Array.isArray(left) || Array.isArray(right)) { + return Array.isArray(left) && + Array.isArray(right) && + left.length === right.length && + left.every((value, index) => valuesEqual(value, right[index])); + } + if (typeof left === "object" || typeof right === "object") { + if (left === null || right === null || typeof left !== "object" || typeof right !== "object") return false; + const leftRecord = left as Record; + const rightRecord = right as Record; + const leftKeys = Object.keys(leftRecord).sort(); + const rightKeys = Object.keys(rightRecord).sort(); + return valuesEqual(leftKeys, rightKeys) && leftKeys.every((key) => valuesEqual(leftRecord[key], rightRecord[key])); + } + return Object.is(left, right); +} + +function mismatch( + path: string, + left: unknown, + right: unknown, +): ComparisonReasonV1 | null { + if (valuesEqual(left, right)) return null; + return { + code: "provenance-mismatch", + path, + message: `baseline and candidate differ (${JSON.stringify(left)} vs ${JSON.stringify(right)})`, + }; +} + +/** + * Source revision/content and binary hash intentionally do not participate: + * they identify the two things being compared. Everything that defines the + * workload or execution environment must match exactly. + */ +export function provenanceMismatches( + baseline: ReceiptV1, + candidate: ReceiptV1, +): ComparisonReasonV1[] { + const pairs: readonly [string, unknown, unknown][] = [ + ["/provenance/scenario/id", baseline.provenance.scenario.id, candidate.provenance.scenario.id], + ["/provenance/scenario/suite", baseline.provenance.scenario.suite, candidate.provenance.scenario.suite], + ["/provenance/scenario/framework", baseline.provenance.scenario.framework, candidate.provenance.scenario.framework], + ["/provenance/scenario/manifestHash", baseline.provenance.scenario.manifestHash, candidate.provenance.scenario.manifestHash], + ["/provenance/scenario/inputTapeHash", baseline.provenance.scenario.inputTapeHash, candidate.provenance.scenario.inputTapeHash], + ["/provenance/toolchain", baseline.provenance.toolchain, candidate.provenance.toolchain], + ["/provenance/build/target", baseline.provenance.build.target, candidate.provenance.build.target], + ["/provenance/build/profile", baseline.provenance.build.profile, candidate.provenance.build.profile], + ["/provenance/build/rustFlags", baseline.provenance.build.rustFlags, candidate.provenance.build.rustFlags], + ["/provenance/build/cFlags", baseline.provenance.build.cFlags, candidate.provenance.build.cFlags], + ["/provenance/build/linkerFlags", baseline.provenance.build.linkerFlags, candidate.provenance.build.linkerFlags], + ["/provenance/executor/id", baseline.provenance.executor.id, candidate.provenance.executor.id], + ["/provenance/executor/version", baseline.provenance.executor.version, candidate.provenance.executor.version], + ["/provenance/executor/profile", baseline.provenance.executor.profile, candidate.provenance.executor.profile], + ["/provenance/executor/fingerprint", baseline.provenance.executor.fingerprint, candidate.provenance.executor.fingerprint], + ["/gateMetrics", baseline.gateMetrics, candidate.gateMetrics], + ]; + return pairs.flatMap(([path, left, right]) => { + const reason = mismatch(path, left, right); + return reason ? [reason] : []; + }); +} + +export function areProvenancesComparable( + baseline: ReceiptV1, + candidate: ReceiptV1, +): boolean { + return provenanceMismatches(baseline, candidate).length === 0; +} + +function receiptInvalidReasons(side: "baseline" | "candidate", receipt: ReceiptV1): ComparisonReasonV1[] { + if (receipt.status === "valid") return []; + return receipt.invalidReasons.map((reason, index) => ({ + code: "receipt-invalid", + path: `/${side}/invalidReasons/${index}`, + message: `${side} receipt is invalid: ${reason}`, + })); +} + +function correctnessMismatches( + baseline: ReceiptV1, + candidate: ReceiptV1, +): ComparisonReasonV1[] { + if (baseline.correctness === null || candidate.correctness === null) return []; + return (Object.keys(baseline.correctness) as (keyof typeof baseline.correctness)[]) + .flatMap((key) => baseline.correctness?.[key] === candidate.correctness?.[key] + ? [] + : [{ + code: "correctness-mismatch" as const, + path: `/correctness/${key}`, + message: `${key} differs between baseline and candidate`, + }]); +} + +function metricSupportMismatches( + baseline: ReceiptV1, + candidate: ReceiptV1, +): ComparisonReasonV1[] { + if (valuesEqual(baseline.unsupportedMetrics, candidate.unsupportedMetrics)) return []; + return [{ + code: "metric-support-mismatch", + path: "/unsupportedMetrics", + message: + `baseline and candidate declare different unsupported gates (` + + `${JSON.stringify(baseline.unsupportedMetrics)} vs ${JSON.stringify(candidate.unsupportedMetrics)})`, + }]; +} + +function mean(values: readonly number[]): number { + return values.reduce((total, value) => total + value, 0) / values.length; +} + +function hashSeed(value: string): number { + let result = BOOTSTRAP_SEED; + for (let index = 0; index < value.length; index += 1) { + result = Math.imul(result ^ value.charCodeAt(index), 16_777_619) >>> 0; + } + return result; +} + +function mulberry32(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let value = state; + value = Math.imul(value ^ value >>> 15, value | 1); + value ^= value + Math.imul(value ^ value >>> 7, value | 61); + return ((value ^ value >>> 14) >>> 0) / 4_294_967_296; + }; +} + +function pairedBootstrap( + metricId: string, + baseline: readonly number[], + candidate: readonly number[], +): NonNullable { + const deltas = candidate.map((value, index) => value - baseline[index]); + const random = mulberry32(hashSeed(metricId)); + const estimates = new Array(BOOTSTRAP_ITERATIONS); + for (let iteration = 0; iteration < BOOTSTRAP_ITERATIONS; iteration += 1) { + let total = 0; + for (let sample = 0; sample < deltas.length; sample += 1) { + total += deltas[Math.floor(random() * deltas.length)]!; + } + estimates[iteration] = total / deltas.length; + } + estimates.sort((left, right) => left - right); + const lowerIndex = Math.floor((BOOTSTRAP_ITERATIONS - 1) * 0.025); + const upperIndex = Math.ceil((BOOTSTRAP_ITERATIONS - 1) * 0.975); + return { + level: 0.95, + lower: estimates[lowerIndex]!, + upper: estimates[upperIndex]!, + method: "paired-bootstrap", + seed: hashSeed(metricId), + iterations: BOOTSTRAP_ITERATIONS, + }; +} + +function statusRank(status: ComparisonStatus): number { + return { pass: 0, warn: 1, regression: 2, invalid: 3 }[status]; +} + +function worstStatus(statuses: readonly ComparisonStatus[]): ComparisonStatus { + return statuses.reduce( + (worst, status) => statusRank(status) > statusRank(worst) ? status : worst, + "pass", + ); +} + +function thresholdExceeded( + worsening: number, + baseline: number, + threshold: RelativeAbsoluteThresholdV1, +): boolean { + if (worsening <= 0) return false; + const relativeWorsening = baseline === 0 ? Number.POSITIVE_INFINITY : worsening / Math.abs(baseline); + return worsening > threshold.absolute && relativeWorsening > threshold.relative; +} + +function thresholdReason( + metricId: string, + level: "warn" | "regression", + threshold: RelativeAbsoluteThresholdV1, +): ComparisonReasonV1 { + return { + code: "threshold-exceeded", + path: `/metrics/${metricId}`, + message: `${level} threshold exceeded (relative ${threshold.relative}, absolute ${threshold.absolute})`, + }; +} + +function unavailableMetric( + metricId: MetricId, + budget: MetricBudgetV1 | null, + reasons: readonly ComparisonReasonV1[], + baseline: number | null = null, + candidate: number | null = null, +): MetricComparisonV1 { + const definition = METRIC_CATALOG[metricId]; + return { + id: metricId, + label: definition.label, + direction: definition.direction, + kind: definition.kind, + unit: definition.unit, + status: "invalid", + baseline, + candidate, + delta: baseline === null || candidate === null ? null : candidate - baseline, + relativeDelta: baseline === null || candidate === null || baseline === 0 + ? null + : (candidate - baseline) / Math.abs(baseline), + sampleKind: "exact", + sampleCount: null, + confidenceInterval: null, + budget, + reasons, + }; +} + +function samplePoint(sample: MetricSampleV1): number { + return sample.kind === "exact" ? sample.value : mean(sample.samples); +} + +function compareMetric( + metricId: MetricId, + baselineSample: MetricSampleV1 | undefined, + candidateSample: MetricSampleV1 | undefined, + budget: MetricBudgetV1 | null, + requiredGate: boolean, +): MetricComparisonV1 { + const definition = METRIC_CATALOG[metricId]; + const metricPath = `/metrics/${metricId}`; + if (!baselineSample || !candidateSample) { + const missing = !baselineSample && !candidateSample + ? "baseline and candidate" + : !baselineSample ? "baseline" : "candidate"; + return unavailableMetric(metricId, budget, [{ + code: "metric-missing", + path: metricPath, + message: `${missing} receipt is missing the metric`, + }]); + } + const baseline = samplePoint(baselineSample); + const candidate = samplePoint(candidateSample); + if (requiredGate && budget === null) { + return unavailableMetric(metricId, budget, [{ + code: "budget-missing", + path: `/budgets/metrics/${metricId}`, + message: "declared gate metric has no budget applicable to this executor and scenario", + }], baseline, candidate); + } + if (baselineSample.unit !== candidateSample.unit) { + return unavailableMetric(metricId, budget, [{ + code: "unit-mismatch", + path: `${metricPath}/unit`, + message: `baseline uses ${baselineSample.unit}, candidate uses ${candidateSample.unit}`, + }], baseline, candidate); + } + if (baselineSample.unit !== definition.unit) { + return unavailableMetric(metricId, budget, [{ + code: "catalog-unit-mismatch", + path: `${metricPath}/unit`, + message: `receipt uses ${baselineSample.unit}, catalog requires ${definition.unit}`, + }], baseline, candidate); + } + if (baselineSample.kind !== candidateSample.kind) { + return unavailableMetric(metricId, budget, [{ + code: "sample-kind-mismatch", + path: `${metricPath}/kind`, + message: `baseline uses ${baselineSample.kind}, candidate uses ${candidateSample.kind}`, + }], baseline, candidate); + } + if ( + baselineSample.kind === "sampled" && + candidateSample.kind === "sampled" && + baselineSample.samples.length !== candidateSample.samples.length + ) { + return unavailableMetric(metricId, budget, [{ + code: "sample-count-mismatch", + path: `${metricPath}/samples`, + message: `paired samples require equal lengths (${baselineSample.samples.length} vs ${candidateSample.samples.length})`, + }], baseline, candidate); + } + + const delta = candidate - baseline; + const relativeDelta = baseline === 0 ? null : delta / Math.abs(baseline); + const worsening = definition.direction === "lower-is-better" ? delta : -delta; + const paired = baselineSample.kind === "sampled" && candidateSample.kind === "sampled"; + const confidenceInterval = paired + ? pairedBootstrap(metricId, baselineSample.samples, candidateSample.samples) + : null; + const confidentWorsening = confidenceInterval === null + ? worsening + : definition.direction === "lower-is-better" + ? confidenceInterval.lower + : -confidenceInterval.upper; + const reasons: ComparisonReasonV1[] = []; + let status: ComparisonStatus = "pass"; + + if (budget) { + if (budget.hardMax !== undefined && candidate > budget.hardMax) { + status = "regression"; + reasons.push({ + code: "hard-limit-exceeded", + path: `${metricPath}/hardMax`, + message: `candidate ${candidate} exceeds hardMax ${budget.hardMax}`, + }); + } + if (budget.hardMin !== undefined && candidate < budget.hardMin) { + status = "regression"; + reasons.push({ + code: "hard-limit-exceeded", + path: `${metricPath}/hardMin`, + message: `candidate ${candidate} is below hardMin ${budget.hardMin}`, + }); + } + if (budget.regression && thresholdExceeded(worsening, baseline, budget.regression)) { + if (!paired || thresholdExceeded(confidentWorsening, baseline, budget.regression)) { + status = "regression"; + reasons.push(thresholdReason(metricId, "regression", budget.regression)); + } else if (status === "pass") { + status = "warn"; + reasons.push({ + code: "threshold-exceeded", + path: metricPath, + message: "regression point estimate exceeded the threshold, but the paired 95% bootstrap interval is not conclusive", + }); + } + } else if (status === "pass" && budget.warn && thresholdExceeded(worsening, baseline, budget.warn)) { + status = "warn"; + reasons.push(thresholdReason(metricId, "warn", budget.warn)); + } + } + + return { + id: metricId, + label: definition.label, + direction: definition.direction, + kind: definition.kind, + unit: definition.unit, + status, + baseline, + candidate, + delta, + relativeDelta, + sampleKind: paired ? "paired" : "exact", + sampleCount: paired ? baselineSample.samples.length : null, + confidenceInterval, + budget, + reasons, + }; +} + +function metricBudgetForExecutor( + budgetSet: BudgetSetV1, + metricId: MetricId, + executorId: string, + scenarioId: string, +): MetricBudgetV1 | null { + const baseScenarioId = scenarioId.split("#", 1)[0]!; + const layers = [ + budgetSet.metrics[metricId], + budgetSet.scenarios?.[baseScenarioId]?.[metricId], + baseScenarioId === scenarioId ? undefined : budgetSet.scenarios?.[scenarioId]?.[metricId], + ].filter((budget): budget is MetricBudgetV1 => + budget !== undefined && (!budget.executors || budget.executors.includes(executorId))); + if (layers.length === 0) return null; + return Object.assign({}, ...layers) as MetricBudgetV1; +} + +function hasConfiguredMetricBudget( + budgetSet: BudgetSetV1, + metricId: MetricId, + scenarioId: string, +): boolean { + const baseScenarioId = scenarioId.split("#", 1)[0]!; + return budgetSet.metrics[metricId] !== undefined || + budgetSet.scenarios?.[baseScenarioId]?.[metricId] !== undefined || + (baseScenarioId !== scenarioId && budgetSet.scenarios?.[scenarioId]?.[metricId] !== undefined); +} + +function comparisonReference(receipt: ReceiptV1): ComparisonV1["base"] { + return { + sourceRevision: receipt.provenance.source.revision, + binarySha256: receipt.provenance.binary.sha256, + }; +} + +export function compareReceipts( + baselineInput: ReceiptV1, + candidateInput: ReceiptV1, + budgetInput: BudgetSetV1 = DEFAULT_BUDGET_SET, +): ComparisonV1 { + const baseline = parseReceiptV1(baselineInput); + const candidate = parseReceiptV1(candidateInput); + const budgetSet = parseBudgetSetV1(budgetInput); + const globalReasons = [ + ...receiptInvalidReasons("baseline", baseline), + ...receiptInvalidReasons("candidate", candidate), + ...provenanceMismatches(baseline, candidate), + ...correctnessMismatches(baseline, candidate), + ...metricSupportMismatches(baseline, candidate), + ]; + const base = comparisonReference(baseline); + const candidateReference = comparisonReference(candidate); + + if (globalReasons.length > 0) { + return { + schemaVersion: 1, + kind: "pocketjs.perf.comparison", + status: "invalid", + comparable: false, + budgetId: budgetSet.id, + base, + candidate: candidateReference, + unsupportedMetrics: [], + reasons: globalReasons, + metrics: [], + }; + } + + const executorId = baseline.provenance.executor.id; + const scenarioId = baseline.provenance.scenario.id; + const gateMetrics = baseline.gateMetrics.filter(isMetricId); + const gateSet = new Set(gateMetrics); + const declaredUnsupported = new Set( + baseline.unsupportedMetrics.filter(isMetricId), + ); + const unsupportedMetrics: MetricId[] = []; + const metricIds = METRIC_IDS.filter((metricId) => { + const present = baseline.metrics[metricId] !== undefined || candidate.metrics[metricId] !== undefined; + return present || gateSet.has(metricId); + }); + const metrics = metricIds.flatMap((metricId): MetricComparisonV1[] => { + const requiredGate = gateSet.has(metricId); + // Budgets govern any observation a subject actually emits. gateMetrics is + // the stricter minimum-observation contract, not an allow-list that could + // silently disable a configured budget such as instruction bytes. + const budget = metricBudgetForExecutor(budgetSet, metricId, executorId, scenarioId); + if (declaredUnsupported.has(metricId)) { + if (budget === null) { + if (!hasConfiguredMetricBudget(budgetSet, metricId, scenarioId)) { + return [unavailableMetric(metricId, null, [{ + code: "budget-missing", + path: `/budgets/metrics/${metricId}`, + message: "declared gate metric has no configured budget", + }])]; + } + unsupportedMetrics.push(metricId); + return []; + } + return [unavailableMetric(metricId, budget, [{ + code: "metric-support-mismatch", + path: `/unsupportedMetrics/${metricId}`, + message: "executor declares this gate unsupported, but an applicable budget requires it", + }])]; + } + return [compareMetric( + metricId, + baseline.metrics[metricId], + candidate.metrics[metricId], + budget, + requiredGate, + )]; + }); + const metricReasons = metrics.flatMap((metric) => metric.reasons); + const status = worstStatus(metrics.map((metric) => metric.status)); + const comparable = status !== "invalid"; + return { + schemaVersion: 1, + kind: "pocketjs.perf.comparison", + status, + comparable, + budgetId: budgetSet.id, + base, + candidate: candidateReference, + unsupportedMetrics, + reasons: metricReasons, + metrics, + }; +} + +function formatValue(value: number | null, unit: string): string { + if (value === null) return "—"; + if (unit === "bytes") return `${new Intl.NumberFormat("en-US", { maximumFractionDigits: 2 }).format(value)} B`; + if (unit === "ns") return `${new Intl.NumberFormat("en-US", { maximumFractionDigits: 2 }).format(value)} ns`; + return new Intl.NumberFormat("en-US", { maximumFractionDigits: 2 }).format(value); +} + +function formatDelta(metric: MetricComparisonV1): string { + if (metric.delta === null) return "—"; + const sign = metric.delta > 0 ? "+" : ""; + const relative = metric.relativeDelta === null + ? "n/a at zero baseline" + : `${metric.relativeDelta > 0 ? "+" : ""}${(metric.relativeDelta * 100).toFixed(2)}%`; + return `${sign}${formatValue(metric.delta, metric.unit)} (${relative})`; +} + +function escapeCell(value: string): string { + return value.replaceAll("|", "\\|").replaceAll("\n", " "); +} + +export function comparisonToJson(comparison: ComparisonV1, pretty = true): string { + return `${JSON.stringify(parseComparisonV1(comparison), null, pretty ? 2 : undefined)}\n`; +} + +export function comparisonToMarkdown(comparisonInput: ComparisonV1): string { + const comparison = parseComparisonV1(comparisonInput); + const lines = [ + "# PocketJS performance comparison", + "", + `Status: **${comparison.status}**`, + "", + `Baseline: \`${comparison.base.sourceRevision}\` (\`${comparison.base.binarySha256.slice(0, 12)}\`)`, + `Candidate: \`${comparison.candidate.sourceRevision}\` (\`${comparison.candidate.binarySha256.slice(0, 12)}\`)`, + `Budget: \`${comparison.budgetId}\``, + ]; + if (comparison.unsupportedMetrics.length > 0) { + lines.push( + "", + `Unsupported gates for this executor: ${comparison.unsupportedMetrics.map((metric) => `\`${metric}\``).join(", ")}`, + ); + } + if (comparison.metrics.length > 0) { + lines.push( + "", + "| Metric | Baseline | Candidate | Delta | Status |", + "| --- | ---: | ---: | ---: | --- |", + ...comparison.metrics.map((metric) => + `| ${escapeCell(metric.label)} | ${formatValue(metric.baseline, metric.unit)} | ${formatValue(metric.candidate, metric.unit)} | ${formatDelta(metric)} | **${metric.status}** |`), + ); + } + if (comparison.reasons.length > 0) { + lines.push( + "", + "## Reasons", + "", + ...comparison.reasons.map((reason) => `- \`${reason.code}\` at \`${reason.path}\`: ${reason.message}`), + ); + } + return `${lines.join("\n")}\n`; +} + +export const renderComparisonJson = comparisonToJson; +export const renderComparisonMarkdown = comparisonToMarkdown; diff --git a/tools/perf/core/index.ts b/tools/perf/core/index.ts new file mode 100644 index 00000000..87b827a0 --- /dev/null +++ b/tools/perf/core/index.ts @@ -0,0 +1,6 @@ +export * from "./types.ts"; +export * from "./catalog.ts"; +export * from "./budgets.ts"; +export * from "./render-config.ts"; +export * from "./schema.ts"; +export * from "./compare.ts"; diff --git a/tools/perf/core/render-config.ts b/tools/perf/core/render-config.ts new file mode 100644 index 00000000..f778684d --- /dev/null +++ b/tools/perf/core/render-config.ts @@ -0,0 +1,124 @@ +import type { ScenarioV1 } from "./types.ts"; + +export const DEFAULT_BUILD_RENDER_CONFIG = Object.freeze({ + width: 480, + height: 272, + rasterDensity: 1, + renderScale: 1, +}); + +export const BUILD_RENDER_LIMITS = Object.freeze({ + width: 32_000, + height: 32_000, + rasterDensity: 255, + renderScale: 4, +}); + +export interface BuildRenderConfig { + readonly width: number; + readonly height: number; + readonly rasterDensity: number; + readonly renderScale: number; +} + +export interface BuildRenderConfigIssue { + /** Path relative to the scenario params object. */ + readonly path: readonly string[]; + readonly message: string; +} + +export class BuildRenderConfigError extends TypeError { + readonly issues: readonly BuildRenderConfigIssue[]; + + constructor(issues: readonly BuildRenderConfigIssue[]) { + super(issues.map(({ path, message }) => ( + `${path.length > 0 ? path.join(".") : "params"}: ${message}` + )).join("; ")); + this.name = "BuildRenderConfigError"; + this.issues = issues; + } +} + +function isPlainRecord(value: unknown): value is Readonly> { + return typeof value === "object" && + value !== null && + !Array.isArray(value) && + [Object.prototype, null].includes(Object.getPrototypeOf(value)); +} + +/** + * Resolve the build/raster contract shared by Native and QEMU perf adapters. + * + * The bounds intentionally match the host ABI: logical dimensions are + * limited by `createWasmUi`, density by the asset builder, and render scale by + * the core software rasterizer. + */ +export function buildRenderConfig(params: unknown): BuildRenderConfig { + const issues: BuildRenderConfigIssue[] = []; + if (!isPlainRecord(params)) { + throw new BuildRenderConfigError([{ path: [], message: "expected an object" }]); + } + + if (!Object.hasOwn(params, "viewport")) return { ...DEFAULT_BUILD_RENDER_CONFIG }; + const viewport = params.viewport; + if (!isPlainRecord(viewport)) { + throw new BuildRenderConfigError([{ + path: ["viewport"], + message: "expected an object", + }]); + } + + const allowed = new Set([ + "width", + "height", + "rasterDensity", + "renderScale", + ]); + for (const key of Object.keys(viewport)) { + if (!allowed.has(key as keyof BuildRenderConfig)) { + issues.push({ path: ["viewport", key], message: "unexpected property" }); + } + } + + const integer = (key: K): number => { + const fallback = DEFAULT_BUILD_RENDER_CONFIG[key]; + if (!Object.hasOwn(viewport, key)) return fallback; + const value = viewport[key]; + const maximum = BUILD_RENDER_LIMITS[key]; + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 1 || value > maximum) { + issues.push({ + path: ["viewport", key], + message: `expected an integer from 1 through ${maximum}`, + }); + return fallback; + } + return value; + }; + + const config: BuildRenderConfig = { + width: integer("width"), + height: integer("height"), + rasterDensity: integer("rasterDensity"), + renderScale: integer("renderScale"), + }; + if (issues.length > 0) throw new BuildRenderConfigError(issues); + return config; +} + +/** Build output varies with the resolved entry, framework and asset density. */ +export function artifactBuildVariantKey( + scenario: Pick, +): string { + const { rasterDensity } = buildRenderConfig(scenario.params); + return `${scenario.subject.id}\0${scenario.subject.entry}\0${scenario.subject.framework}\0density=${rasterDensity}`; +} + +export function rgbaFramebufferByteLength(config: BuildRenderConfig): number { + const width = config.width * config.renderScale; + const height = config.height * config.renderScale; + const bytes = width * height * 4; + if (!Number.isSafeInteger(bytes) || bytes <= 0) { + throw new RangeError("scaled framebuffer dimensions overflow"); + } + return bytes; +} diff --git a/tools/perf/core/schema.ts b/tools/perf/core/schema.ts new file mode 100644 index 00000000..69435aca --- /dev/null +++ b/tools/perf/core/schema.ts @@ -0,0 +1,959 @@ +import { isMetricId, METRIC_CATALOG } from "./catalog.ts"; +import { buildRenderConfig, BuildRenderConfigError } from "./render-config.ts"; +import type { + BudgetSetV1, + ComparisonReasonV1, + ComparisonV1, + CorrectnessCapture, + CorrectnessReceiptV1, + FrameworkId, + InputTapeV1, + InputTrackV1, + JsonValue, + MetricBudgetV1, + MetricComparisonV1, + MetricSampleV1, + ReceiptProvenanceV1, + ReceiptV1, + SafeParseResult, + ScenarioV1, + SchemaIssue, +} from "./types.ts"; +import { SchemaValidationError } from "./types.ts"; + +type UnknownRecord = Record; + +const FRAMEWORKS = new Set([ + "solid", + "vue-vapor", + "octane", + "core", +]); +const CAPTURES = new Set([ + "framebuffer", + "drawList", + "state", + "effects", +]); +const COMPARISON_STATUSES = new Set([ + "pass", + "warn", + "regression", + "invalid", +]); +const REASON_CODES = new Set([ + "receipt-invalid", + "provenance-mismatch", + "correctness-mismatch", + "metric-missing", + "budget-missing", + "metric-support-mismatch", + "unit-mismatch", + "catalog-unit-mismatch", + "sample-kind-mismatch", + "sample-count-mismatch", + "threshold-exceeded", + "hard-limit-exceeded", +]); +const SHA256 = /^[a-f0-9]{64}$/; + +function pointer(parent: string, key: string | number): string { + const encoded = String(key).replaceAll("~", "~0").replaceAll("/", "~1"); + return parent === "/" ? `/${encoded}` : `${parent}/${encoded}`; +} + +function issue(issues: SchemaIssue[], path: string, message: string): void { + issues.push({ path, message }); +} + +function record( + value: unknown, + path: string, + allowedKeys: readonly string[] | null, + requiredKeys: readonly string[], + issues: SchemaIssue[], +): UnknownRecord | null { + if ( + typeof value !== "object" || + value === null || + Array.isArray(value) || + ![Object.prototype, null].includes(Object.getPrototypeOf(value)) + ) { + issue(issues, path, "expected an object"); + return null; + } + const result = value as UnknownRecord; + if (allowedKeys) { + const allowed = new Set(allowedKeys); + for (const key of Object.keys(result)) { + if (!allowed.has(key)) issue(issues, pointer(path, key), "unexpected property"); + } + } + for (const key of requiredKeys) { + if (!Object.hasOwn(result, key)) issue(issues, pointer(path, key), "required property is missing"); + } + return result; +} + +function array(value: unknown, path: string, issues: SchemaIssue[]): unknown[] | null { + if (!Array.isArray(value)) { + issue(issues, path, "expected an array"); + return null; + } + return value; +} + +function nonEmptyString(value: unknown, path: string, issues: SchemaIssue[]): value is string { + if (typeof value !== "string" || value.trim().length === 0) { + issue(issues, path, "expected a non-empty string"); + return false; + } + return true; +} + +function literal(value: unknown, expected: string | number, path: string, issues: SchemaIssue[]): boolean { + if (value !== expected) { + issue(issues, path, `expected ${JSON.stringify(expected)}`); + return false; + } + return true; +} + +function boolean(value: unknown, path: string, issues: SchemaIssue[]): value is boolean { + if (typeof value !== "boolean") { + issue(issues, path, "expected a boolean"); + return false; + } + return true; +} + +function finiteNumber(value: unknown, path: string, issues: SchemaIssue[]): value is number { + if (typeof value !== "number" || !Number.isFinite(value)) { + issue(issues, path, "expected a finite number"); + return false; + } + return true; +} + +function nonNegativeNumber(value: unknown, path: string, issues: SchemaIssue[]): value is number { + if (!finiteNumber(value, path, issues)) return false; + if (value < 0) { + issue(issues, path, "expected a non-negative number"); + return false; + } + return true; +} + +function nonNegativeInteger(value: unknown, path: string, issues: SchemaIssue[]): value is number { + if (!finiteNumber(value, path, issues)) return false; + if (!Number.isSafeInteger(value) || value < 0) { + issue(issues, path, "expected a non-negative safe integer"); + return false; + } + return true; +} + +function positiveInteger(value: unknown, path: string, issues: SchemaIssue[]): value is number { + if (!nonNegativeInteger(value, path, issues)) return false; + if (value === 0) { + issue(issues, path, "expected a positive integer"); + return false; + } + return true; +} + +function stringArray( + value: unknown, + path: string, + issues: SchemaIssue[], + options: { nonEmpty?: boolean; unique?: boolean } = {}, +): value is string[] { + const values = array(value, path, issues); + if (!values) return false; + if (options.nonEmpty && values.length === 0) issue(issues, path, "expected at least one item"); + const seen = new Set(); + for (let index = 0; index < values.length; index += 1) { + const itemPath = pointer(path, index); + if (!nonEmptyString(values[index], itemPath, issues)) continue; + const item = values[index] as string; + if (options.unique && seen.has(item)) issue(issues, itemPath, "duplicate value"); + seen.add(item); + } + return true; +} + +function enumString( + value: unknown, + values: ReadonlySet, + path: string, + issues: SchemaIssue[], +): value is T { + if (typeof value !== "string" || !values.has(value as T)) { + issue(issues, path, `expected one of ${[...values].join(", ")}`); + return false; + } + return true; +} + +function sha256(value: unknown, path: string, issues: SchemaIssue[]): value is string { + if (typeof value !== "string" || !SHA256.test(value)) { + issue(issues, path, "expected a lowercase SHA-256 hex digest"); + return false; + } + return true; +} + +function jsonValue( + value: unknown, + path: string, + issues: SchemaIssue[], + ancestors: Set = new Set(), +): value is JsonValue { + if (value === null || typeof value === "string" || typeof value === "boolean") return true; + if (typeof value === "number") return finiteNumber(value, path, issues); + if (typeof value !== "object" || value === null) { + issue(issues, path, "expected a JSON value"); + return false; + } + if (ancestors.has(value)) { + issue(issues, path, "cyclic values are not valid JSON"); + return false; + } + ancestors.add(value); + if (Array.isArray(value)) { + value.forEach((item, index) => jsonValue(item, pointer(path, index), issues, ancestors)); + ancestors.delete(value); + return true; + } + if (![Object.prototype, null].includes(Object.getPrototypeOf(value))) { + issue(issues, path, "expected a plain JSON object"); + ancestors.delete(value); + return false; + } + for (const [key, item] of Object.entries(value)) { + jsonValue(item, pointer(path, key), issues, ancestors); + } + ancestors.delete(value); + return true; +} + +function validateFrame( + value: unknown, + frames: number | null, + path: string, + issues: SchemaIssue[], +): value is number { + if (!nonNegativeInteger(value, path, issues)) return false; + if (frames !== null && value >= frames) { + issue(issues, path, `must be less than tape frame count ${frames}`); + return false; + } + return true; +} + +function validateSamples( + value: unknown, + path: string, + frames: number | null, + allowedKeys: readonly string[], + requiredKeys: readonly string[], + validateSample: (sample: UnknownRecord, samplePath: string) => void, + issues: SchemaIssue[], +): void { + const samples = array(value, path, issues); + if (!samples) return; + let previousFrame = -1; + samples.forEach((sampleValue, index) => { + const samplePath = pointer(path, index); + const sample = record(sampleValue, samplePath, allowedKeys, requiredKeys, issues); + if (!sample) return; + if (validateFrame(sample.frame, frames, pointer(samplePath, "frame"), issues)) { + if (sample.frame <= previousFrame) { + issue(issues, pointer(samplePath, "frame"), "sample frames must be strictly increasing"); + } + previousFrame = sample.frame; + } + validateSample(sample, samplePath); + }); +} + +function validateInputTrack( + value: unknown, + path: string, + frames: number | null, + issues: SchemaIssue[], +): InputTrackV1 | null { + const base = record(value, path, null, ["kind"], issues); + if (!base) return null; + if (typeof base.kind !== "string") { + issue(issues, pointer(path, "kind"), "expected an input track kind"); + return null; + } + if (base.kind === "button") { + const track = record(value, path, ["kind", "control", "samples"], ["kind", "control", "samples"], issues); + if (!track) return null; + nonEmptyString(track.control, pointer(path, "control"), issues); + validateSamples(track.samples, pointer(path, "samples"), frames, ["frame", "pressed"], ["frame", "pressed"], (sample, samplePath) => { + boolean(sample.pressed, pointer(samplePath, "pressed"), issues); + }, issues); + return value as InputTrackV1; + } + if (base.kind === "analog") { + const track = record(value, path, ["kind", "control", "samples"], ["kind", "control", "samples"], issues); + if (!track) return null; + nonEmptyString(track.control, pointer(path, "control"), issues); + validateSamples(track.samples, pointer(path, "samples"), frames, ["frame", "value"], ["frame", "value"], (sample, samplePath) => { + if (finiteNumber(sample.value, pointer(samplePath, "value"), issues) && (sample.value < -1 || sample.value > 1)) { + issue(issues, pointer(samplePath, "value"), "analog levels must be between -1 and 1"); + } + }, issues); + return value as InputTrackV1; + } + if (base.kind === "touch") { + const track = record(value, path, ["kind", "control", "samples"], ["kind", "control", "samples"], issues); + if (!track) return null; + nonEmptyString(track.control, pointer(path, "control"), issues); + validateSamples(track.samples, pointer(path, "samples"), frames, ["frame", "phase", "x", "y"], ["frame", "phase", "x", "y"], (sample, samplePath) => { + enumString(sample.phase, new Set(["start", "move", "end", "cancel"] as const), pointer(samplePath, "phase"), issues); + finiteNumber(sample.x, pointer(samplePath, "x"), issues); + finiteNumber(sample.y, pointer(samplePath, "y"), issues); + }, issues); + return value as InputTrackV1; + } + if (base.kind === "relative-axis") { + const track = record(value, path, ["kind", "control", "samples"], ["kind", "control", "samples"], issues); + if (!track) return null; + nonEmptyString(track.control, pointer(path, "control"), issues); + validateSamples(track.samples, pointer(path, "samples"), frames, ["frame", "delta"], ["frame", "delta"], (sample, samplePath) => { + finiteNumber(sample.delta, pointer(samplePath, "delta"), issues); + }, issues); + return value as InputTrackV1; + } + if (base.kind === "effect") { + const track = record(value, path, ["kind", "effect", "samples"], ["kind", "effect", "samples"], issues); + if (!track) return null; + nonEmptyString(track.effect, pointer(path, "effect"), issues); + validateSamples(track.samples, pointer(path, "samples"), frames, ["frame", "value"], ["frame", "value"], (sample, samplePath) => { + jsonValue(sample.value, pointer(samplePath, "value"), issues); + }, issues); + return value as InputTrackV1; + } + issue(issues, pointer(path, "kind"), "unknown input track kind"); + return null; +} + +function validateInputTape(value: unknown, path: string, issues: SchemaIssue[]): InputTapeV1 | null { + const tape = record( + value, + path, + ["schemaVersion", "kind", "id", "frames", "tracks"], + ["schemaVersion", "kind", "id", "frames", "tracks"], + issues, + ); + if (!tape) return null; + literal(tape.schemaVersion, 1, pointer(path, "schemaVersion"), issues); + literal(tape.kind, "pocketjs.perf.input-tape", pointer(path, "kind"), issues); + nonEmptyString(tape.id, pointer(path, "id"), issues); + const frames = positiveInteger(tape.frames, pointer(path, "frames"), issues) + ? tape.frames as number + : null; + const tracks = array(tape.tracks, pointer(path, "tracks"), issues); + if (tracks) { + const identities = new Set(); + tracks.forEach((trackValue, index) => { + const trackPath = pointer(pointer(path, "tracks"), index); + const track = validateInputTrack(trackValue, trackPath, frames, issues); + if (!track) return; + const identity = track.kind === "effect" + ? `${track.kind}:${track.effect}` + : `${track.kind}:${track.control}`; + if (identities.has(identity)) issue(issues, trackPath, `duplicate track ${identity}`); + identities.add(identity); + }); + } + return value as InputTapeV1; +} + +function validateScenario(value: unknown, path: string, issues: SchemaIssue[]): ScenarioV1 | null { + const scenario = record( + value, + path, + ["schemaVersion", "kind", "id", "suite", "subject", "executorRequirements", "frames", "tape", "phases", "checkpoints", "params"], + ["schemaVersion", "kind", "id", "suite", "subject", "executorRequirements", "frames", "tape", "phases", "checkpoints", "params"], + issues, + ); + if (!scenario) return null; + literal(scenario.schemaVersion, 1, pointer(path, "schemaVersion"), issues); + literal(scenario.kind, "pocketjs.perf.scenario", pointer(path, "kind"), issues); + nonEmptyString(scenario.id, pointer(path, "id"), issues); + nonEmptyString(scenario.suite, pointer(path, "suite"), issues); + const subjectPath = pointer(path, "subject"); + const subject = record(scenario.subject, subjectPath, ["id", "family", "framework", "entry"], ["id", "family", "framework", "entry"], issues); + if (subject) { + nonEmptyString(subject.id, pointer(subjectPath, "id"), issues); + nonEmptyString(subject.family, pointer(subjectPath, "family"), issues); + enumString(subject.framework, FRAMEWORKS, pointer(subjectPath, "framework"), issues); + nonEmptyString(subject.entry, pointer(subjectPath, "entry"), issues); + } + stringArray(scenario.executorRequirements, pointer(path, "executorRequirements"), issues, { nonEmpty: true, unique: true }); + const frames = positiveInteger(scenario.frames, pointer(path, "frames"), issues) + ? scenario.frames as number + : null; + const tape = validateInputTape(scenario.tape, pointer(path, "tape"), issues); + if (frames !== null && tape && frames !== tape.frames) { + issue(issues, pointer(path, "tape/frames"), "must equal the scenario frame count"); + } + const phases = array(scenario.phases, pointer(path, "phases"), issues); + if (phases) { + if (phases.length === 0) issue(issues, pointer(path, "phases"), "expected at least one phase"); + const names = new Set(); + let previousEnd = 0; + phases.forEach((phaseValue, index) => { + const phasePath = pointer(pointer(path, "phases"), index); + const phase = record(phaseValue, phasePath, ["name", "startFrame", "endFrame", "collect"], ["name", "startFrame", "endFrame", "collect"], issues); + if (!phase) return; + if (nonEmptyString(phase.name, pointer(phasePath, "name"), issues)) { + if (names.has(phase.name)) issue(issues, pointer(phasePath, "name"), "duplicate phase name"); + names.add(phase.name); + } + const startOk = nonNegativeInteger(phase.startFrame, pointer(phasePath, "startFrame"), issues); + const endOk = positiveInteger(phase.endFrame, pointer(phasePath, "endFrame"), issues); + if (startOk && endOk) { + const start = phase.startFrame as number; + const end = phase.endFrame as number; + if (start >= end) issue(issues, pointer(phasePath, "endFrame"), "must be greater than startFrame"); + if (frames !== null && end > frames) issue(issues, pointer(phasePath, "endFrame"), `must not exceed scenario frame count ${frames}`); + if (index > 0 && start < previousEnd) issue(issues, pointer(phasePath, "startFrame"), "phases must be ordered and non-overlapping"); + previousEnd = end; + } + boolean(phase.collect, pointer(phasePath, "collect"), issues); + }); + } + const checkpoints = array(scenario.checkpoints, pointer(path, "checkpoints"), issues); + if (checkpoints) { + let previousFrame = -1; + checkpoints.forEach((checkpointValue, index) => { + const checkpointPath = pointer(pointer(path, "checkpoints"), index); + const checkpoint = record(checkpointValue, checkpointPath, ["frame", "capture"], ["frame", "capture"], issues); + if (!checkpoint) return; + if (validateFrame(checkpoint.frame, frames, pointer(checkpointPath, "frame"), issues)) { + if (checkpoint.frame <= previousFrame) issue(issues, pointer(checkpointPath, "frame"), "checkpoint frames must be strictly increasing"); + previousFrame = checkpoint.frame; + } + const captures = array(checkpoint.capture, pointer(checkpointPath, "capture"), issues); + if (captures) { + if (captures.length === 0) issue(issues, pointer(checkpointPath, "capture"), "expected at least one capture"); + const seen = new Set(); + captures.forEach((capture, captureIndex) => { + const capturePath = pointer(pointer(checkpointPath, "capture"), captureIndex); + if (enumString(capture, CAPTURES, capturePath, issues)) { + if (seen.has(capture)) issue(issues, capturePath, "duplicate capture"); + seen.add(capture); + } + }); + } + }); + } + const params = record(scenario.params, pointer(path, "params"), null, [], issues); + if (params) { + for (const [key, param] of Object.entries(params)) { + jsonValue(param, pointer(pointer(path, "params"), key), issues); + } + if (Object.hasOwn(params, "gateMetrics")) { + const gatePath = pointer(pointer(path, "params"), "gateMetrics"); + const gates = array(params.gateMetrics, gatePath, issues); + if (gates) { + if (gates.length === 0) issue(issues, gatePath, "expected at least one gate metric"); + const seen = new Set(); + gates.forEach((metric, index) => { + const metricPath = pointer(gatePath, index); + if (!nonEmptyString(metric, metricPath, issues)) return; + if (!isMetricId(metric)) { + issue(issues, metricPath, "unknown metric id"); + return; + } + if (seen.has(metric)) issue(issues, metricPath, "duplicate gate metric"); + seen.add(metric); + if (METRIC_CATALOG[metric].diagnostic) { + issue(issues, metricPath, "diagnostic metrics cannot be regression gates"); + } + }); + } + } + try { + buildRenderConfig(params); + } catch (error) { + if (!(error instanceof BuildRenderConfigError)) throw error; + for (const renderIssue of error.issues) { + const renderPath = renderIssue.path.reduce( + (current, key) => pointer(current, key), + pointer(path, "params"), + ); + issue(issues, renderPath, renderIssue.message); + } + } + } + return value as ScenarioV1; +} + +function validateStringRecord( + value: unknown, + path: string, + allowed: readonly string[], + required: readonly string[], + issues: SchemaIssue[], +): UnknownRecord | null { + const result = record(value, path, allowed, required, issues); + if (!result) return null; + for (const key of Object.keys(result)) nonEmptyString(result[key], pointer(path, key), issues); + return result; +} + +function validateProvenance(value: unknown, path: string, issues: SchemaIssue[]): ReceiptProvenanceV1 | null { + const provenance = record(value, path, ["source", "scenario", "toolchain", "build", "executor", "binary"], ["source", "scenario", "toolchain", "build", "executor", "binary"], issues); + if (!provenance) return null; + const sourcePath = pointer(path, "source"); + const source = record(provenance.source, sourcePath, ["revision", "dirty", "contentHash"], ["revision", "dirty", "contentHash"], issues); + if (source) { + nonEmptyString(source.revision, pointer(sourcePath, "revision"), issues); + boolean(source.dirty, pointer(sourcePath, "dirty"), issues); + sha256(source.contentHash, pointer(sourcePath, "contentHash"), issues); + } + const scenarioPath = pointer(path, "scenario"); + const scenario = record(provenance.scenario, scenarioPath, ["id", "suite", "framework", "manifestHash", "inputTapeHash"], ["id", "suite", "framework", "manifestHash", "inputTapeHash"], issues); + if (scenario) { + nonEmptyString(scenario.id, pointer(scenarioPath, "id"), issues); + nonEmptyString(scenario.suite, pointer(scenarioPath, "suite"), issues); + enumString(scenario.framework, FRAMEWORKS, pointer(scenarioPath, "framework"), issues); + sha256(scenario.manifestHash, pointer(scenarioPath, "manifestHash"), issues); + sha256(scenario.inputTapeHash, pointer(scenarioPath, "inputTapeHash"), issues); + } + const toolchainPath = pointer(path, "toolchain"); + validateStringRecord( + provenance.toolchain, + toolchainPath, + ["rustc", "cCompiler", "sysroot", "qemu", "bun"], + ["rustc", "cCompiler", "sysroot"], + issues, + ); + const buildPath = pointer(path, "build"); + const build = record(provenance.build, buildPath, ["target", "profile", "rustFlags", "cFlags", "linkerFlags"], ["target", "profile", "rustFlags", "cFlags", "linkerFlags"], issues); + if (build) { + nonEmptyString(build.target, pointer(buildPath, "target"), issues); + nonEmptyString(build.profile, pointer(buildPath, "profile"), issues); + stringArray(build.rustFlags, pointer(buildPath, "rustFlags"), issues); + stringArray(build.cFlags, pointer(buildPath, "cFlags"), issues); + stringArray(build.linkerFlags, pointer(buildPath, "linkerFlags"), issues); + } + const executorPath = pointer(path, "executor"); + validateStringRecord(provenance.executor, executorPath, ["id", "version", "profile", "fingerprint"], ["id", "version", "profile", "fingerprint"], issues); + if (typeof provenance.executor === "object" && provenance.executor !== null) { + sha256((provenance.executor as UnknownRecord).fingerprint, pointer(executorPath, "fingerprint"), issues); + } + const binaryPath = pointer(path, "binary"); + const binary = record(provenance.binary, binaryPath, ["sha256"], ["sha256"], issues); + if (binary) sha256(binary.sha256, pointer(binaryPath, "sha256"), issues); + return value as ReceiptProvenanceV1; +} + +function validateCorrectness(value: unknown, path: string, issues: SchemaIssue[]): CorrectnessReceiptV1 | null { + const correctness = record(value, path, ["framebufferHash", "drawListHash", "stateHash", "effectHash"], ["framebufferHash", "drawListHash", "stateHash", "effectHash"], issues); + if (!correctness) return null; + for (const key of ["framebufferHash", "drawListHash", "stateHash", "effectHash"] as const) { + sha256(correctness[key], pointer(path, key), issues); + } + return value as CorrectnessReceiptV1; +} + +function validateMetrics(value: unknown, path: string, issues: SchemaIssue[]): Record | null { + const metrics = record(value, path, null, [], issues); + if (!metrics) return null; + for (const [metricId, sampleValue] of Object.entries(metrics)) { + const samplePath = pointer(path, metricId); + if (!isMetricId(metricId)) issue(issues, samplePath, "unknown metric id"); + const base = record(sampleValue, samplePath, null, ["kind", "unit"], issues); + if (!base) continue; + enumString(base.unit, new Set(["count", "bytes", "ns"] as const), pointer(samplePath, "unit"), issues); + if (base.kind === "exact") { + const sample = record(sampleValue, samplePath, ["kind", "value", "unit"], ["kind", "value", "unit"], issues); + if (sample) nonNegativeInteger(sample.value, pointer(samplePath, "value"), issues); + } else if (base.kind === "sampled") { + const sample = record(sampleValue, samplePath, ["kind", "samples", "unit"], ["kind", "samples", "unit"], issues); + if (sample) { + const samples = array(sample.samples, pointer(samplePath, "samples"), issues); + if (samples) { + if (samples.length === 0) issue(issues, pointer(samplePath, "samples"), "expected at least one observation"); + samples.forEach((observation, index) => nonNegativeInteger(observation, pointer(pointer(samplePath, "samples"), index), issues)); + } + } + } else { + issue(issues, pointer(samplePath, "kind"), "expected exact or sampled"); + } + } + return value as Record; +} + +function validateReceipt(value: unknown, path: string, issues: SchemaIssue[]): ReceiptV1 | null { + const receipt = record(value, path, ["schemaVersion", "kind", "createdAt", "status", "invalidReasons", "provenance", "correctness", "gateMetrics", "unsupportedMetrics", "metrics"], ["schemaVersion", "kind", "createdAt", "status", "invalidReasons", "provenance", "correctness", "gateMetrics", "unsupportedMetrics", "metrics"], issues); + if (!receipt) return null; + literal(receipt.schemaVersion, 1, pointer(path, "schemaVersion"), issues); + literal(receipt.kind, "pocketjs.perf.receipt", pointer(path, "kind"), issues); + if (nonEmptyString(receipt.createdAt, pointer(path, "createdAt"), issues)) { + const date = new Date(receipt.createdAt); + if (!Number.isFinite(date.valueOf()) || date.toISOString() !== receipt.createdAt) { + issue(issues, pointer(path, "createdAt"), "expected an ISO 8601 UTC timestamp"); + } + } + if (receipt.status !== "valid" && receipt.status !== "invalid") { + issue(issues, pointer(path, "status"), "expected valid or invalid"); + } + const reasons = array(receipt.invalidReasons, pointer(path, "invalidReasons"), issues); + if (reasons) reasons.forEach((reason, index) => nonEmptyString(reason, pointer(pointer(path, "invalidReasons"), index), issues)); + validateProvenance(receipt.provenance, pointer(path, "provenance"), issues); + const gatePath = pointer(path, "gateMetrics"); + const gates = array(receipt.gateMetrics, gatePath, issues); + const gateIds = new Set(); + const gateIndexes = new Map(); + if (gates) gates.forEach((metric, index) => { + const metricPath = pointer(gatePath, index); + if (!nonEmptyString(metric, metricPath, issues)) return; + if (!isMetricId(metric)) { + issue(issues, metricPath, "unknown metric id"); + return; + } + if (gateIds.has(metric)) issue(issues, metricPath, "duplicate gate metric"); + gateIds.add(metric); + if (!gateIndexes.has(metric)) gateIndexes.set(metric, index); + if (METRIC_CATALOG[metric].diagnostic) { + issue(issues, metricPath, "diagnostic metrics cannot be regression gates"); + } + }); + const unsupportedPath = pointer(path, "unsupportedMetrics"); + const unsupported = array(receipt.unsupportedMetrics, unsupportedPath, issues); + const unsupportedIds = new Set(); + const unsupportedIndexes = new Map(); + if (unsupported) unsupported.forEach((metric, index) => { + const metricPath = pointer(unsupportedPath, index); + if (!nonEmptyString(metric, metricPath, issues)) return; + if (!isMetricId(metric)) { + issue(issues, metricPath, "unknown metric id"); + return; + } + if (unsupportedIds.has(metric)) issue(issues, metricPath, "duplicate unsupported metric"); + unsupportedIds.add(metric); + if (!unsupportedIndexes.has(metric)) unsupportedIndexes.set(metric, index); + }); + const metrics = validateMetrics(receipt.metrics, pointer(path, "metrics"), issues); + if (receipt.status === "valid") { + if (reasons && reasons.length !== 0) issue(issues, pointer(path, "invalidReasons"), "must be empty for a valid receipt"); + validateCorrectness(receipt.correctness, pointer(path, "correctness"), issues); + if (metrics && Object.keys(metrics).length === 0) issue(issues, pointer(path, "metrics"), "a valid receipt must contain metrics"); + if (metrics && gates) { + for (const metric of unsupportedIds) { + if (!gateIds.has(metric)) { + issue( + issues, + pointer(unsupportedPath, unsupportedIndexes.get(metric)!), + "must also be declared in gateMetrics", + ); + } + } + for (const metric of gateIds) { + const observed = Object.hasOwn(metrics, metric); + const unsupportedByExecutor = unsupportedIds.has(metric); + if (observed && unsupportedByExecutor) { + issue( + issues, + pointer(unsupportedPath, unsupportedIndexes.get(metric)!), + "a gate metric cannot be both observed and unsupported", + ); + } else if (!observed && !unsupportedByExecutor) { + issue( + issues, + pointer(gatePath, gateIndexes.get(metric)!), + "gate metric is neither observed nor explicitly unsupported", + ); + } + } + } + } else if (receipt.status === "invalid") { + if (reasons && reasons.length === 0) issue(issues, pointer(path, "invalidReasons"), "must contain a reason for an invalid receipt"); + if (receipt.correctness !== null) validateCorrectness(receipt.correctness, pointer(path, "correctness"), issues); + } + return value as ReceiptV1; +} + +function validateThreshold(value: unknown, path: string, issues: SchemaIssue[]): void { + const threshold = record(value, path, ["relative", "absolute"], ["relative", "absolute"], issues); + if (!threshold) return; + nonNegativeNumber(threshold.relative, pointer(path, "relative"), issues); + nonNegativeNumber(threshold.absolute, pointer(path, "absolute"), issues); +} + +function validateMetricBudget(value: unknown, path: string, issues: SchemaIssue[]): MetricBudgetV1 | null { + const budget = record(value, path, ["warn", "regression", "hardMax", "hardMin", "executors"], [], issues); + if (!budget) return null; + if (!["warn", "regression", "hardMax", "hardMin"].some((key) => Object.hasOwn(budget, key))) { + issue(issues, path, "expected at least one threshold or hard limit"); + } + if (Object.hasOwn(budget, "warn")) validateThreshold(budget.warn, pointer(path, "warn"), issues); + if (Object.hasOwn(budget, "regression")) validateThreshold(budget.regression, pointer(path, "regression"), issues); + if (Object.hasOwn(budget, "hardMax")) nonNegativeNumber(budget.hardMax, pointer(path, "hardMax"), issues); + if (Object.hasOwn(budget, "hardMin")) nonNegativeNumber(budget.hardMin, pointer(path, "hardMin"), issues); + if (Object.hasOwn(budget, "executors")) stringArray(budget.executors, pointer(path, "executors"), issues, { nonEmpty: true, unique: true }); + if (typeof budget.hardMin === "number" && typeof budget.hardMax === "number" && budget.hardMin > budget.hardMax) { + issue(issues, path, "hardMin must not exceed hardMax"); + } + const warn = budget.warn as UnknownRecord | undefined; + const regression = budget.regression as UnknownRecord | undefined; + if (warn && regression) { + if (typeof warn.relative === "number" && typeof regression.relative === "number" && regression.relative < warn.relative) { + issue(issues, pointer(path, "regression/relative"), "must be greater than or equal to warn.relative"); + } + if (typeof warn.absolute === "number" && typeof regression.absolute === "number" && regression.absolute < warn.absolute) { + issue(issues, pointer(path, "regression/absolute"), "must be greater than or equal to warn.absolute"); + } + } + return value as MetricBudgetV1; +} + +function validateBudgetMetricId(metricId: string, path: string, issues: SchemaIssue[]): void { + if (!isMetricId(metricId)) { + issue(issues, path, "unknown metric id"); + return; + } + if (METRIC_CATALOG[metricId].diagnostic) { + issue(issues, path, "diagnostic metrics cannot have regression budgets"); + } +} + +function validateBudgetSet(value: unknown, path: string, issues: SchemaIssue[]): BudgetSetV1 | null { + const budgetSet = record(value, path, ["schemaVersion", "kind", "id", "metrics", "scenarios"], ["schemaVersion", "kind", "id", "metrics"], issues); + if (!budgetSet) return null; + literal(budgetSet.schemaVersion, 1, pointer(path, "schemaVersion"), issues); + literal(budgetSet.kind, "pocketjs.perf.budget-set", pointer(path, "kind"), issues); + nonEmptyString(budgetSet.id, pointer(path, "id"), issues); + const metrics = record(budgetSet.metrics, pointer(path, "metrics"), null, [], issues); + if (metrics) { + if (Object.keys(metrics).length === 0) issue(issues, pointer(path, "metrics"), "expected at least one metric budget"); + for (const [metricId, metricBudget] of Object.entries(metrics)) { + const metricPath = pointer(pointer(path, "metrics"), metricId); + validateBudgetMetricId(metricId, metricPath, issues); + validateMetricBudget(metricBudget, metricPath, issues); + } + } + if (Object.hasOwn(budgetSet, "scenarios")) { + const scenarios = record(budgetSet.scenarios, pointer(path, "scenarios"), null, [], issues); + if (scenarios) { + for (const [scenarioId, scenarioMetricsValue] of Object.entries(scenarios)) { + const scenarioPath = pointer(pointer(path, "scenarios"), scenarioId); + nonEmptyString(scenarioId, scenarioPath, issues); + const scenarioMetrics = record(scenarioMetricsValue, scenarioPath, null, [], issues); + if (!scenarioMetrics) continue; + if (Object.keys(scenarioMetrics).length === 0) { + issue(issues, scenarioPath, "expected at least one metric budget"); + } + for (const [metricId, metricBudget] of Object.entries(scenarioMetrics)) { + const metricPath = pointer(scenarioPath, metricId); + validateBudgetMetricId(metricId, metricPath, issues); + validateMetricBudget(metricBudget, metricPath, issues); + } + } + } + } + return value as BudgetSetV1; +} + +function validateReason(value: unknown, path: string, issues: SchemaIssue[]): void { + const reason = record(value, path, ["code", "path", "message"], ["code", "path", "message"], issues); + if (!reason) return; + enumString(reason.code, REASON_CODES, pointer(path, "code"), issues); + nonEmptyString(reason.path, pointer(path, "path"), issues); + nonEmptyString(reason.message, pointer(path, "message"), issues); +} + +function validateComparison(value: unknown, path: string, issues: SchemaIssue[]): ComparisonV1 | null { + const comparison = record(value, path, ["schemaVersion", "kind", "status", "comparable", "budgetId", "base", "candidate", "unsupportedMetrics", "reasons", "metrics"], ["schemaVersion", "kind", "status", "comparable", "budgetId", "base", "candidate", "unsupportedMetrics", "reasons", "metrics"], issues); + if (!comparison) return null; + literal(comparison.schemaVersion, 1, pointer(path, "schemaVersion"), issues); + literal(comparison.kind, "pocketjs.perf.comparison", pointer(path, "kind"), issues); + enumString(comparison.status, COMPARISON_STATUSES, pointer(path, "status"), issues); + boolean(comparison.comparable, pointer(path, "comparable"), issues); + nonEmptyString(comparison.budgetId, pointer(path, "budgetId"), issues); + for (const side of ["base", "candidate"] as const) { + const sidePath = pointer(path, side); + const reference = record(comparison[side], sidePath, ["sourceRevision", "binarySha256"], ["sourceRevision", "binarySha256"], issues); + if (reference) { + nonEmptyString(reference.sourceRevision, pointer(sidePath, "sourceRevision"), issues); + sha256(reference.binarySha256, pointer(sidePath, "binarySha256"), issues); + } + } + const unsupportedMetrics = array(comparison.unsupportedMetrics, pointer(path, "unsupportedMetrics"), issues); + if (unsupportedMetrics) { + const seen = new Set(); + unsupportedMetrics.forEach((metric, index) => { + const metricPath = pointer(pointer(path, "unsupportedMetrics"), index); + if (!nonEmptyString(metric, metricPath, issues)) return; + if (!isMetricId(metric)) issue(issues, metricPath, "unknown metric id"); + if (seen.has(metric)) issue(issues, metricPath, "duplicate unsupported metric"); + seen.add(metric); + }); + } + const reasons = array(comparison.reasons, pointer(path, "reasons"), issues); + if (reasons) reasons.forEach((reason, index) => validateReason(reason, pointer(pointer(path, "reasons"), index), issues)); + const metrics = array(comparison.metrics, pointer(path, "metrics"), issues); + const seenMetricIds = new Set(); + if (metrics) metrics.forEach((metricValue, index) => { + const metricPath = pointer(pointer(path, "metrics"), index); + const metric = record(metricValue, metricPath, ["id", "label", "direction", "kind", "unit", "status", "baseline", "candidate", "delta", "relativeDelta", "sampleKind", "sampleCount", "confidenceInterval", "budget", "reasons"], ["id", "label", "direction", "kind", "unit", "status", "baseline", "candidate", "delta", "relativeDelta", "sampleKind", "sampleCount", "confidenceInterval", "budget", "reasons"], issues); + if (!metric) return; + if (nonEmptyString(metric.id, pointer(metricPath, "id"), issues)) { + if (!isMetricId(metric.id)) { + issue(issues, pointer(metricPath, "id"), "unknown metric id"); + } else { + if (seenMetricIds.has(metric.id)) issue(issues, pointer(metricPath, "id"), "duplicate metric id"); + seenMetricIds.add(metric.id); + const definition = METRIC_CATALOG[metric.id]; + if (metric.label !== definition.label) issue(issues, pointer(metricPath, "label"), "must match the metric catalog"); + if (metric.direction !== definition.direction) issue(issues, pointer(metricPath, "direction"), "must match the metric catalog"); + if (metric.kind !== definition.kind) issue(issues, pointer(metricPath, "kind"), "must match the metric catalog"); + if (metric.unit !== definition.unit) issue(issues, pointer(metricPath, "unit"), "must match the metric catalog"); + } + } + nonEmptyString(metric.label, pointer(metricPath, "label"), issues); + enumString(metric.direction, new Set(["lower-is-better", "higher-is-better"] as const), pointer(metricPath, "direction"), issues); + enumString(metric.kind, new Set(["counter", "gauge"] as const), pointer(metricPath, "kind"), issues); + enumString(metric.unit, new Set(["count", "bytes", "ns"] as const), pointer(metricPath, "unit"), issues); + enumString(metric.status, COMPARISON_STATUSES, pointer(metricPath, "status"), issues); + for (const key of ["baseline", "candidate", "delta", "relativeDelta"] as const) { + if (metric[key] !== null) finiteNumber(metric[key], pointer(metricPath, key), issues); + } + enumString(metric.sampleKind, new Set(["exact", "paired"] as const), pointer(metricPath, "sampleKind"), issues); + if (metric.sampleCount !== null) positiveInteger(metric.sampleCount, pointer(metricPath, "sampleCount"), issues); + if (metric.confidenceInterval !== null) { + const intervalPath = pointer(metricPath, "confidenceInterval"); + const interval = record(metric.confidenceInterval, intervalPath, ["level", "lower", "upper", "method", "seed", "iterations"], ["level", "lower", "upper", "method", "seed", "iterations"], issues); + if (interval) { + literal(interval.level, 0.95, pointer(intervalPath, "level"), issues); + finiteNumber(interval.lower, pointer(intervalPath, "lower"), issues); + finiteNumber(interval.upper, pointer(intervalPath, "upper"), issues); + literal(interval.method, "paired-bootstrap", pointer(intervalPath, "method"), issues); + nonNegativeInteger(interval.seed, pointer(intervalPath, "seed"), issues); + positiveInteger(interval.iterations, pointer(intervalPath, "iterations"), issues); + if (typeof interval.lower === "number" && typeof interval.upper === "number" && interval.lower > interval.upper) { + issue(issues, intervalPath, "lower must not exceed upper"); + } + } + } + if (metric.sampleKind === "exact" && (metric.sampleCount !== null || metric.confidenceInterval !== null)) { + issue(issues, metricPath, "exact comparisons must not contain sample metadata"); + } + if (metric.sampleKind === "paired" && (metric.sampleCount === null || metric.confidenceInterval === null)) { + issue(issues, metricPath, "paired comparisons require sampleCount and confidenceInterval"); + } + if (metric.budget !== null) validateMetricBudget(metric.budget, pointer(metricPath, "budget"), issues); + const metricReasons = array(metric.reasons, pointer(metricPath, "reasons"), issues); + if (metricReasons) { + metricReasons.forEach((reason, reasonIndex) => validateReason(reason, pointer(pointer(metricPath, "reasons"), reasonIndex), issues)); + if (metric.status === "pass" && metricReasons.length > 0) issue(issues, pointer(metricPath, "reasons"), "pass metrics must not contain reasons"); + if (metric.status !== "pass" && metricReasons.length === 0) issue(issues, pointer(metricPath, "reasons"), "non-pass metrics must contain a reason"); + } + }); + if (comparison.status === "invalid" && comparison.comparable !== false) { + issue(issues, pointer(path, "comparable"), "must be false when status is invalid"); + } + if (comparison.status !== "invalid" && comparison.comparable !== true) { + issue(issues, pointer(path, "comparable"), "must be true when status is not invalid"); + } + if (comparison.status === "invalid" && reasons && reasons.length === 0) { + issue(issues, pointer(path, "reasons"), "invalid comparisons must contain a reason"); + } + return value as ComparisonV1; +} + +function parse(schema: string, value: unknown, validator: (value: unknown, path: string, issues: SchemaIssue[]) => T | null): T { + const issues: SchemaIssue[] = []; + const result = validator(value, "/", issues); + if (!result || issues.length > 0) throw new SchemaValidationError(schema, issues); + return result; +} + +function safeParse(parser: (value: unknown) => T, value: unknown): SafeParseResult { + try { + return { success: true, data: parser(value) }; + } catch (error) { + if (error instanceof SchemaValidationError) return { success: false, error }; + throw error; + } +} + +export function parseInputTapeV1(value: unknown): InputTapeV1 { + return parse("InputTapeV1", value, validateInputTape); +} + +export function safeParseInputTapeV1(value: unknown): SafeParseResult { + return safeParse(parseInputTapeV1, value); +} + +export function assertInputTapeV1(value: unknown): asserts value is InputTapeV1 { + parseInputTapeV1(value); +} + +export function parseScenarioV1(value: unknown): ScenarioV1 { + return parse("ScenarioV1", value, validateScenario); +} + +export function safeParseScenarioV1(value: unknown): SafeParseResult { + return safeParse(parseScenarioV1, value); +} + +export function assertScenarioV1(value: unknown): asserts value is ScenarioV1 { + parseScenarioV1(value); +} + +export function parseReceiptV1(value: unknown): ReceiptV1 { + return parse("ReceiptV1", value, validateReceipt); +} + +export function safeParseReceiptV1(value: unknown): SafeParseResult { + return safeParse(parseReceiptV1, value); +} + +export function assertReceiptV1(value: unknown): asserts value is ReceiptV1 { + parseReceiptV1(value); +} + +export function parseBudgetSetV1(value: unknown): BudgetSetV1 { + return parse("BudgetSetV1", value, validateBudgetSet); +} + +export function safeParseBudgetSetV1(value: unknown): SafeParseResult { + return safeParse(parseBudgetSetV1, value); +} + +export function assertBudgetSetV1(value: unknown): asserts value is BudgetSetV1 { + parseBudgetSetV1(value); +} + +export function parseComparisonV1(value: unknown): ComparisonV1 { + return parse("ComparisonV1", value, validateComparison); +} + +export function safeParseComparisonV1(value: unknown): SafeParseResult { + return safeParse(parseComparisonV1, value); +} + +export function assertComparisonV1(value: unknown): asserts value is ComparisonV1 { + parseComparisonV1(value); +} + +// Unversioned aliases always point at the current wire format. +export const parseInputTape = parseInputTapeV1; +export const parseScenario = parseScenarioV1; +export const parseReceipt = parseReceiptV1; +export const parseBudgetSet = parseBudgetSetV1; +export const parseComparison = parseComparisonV1; + +export function expectedMetricUnit(metricId: string): string | undefined { + return isMetricId(metricId) ? METRIC_CATALOG[metricId].unit : undefined; +} diff --git a/tools/perf/core/types.ts b/tools/perf/core/types.ts new file mode 100644 index 00000000..afa9b25c --- /dev/null +++ b/tools/perf/core/types.ts @@ -0,0 +1,316 @@ +export type JsonPrimitive = string | number | boolean | null; +export type JsonValue = + | JsonPrimitive + | readonly JsonValue[] + | { readonly [key: string]: JsonValue }; + +export const PERF_SCHEMA_VERSION = 1 as const; + +export type FrameworkId = "solid" | "vue-vapor" | "octane" | "core"; + +export interface ButtonInputTrackV1 { + readonly kind: "button"; + readonly control: string; + readonly samples: readonly { + readonly frame: number; + readonly pressed: boolean; + }[]; +} + +export interface AnalogInputTrackV1 { + readonly kind: "analog"; + readonly control: string; + readonly samples: readonly { + readonly frame: number; + readonly value: number; + }[]; +} + +export interface TouchInputTrackV1 { + readonly kind: "touch"; + readonly control: string; + readonly samples: readonly { + readonly frame: number; + readonly phase: "start" | "move" | "end" | "cancel"; + readonly x: number; + readonly y: number; + }[]; +} + +export interface RelativeAxisInputTrackV1 { + readonly kind: "relative-axis"; + readonly control: string; + readonly samples: readonly { + readonly frame: number; + readonly delta: number; + }[]; +} + +export interface EffectInputTrackV1 { + readonly kind: "effect"; + readonly effect: string; + readonly samples: readonly { + readonly frame: number; + readonly value: JsonValue; + }[]; +} + +export type InputTrackV1 = + | ButtonInputTrackV1 + | AnalogInputTrackV1 + | TouchInputTrackV1 + | RelativeAxisInputTrackV1 + | EffectInputTrackV1; + +export interface InputTapeV1 { + readonly schemaVersion: 1; + readonly kind: "pocketjs.perf.input-tape"; + readonly id: string; + readonly frames: number; + readonly tracks: readonly InputTrackV1[]; +} + +export type CorrectnessCapture = + | "framebuffer" + | "drawList" + | "state" + | "effects"; + +export interface ScenarioV1 { + readonly schemaVersion: 1; + readonly kind: "pocketjs.perf.scenario"; + readonly id: string; + readonly suite: string; + readonly subject: { + readonly id: string; + readonly family: string; + readonly framework: FrameworkId; + readonly entry: string; + }; + readonly executorRequirements: readonly string[]; + readonly frames: number; + readonly tape: InputTapeV1; + readonly phases: readonly { + readonly name: string; + readonly startFrame: number; + readonly endFrame: number; + readonly collect: boolean; + }[]; + readonly checkpoints: readonly { + readonly frame: number; + readonly capture: readonly CorrectnessCapture[]; + }[]; + readonly params: Readonly>; +} + +export type MetricDirection = "lower-is-better" | "higher-is-better"; +export type MetricKind = "counter" | "gauge"; +export type MetricUnit = "count" | "bytes" | "ns"; + +export interface MetricDefinition { + readonly id: string; + readonly label: string; + readonly direction: MetricDirection; + readonly kind: MetricKind; + readonly unit: MetricUnit; + readonly diagnostic: boolean; +} + +export interface ExactMetricSampleV1 { + readonly kind: "exact"; + readonly value: number; + readonly unit: MetricUnit; +} + +export interface SampledMetricSampleV1 { + readonly kind: "sampled"; + /** Raw observations, kept in execution order for paired comparisons. */ + readonly samples: readonly number[]; + readonly unit: MetricUnit; +} + +export type MetricSampleV1 = ExactMetricSampleV1 | SampledMetricSampleV1; + +export interface ReceiptProvenanceV1 { + readonly source: { + readonly revision: string; + readonly dirty: boolean; + readonly contentHash: string; + }; + readonly scenario: { + readonly id: string; + readonly suite: string; + readonly framework: FrameworkId; + readonly manifestHash: string; + readonly inputTapeHash: string; + }; + readonly toolchain: { + readonly rustc: string; + readonly cCompiler: string; + readonly sysroot: string; + readonly qemu?: string; + readonly bun?: string; + }; + readonly build: { + readonly target: string; + readonly profile: string; + readonly rustFlags: readonly string[]; + readonly cFlags: readonly string[]; + readonly linkerFlags: readonly string[]; + }; + readonly executor: { + readonly id: string; + readonly version: string; + readonly profile: string; + readonly fingerprint: string; + }; + readonly binary: { + readonly sha256: string; + }; +} + +export interface CorrectnessReceiptV1 { + readonly framebufferHash: string; + readonly drawListHash: string; + readonly stateHash: string; + readonly effectHash: string; +} + +interface ReceiptBaseV1 { + readonly schemaVersion: 1; + readonly kind: "pocketjs.perf.receipt"; + readonly createdAt: string; + readonly provenance: ReceiptProvenanceV1; + /** Scenario-declared observations that must have an applicable budget. */ + readonly gateMetrics: readonly string[]; + /** Gate observations this executor cannot truthfully produce. */ + readonly unsupportedMetrics: readonly string[]; + readonly metrics: Readonly>; +} + +export interface ValidReceiptV1 extends ReceiptBaseV1 { + readonly status: "valid"; + readonly invalidReasons: readonly []; + readonly correctness: CorrectnessReceiptV1; +} + +export interface InvalidReceiptV1 extends ReceiptBaseV1 { + readonly status: "invalid"; + readonly invalidReasons: readonly string[]; + readonly correctness: CorrectnessReceiptV1 | null; +} + +export type ReceiptV1 = ValidReceiptV1 | InvalidReceiptV1; + +export interface RelativeAbsoluteThresholdV1 { + /** Ratio rather than percentage: 0.01 means one percent. */ + readonly relative: number; + readonly absolute: number; +} + +export interface MetricBudgetV1 { + readonly warn?: RelativeAbsoluteThresholdV1; + readonly regression?: RelativeAbsoluteThresholdV1; + readonly hardMax?: number; + readonly hardMin?: number; + /** Omitted means the budget applies to every executor. */ + readonly executors?: readonly string[]; +} + +export interface BudgetSetV1 { + readonly schemaVersion: 1; + readonly kind: "pocketjs.perf.budget-set"; + readonly id: string; + readonly metrics: Readonly>; + /** Exact scenario id, or the base id before a `#phase` suffix. */ + readonly scenarios?: Readonly>>>; +} + +export type ComparisonStatus = "pass" | "warn" | "regression" | "invalid"; + +export interface ComparisonReasonV1 { + readonly code: + | "receipt-invalid" + | "provenance-mismatch" + | "correctness-mismatch" + | "metric-missing" + | "budget-missing" + | "metric-support-mismatch" + | "unit-mismatch" + | "catalog-unit-mismatch" + | "sample-kind-mismatch" + | "sample-count-mismatch" + | "threshold-exceeded" + | "hard-limit-exceeded"; + readonly path: string; + readonly message: string; +} + +export interface MetricComparisonV1 { + readonly id: string; + readonly label: string; + readonly direction: MetricDirection; + readonly kind: MetricKind; + readonly unit: MetricUnit; + readonly status: ComparisonStatus; + readonly baseline: number | null; + readonly candidate: number | null; + readonly delta: number | null; + /** Signed candidate delta divided by abs(baseline), or null at baseline zero. */ + readonly relativeDelta: number | null; + readonly sampleKind: "exact" | "paired"; + readonly sampleCount: number | null; + readonly confidenceInterval: { + readonly level: 0.95; + readonly lower: number; + readonly upper: number; + readonly method: "paired-bootstrap"; + readonly seed: number; + readonly iterations: number; + } | null; + readonly budget: MetricBudgetV1 | null; + readonly reasons: readonly ComparisonReasonV1[]; +} + +export interface ComparisonV1 { + readonly schemaVersion: 1; + readonly kind: "pocketjs.perf.comparison"; + readonly status: ComparisonStatus; + readonly comparable: boolean; + readonly budgetId: string; + readonly base: { + readonly sourceRevision: string; + readonly binarySha256: string; + }; + readonly candidate: { + readonly sourceRevision: string; + readonly binarySha256: string; + }; + /** Gates intentionally unavailable on both sides for this executor. */ + readonly unsupportedMetrics: readonly string[]; + readonly reasons: readonly ComparisonReasonV1[]; + readonly metrics: readonly MetricComparisonV1[]; +} + +export interface SchemaIssue { + readonly path: string; + readonly message: string; +} + +export type SafeParseResult = + | { readonly success: true; readonly data: T } + | { readonly success: false; readonly error: SchemaValidationError }; + +export class SchemaValidationError extends Error { + readonly issues: readonly SchemaIssue[]; + + constructor(schema: string, issues: readonly SchemaIssue[]) { + super( + `${schema} validation failed: ${issues + .map((issue) => `${issue.path}: ${issue.message}`) + .join("; ")}`, + ); + this.name = "SchemaValidationError"; + this.issues = issues; + } +} diff --git a/tools/perf/damage-fixture/.gitignore b/tools/perf/damage-fixture/.gitignore new file mode 100644 index 00000000..b83d2226 --- /dev/null +++ b/tools/perf/damage-fixture/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/tools/perf/damage-fixture/Cargo.lock b/tools/perf/damage-fixture/Cargo.lock new file mode 100644 index 00000000..846dba73 --- /dev/null +++ b/tools/perf/damage-fixture/Cargo.lock @@ -0,0 +1,233 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "grid" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b40ca9252762c466af32d0b1002e91e4e1bc5398f77455e55474deb466355ff5" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "pocketjs-core" +version = "0.1.0" +dependencies = [ + "taffy", +] + +[[package]] +name = "pocketjs-perf-damage" +version = "0.1.0" +dependencies = [ + "libc", + "pocketjs-core", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "slotmap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" +dependencies = [ + "version_check", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "taffy" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfde4e2f8595f222ceaae1fb16b4963952e9b33e358869dc4cd6316b0e0790cd" +dependencies = [ + "arrayvec", + "grid", + "serde", + "slotmap", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tools/perf/damage-fixture/Cargo.toml b/tools/perf/damage-fixture/Cargo.toml new file mode 100644 index 00000000..6bb7e8f8 --- /dev/null +++ b/tools/perf/damage-fixture/Cargo.toml @@ -0,0 +1,21 @@ +[workspace] + +[package] +name = "pocketjs-perf-damage" +version = "0.1.0" +edition = "2024" +publish = false + +[dependencies] +libc = "0.2" +pocketjs-core = { path = "../../../engine/core", features = ["std"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" + +[profile.release] +opt-level = 3 +lto = "thin" +codegen-units = 1 +panic = "abort" +strip = "debuginfo" diff --git a/tools/perf/damage-fixture/src/main.rs b/tools/perf/damage-fixture/src/main.rs new file mode 100644 index 00000000..97de45f3 --- /dev/null +++ b/tools/perf/damage-fixture/src/main.rs @@ -0,0 +1,899 @@ +use std::alloc::{GlobalAlloc, Layout, System}; +use std::collections::BTreeMap; +use std::env; +use std::fs; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Instant; + +use pocketjs_core::Ui; +use pocketjs_core::damage::{DEFAULT_DAMAGE_REGIONS, DamagePlan, DamagePolicy, DamageTracker}; +use pocketjs_core::raster; +use pocketjs_core::spec::{self, draw_op}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +const DAMAGE_PREFIX: &str = "POCKETJS_PERF_DAMAGE "; +const GUEST_PREFIX: &str = "POCKETJS_PERF_GUEST "; +const WIDTH: usize = 96; +const HEIGHT: usize = 64; +const SCALE: u32 = 1; + +#[cfg(target_os = "linux")] +const MARKER_SYSCALL: u32 = 4096; +#[cfg(target_os = "linux")] +const MARKER_MAGIC: u32 = 0x504a_424d; +#[cfg(target_os = "linux")] +const MARKER_VERSION: u32 = 1; +#[cfg(target_os = "linux")] +const MARKER_COOKIE: u32 = 0xc001_c0de; +const MARKER_BEGIN: u32 = 1; +const MARKER_END: u32 = 2; + +const EXPECTED_PHASES: [&str; 8] = [ + "single-small", + "corner-touch", + "overlap", + "eight-sparse", + "structural", + "clip-transform", + "texture-in-place", + "settle", +]; + +struct CountingGlobal; + +static ALLOC_CALLS: AtomicU64 = AtomicU64::new(0); +static ALLOC_BYTES: AtomicU64 = AtomicU64::new(0); +static CURRENT_BYTES: AtomicU64 = AtomicU64::new(0); +static PEAK_BYTES: AtomicU64 = AtomicU64::new(0); +static PHASE_BASELINE_BYTES: AtomicU64 = AtomicU64::new(0); + +#[global_allocator] +static GLOBAL: CountingGlobal = CountingGlobal; + +#[inline] +fn record_alloc(bytes: usize) { + let bytes = bytes as u64; + ALLOC_CALLS.fetch_add(1, Ordering::Relaxed); + ALLOC_BYTES.fetch_add(bytes, Ordering::Relaxed); + let current = CURRENT_BYTES.fetch_add(bytes, Ordering::Relaxed) + bytes; + let mut peak = PEAK_BYTES.load(Ordering::Relaxed); + while current > peak { + match PEAK_BYTES.compare_exchange_weak(peak, current, Ordering::Relaxed, Ordering::Relaxed) + { + Ok(_) => break, + Err(next) => peak = next, + } + } +} + +unsafe impl GlobalAlloc for CountingGlobal { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let ptr = unsafe { System.alloc(layout) }; + if !ptr.is_null() { + record_alloc(layout.size()); + } + ptr + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let ptr = unsafe { System.alloc_zeroed(layout) }; + if !ptr.is_null() { + record_alloc(layout.size()); + } + ptr + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + CURRENT_BYTES.fetch_sub(layout.size() as u64, Ordering::Relaxed); + unsafe { System.dealloc(ptr, layout) }; + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let next = unsafe { System.realloc(ptr, layout, new_size) }; + if !next.is_null() { + CURRENT_BYTES.fetch_sub(layout.size() as u64, Ordering::Relaxed); + record_alloc(new_size); + } + next + } +} + +#[derive(Clone, Copy)] +struct AllocationSnapshot { + calls: u64, + bytes: u64, + current: u64, + peak: u64, + baseline: u64, +} + +fn reset_allocation_phase() { + ALLOC_CALLS.store(0, Ordering::Relaxed); + ALLOC_BYTES.store(0, Ordering::Relaxed); + let current = CURRENT_BYTES.load(Ordering::Relaxed); + PHASE_BASELINE_BYTES.store(current, Ordering::Relaxed); + PEAK_BYTES.store(current, Ordering::Relaxed); +} + +fn allocation_snapshot() -> AllocationSnapshot { + AllocationSnapshot { + calls: ALLOC_CALLS.load(Ordering::Relaxed), + bytes: ALLOC_BYTES.load(Ordering::Relaxed), + current: CURRENT_BYTES.load(Ordering::Relaxed), + peak: PEAK_BYTES.load(Ordering::Relaxed), + baseline: PHASE_BASELINE_BYTES.load(Ordering::Relaxed), + } +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Scenario { + schema_version: u32, + kind: String, + id: String, + suite: String, + subject: Subject, + executor_requirements: Vec, + frames: u32, + tape: serde_json::Value, + phases: Vec, + checkpoints: Vec, + params: serde_json::Value, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Subject { + id: String, + family: String, + framework: String, + entry: String, +} + +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Phase { + name: String, + start_frame: u32, + end_frame: u32, + collect: bool, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Checkpoint { + frame: u32, + capture: Vec, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Mode { + Correctness, + Measurement, + Markers, +} + +struct Args { + scenario: PathBuf, + mode: Mode, +} + +fn parse_args() -> Result { + let mut scenario = None; + let mut mode = None; + let mut args = env::args().skip(1); + while let Some(arg) = args.next() { + match arg.as_str() { + "--scenario" => scenario = args.next().map(PathBuf::from), + "--correctness" => set_mode(&mut mode, Mode::Correctness)?, + "--measurement" => set_mode(&mut mode, Mode::Measurement)?, + "--markers" => set_mode(&mut mode, Mode::Markers)?, + "--help" | "-h" => { + println!( + "usage: pocketjs-perf-damage --scenario FILE (--correctness|--measurement|--markers)" + ); + std::process::exit(0); + } + _ => return Err(format!("unknown argument {arg}")), + } + } + Ok(Args { + scenario: scenario.ok_or_else(|| "--scenario is required".to_string())?, + mode: mode.ok_or_else(|| { + "exactly one of --correctness, --measurement, or --markers is required".to_string() + })?, + }) +} + +fn set_mode(slot: &mut Option, next: Mode) -> Result<(), String> { + if slot.replace(next).is_some() { + return Err("mode flags are mutually exclusive".to_string()); + } + Ok(()) +} + +fn read_scenario(path: &PathBuf) -> Result { + let bytes = fs::read(path).map_err(|error| format!("reading {}: {error}", path.display()))?; + let scenario: Scenario = serde_json::from_slice(&bytes) + .map_err(|error| format!("parsing {}: {error}", path.display()))?; + validate_scenario(&scenario)?; + Ok(scenario) +} + +fn validate_scenario(scenario: &Scenario) -> Result<(), String> { + if scenario.schema_version != 1 || scenario.kind != "pocketjs.perf.scenario" { + return Err("unsupported scenario schema".to_string()); + } + if scenario.subject.family != "core-lab" || scenario.subject.framework != "core" { + return Err( + "damage fixture requires subject family core-lab and framework core".to_string(), + ); + } + if scenario.subject.id != "core-damage-lab" + || scenario.subject.entry != "tools/perf/fixtures/core-damage-lab" + { + return Err("damage fixture received an unknown subject".to_string()); + } + if scenario.frames == 0 || scenario.phases.len() != EXPECTED_PHASES.len() { + return Err("damage fixture requires all eight non-empty phases".to_string()); + } + let mut previous_end = 0; + for (index, phase) in scenario.phases.iter().enumerate() { + if phase.name != EXPECTED_PHASES[index] { + return Err(format!( + "damage phase {index} must be {}, got {}", + EXPECTED_PHASES[index], phase.name + )); + } + if !phase.collect + || phase.start_frame != previous_end + || phase.end_frame <= phase.start_frame + { + return Err(format!( + "damage phase {} must be collected and contiguous", + phase.name + )); + } + previous_end = phase.end_frame; + } + if previous_end != scenario.frames { + return Err("damage phases must cover every scenario frame".to_string()); + } + for checkpoint in &scenario.checkpoints { + if checkpoint.frame >= scenario.frames { + return Err(format!( + "checkpoint {} is outside the scenario", + checkpoint.frame + )); + } + for capture in &checkpoint.capture { + if !matches!( + capture.as_str(), + "framebuffer" | "drawList" | "state" | "effects" + ) { + return Err(format!("unsupported correctness capture {capture}")); + } + } + } + // Deserializing these fields is deliberate: malformed manifests must not + // turn into a valid-looking benchmark merely because this fixture ignores input. + if !scenario.tape.is_object() || !scenario.params.is_object() { + return Err("damage tape and params must be objects".to_string()); + } + if !scenario + .executor_requirements + .iter() + .any(|item| item == "fixture.core.damage") + { + return Err("damage scenario is missing fixture.core.damage".to_string()); + } + Ok(()) +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct PhaseStats { + name: String, + frames: u32, + full_redraw_frames: u32, + empty_frames: u32, + max_regions: usize, + total_damage_area: u64, +} + +impl PhaseStats { + fn new(name: &str) -> Self { + Self { + name: name.to_string(), + frames: 0, + full_redraw_frames: 0, + empty_frames: 0, + max_regions: 0, + total_damage_area: 0, + } + } + + fn observe(&mut self, plan: &DamagePlan) { + self.frames += 1; + self.full_redraw_frames += u32::from(plan.is_full_redraw()); + self.empty_frames += u32::from(plan.is_empty()); + self.max_regions = self.max_regions.max(plan.region_count()); + self.total_damage_area = self.total_damage_area.saturating_add(plan.area()); + } +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct CorrectnessRecord { + schema_version: u32, + event: &'static str, + scenario_id: String, + framebuffer_trace_hash: String, + final_framebuffer_hash: String, + draw_list_hash: String, + state_hash: String, + effect_hash: String, + checkpoints: BTreeMap>, + phase_stats: Vec, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct PhaseTiming { + name: String, + start_frame: u32, + end_frame: u32, + wall_time_ns: u64, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct MeasurementRecord { + schema_version: u32, + event: &'static str, + scenario_id: String, + boot_wall_time_ns: u64, + phases: Vec, + final_framebuffer_hash: String, + final_draw_list_hash: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct GuestPhaseRecord { + schema_version: u32, + event: &'static str, + scenario_id: String, + phase: String, + phase_id: u32, + iteration: u32, + alloc_calls: u64, + allocated_bytes: u64, + current_bytes: u64, + peak_bytes: u64, + quickjs_live_bytes_after_gc: u64, + draw_list_hash: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct GuestCompleteRecord { + schema_version: u32, + event: &'static str, + scenario_id: String, + suite: String, + framework: String, + final_draw_list_hash: String, + final_state_hash: String, + effect_hash: String, +} + +struct Fixture { + ui: Ui, + tracker: DamageTracker, + framebuffer: Vec, + words: Vec, + texture: i32, + palette: [u8; 1024], + pixels: [u8; 64], +} + +impl Fixture { + fn new() -> Result { + let mut ui = Ui::new(); + ui.set_viewport(WIDTH as f32, HEIGHT as f32); + let mut palette = [0u8; 1024]; + for index in 0..256usize { + let color = 0xff00_0000u32 + | ((index as u32) << 16) + | (((255 - index) as u32) << 8) + | (index as u32 / 2); + palette[index * 4..index * 4 + 4].copy_from_slice(&color.to_le_bytes()); + } + let mut pixels = [0u8; 64]; + for (index, pixel) in pixels.iter_mut().enumerate() { + *pixel = index as u8; + } + let mut texture_data = Vec::with_capacity(palette.len() + pixels.len()); + texture_data.extend_from_slice(&palette); + texture_data.extend_from_slice(&pixels); + let texture = ui.upload_texture(&texture_data, 8, 8, spec::psm::PSM_T8); + if texture < 0 { + return Err("core rejected the fixture texture".to_string()); + } + Ok(Self { + ui, + tracker: DamageTracker::new(), + framebuffer: vec![0; WIDTH * HEIGHT * 4], + words: Vec::with_capacity(64), + texture, + palette, + pixels, + }) + } + + fn render_frame( + &mut self, + phase_name: &str, + local_frame: u32, + ) -> Result, String> { + self.build_draw_list(phase_name, local_frame)?; + raster::render_scaled_incremental( + &self.ui, + &self.words, + &mut self.framebuffer, + SCALE, + &mut self.tracker, + DamagePolicy::new(100), + ) + .map_err(|error| format!("damage planning failed in {phase_name}: {error:?}")) + } + + fn build_draw_list(&mut self, phase: &str, frame: u32) -> Result<(), String> { + self.words.clear(); + push_rect( + &mut self.words, + 0, + 0, + WIDTH as u16, + HEIGHT as u16, + 0xff18_1008, + ); + let alternate = frame & 1; + match phase { + "single-small" => { + push_rect( + &mut self.words, + 8 + (frame % 3) as i16, + 8, + 5, + 5, + if alternate == 0 { + 0xff44_ccff + } else { + 0xffff_8844 + }, + ); + } + "corner-touch" => { + push_rect(&mut self.words, 12, 12, 4, 4, phase_color(alternate, 0)); + push_rect(&mut self.words, 16, 16, 4, 4, phase_color(alternate, 1)); + } + "overlap" => { + push_rect(&mut self.words, 20, 16, 10, 8, phase_color(alternate, 0)); + push_rect(&mut self.words, 25, 20, 10, 8, phase_color(alternate, 1)); + } + "eight-sparse" => { + for index in 0..8u32 { + let x = 5 + (index % 4) as i16 * 22; + let y = 7 + (index / 4) as i16 * 30; + push_rect(&mut self.words, x, y, 4, 4, phase_color(alternate, index)); + } + } + "structural" => { + if alternate == 0 { + push_rect(&mut self.words, 18, 14, 22, 13, 0xff40_a0e0); + } else { + push_gradient(&mut self.words, 18, 14, 22, 13, 0xff40_a0e0, 0xffe0_6040); + push_rect(&mut self.words, 48, 22, 9, 9, 0xff70_d060); + } + } + "clip-transform" => { + let shift = (frame % 5) as i16; + push_scissor(&mut self.words, 11 + shift, 10, 34, 24); + push_rect( + &mut self.words, + 4 + shift, + 5, + 48, + 34, + phase_color(alternate, 3), + ); + self.words.push(draw_op::SCISSOR_POP); + } + "texture-in-place" => { + for (index, pixel) in self.pixels.iter_mut().enumerate() { + *pixel = ((index as u32 + frame * 7) & 0xff) as u8; + } + self.palette[0..4].copy_from_slice( + &(if alternate == 0 { + 0xff20_e080u32 + } else { + 0xffe0_4080u32 + }) + .to_le_bytes(), + ); + if !self + .ui + .update_texture_t8(self.texture, &self.palette, &self.pixels) + { + return Err("core rejected an in-place T8 texture update".to_string()); + } + push_texture(&mut self.words, self.texture, 32, 18, 24, 24); + } + "settle" => { + push_rect(&mut self.words, 36, 20, 18, 12, 0xff60_c080); + } + _ => return Err(format!("unknown damage phase {phase}")), + } + Ok(()) + } + + fn verify_full_render(&self) -> Result<(), String> { + let mut full = vec![0; self.framebuffer.len()]; + raster::render_scaled(&self.ui, &self.words, &mut full, SCALE); + if full != self.framebuffer { + return Err("incremental framebuffer differs from a full software render".to_string()); + } + Ok(()) + } +} + +fn phase_color(alternate: u32, seed: u32) -> u32 { + if alternate == 0 { + 0xff20_80d0u32.wrapping_add(seed.wrapping_mul(0x0008_1107)) + } else { + 0xffd0_7020u32.wrapping_sub(seed.wrapping_mul(0x0007_0905)) + } +} + +fn xy_word(x: i16, y: i16) -> u32 { + x as u16 as u32 | ((y as u16 as u32) << 16) +} + +fn wh_word(w: u16, h: u16) -> u32 { + w as u32 | ((h as u32) << 16) +} + +fn push_rect(words: &mut Vec, x: i16, y: i16, w: u16, h: u16, color: u32) { + words.extend_from_slice(&[draw_op::RECT, xy_word(x, y), wh_word(w, h), color]); +} + +fn push_gradient(words: &mut Vec, x: i16, y: i16, w: u16, h: u16, from: u32, to: u32) { + words.extend_from_slice(&[ + draw_op::GRAD_RECT, + xy_word(x, y), + wh_word(w, h), + from, + to, + spec::GradDir::ToRight as u32, + ]); +} + +fn push_scissor(words: &mut Vec, x: i16, y: i16, w: u16, h: u16) { + words.extend_from_slice(&[draw_op::SCISSOR, xy_word(x, y), wh_word(w, h)]); +} + +fn push_texture(words: &mut Vec, texture: i32, x: i16, y: i16, w: u16, h: u16) { + words.extend_from_slice(&[ + draw_op::TEX_QUAD, + texture as u32, + xy_word(x, y), + wh_word(w, h), + 0.0f32.to_bits(), + 0.0f32.to_bits(), + 1.0f32.to_bits(), + 1.0f32.to_bits(), + 0xffff_ffff, + ]); +} + +fn phase_for_frame(scenario: &Scenario, frame: u32) -> (&Phase, u32) { + let phase = scenario + .phases + .iter() + .find(|phase| phase.start_frame <= frame && frame < phase.end_frame) + .expect("validated phases cover all frames"); + (phase, frame - phase.start_frame) +} + +fn phase_id(scenario: &str, phase: &str) -> u32 { + let mut hash = 0x811c_9dc5u32; + for byte in scenario.bytes().chain([0]).chain(phase.bytes()) { + hash ^= u32::from(byte); + hash = hash.wrapping_mul(0x0100_0193); + } + hash +} + +fn marker(opcode: u32, id: u32) -> Result<(), String> { + #[cfg(target_os = "linux")] + let packed = (MARKER_VERSION << 8) | opcode; + #[cfg(target_os = "linux")] + let result = unsafe { + libc::syscall( + MARKER_SYSCALL as libc::c_long, + MARKER_MAGIC as libc::c_long, + packed as libc::c_long, + id as libc::c_long, + 0 as libc::c_long, + MARKER_COOKIE as libc::c_long, + 0 as libc::c_long, + ) + } as i64; + #[cfg(not(target_os = "linux"))] + let result = -1i64; + if result != 0 { + return Err(format!( + "QEMU marker rejected opcode {opcode}, phase {id}: return {result}" + )); + } + Ok(()) +} + +fn sha256(bytes: &[u8]) -> String { + hex_digest(Sha256::digest(bytes).as_slice()) +} + +fn hex_digest(bytes: &[u8]) -> String { + let mut output = String::with_capacity(bytes.len() * 2); + for byte in bytes { + use std::fmt::Write as _; + write!(&mut output, "{byte:02x}").expect("writing to String cannot fail"); + } + output +} + +fn fnv1a64(bytes: impl IntoIterator) -> String { + let mut hash = 0xcbf2_9ce4_8422_2325u64; + for byte in bytes { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + format!("fnv1a64:{hash:016x}") +} + +fn draw_hash(words: &[u32]) -> String { + fnv1a64(words.iter().flat_map(|word| word.to_le_bytes())) +} + +fn phase_state_bytes(stats: &[PhaseStats]) -> Result, String> { + serde_json::to_vec(stats).map_err(|error| format!("serializing damage state: {error}")) +} + +fn validate_stats(stats: &[PhaseStats]) -> Result<(), String> { + let stat = |name: &str| { + stats + .iter() + .find(|item| item.name == name) + .expect("phase stats contain all validated phases") + }; + for item in stats { + if item.frames == 0 { + return Err(format!("damage phase {} did not execute", item.name)); + } + } + if stat("single-small").max_regions != 1 { + return Err("single-small did not stay within one damage region".to_string()); + } + if stat("corner-touch").max_regions != 1 { + return Err("corner-touch regions were not merged".to_string()); + } + if stat("overlap").max_regions != 1 { + return Err("overlap regions were not merged".to_string()); + } + if stat("eight-sparse").max_regions != DEFAULT_DAMAGE_REGIONS { + return Err("eight-sparse did not exercise all damage slots".to_string()); + } + let structural = stat("structural"); + if structural.full_redraw_frames + 1 < structural.frames { + return Err("structural did not force full redraws".to_string()); + } + let texture = stat("texture-in-place"); + if texture.full_redraw_frames != texture.frames { + return Err("texture-in-place did not invalidate every retained frame".to_string()); + } + let settle = stat("settle"); + if settle.empty_frames + 1 < settle.frames { + return Err("settle did not converge to empty damage".to_string()); + } + Ok(()) +} + +fn run_correctness(scenario: &Scenario) -> Result<(), String> { + let mut fixture = Fixture::new()?; + let mut trace = Sha256::new(); + let mut stats = scenario + .phases + .iter() + .map(|phase| PhaseStats::new(&phase.name)) + .collect::>(); + let mut checkpoints = BTreeMap::>::new(); + + for frame in 0..scenario.frames { + let (phase, local_frame) = phase_for_frame(scenario, frame); + let plan = fixture.render_frame(&phase.name, local_frame)?; + fixture.verify_full_render()?; + stats + .iter_mut() + .find(|item| item.name == phase.name) + .expect("phase stat exists") + .observe(&plan); + let frame_hash = Sha256::digest(&fixture.framebuffer); + trace.update(frame_hash); + let framebuffer_hash = hex_digest(frame_hash.as_slice()); + let current_draw_hash = draw_hash(&fixture.words); + if let Some(checkpoint) = scenario.checkpoints.iter().find(|item| item.frame == frame) { + let mut captures = BTreeMap::new(); + for capture in &checkpoint.capture { + match capture.as_str() { + "framebuffer" => { + captures.insert("framebuffer".to_string(), framebuffer_hash.clone()); + } + "drawList" => { + captures.insert("drawList".to_string(), current_draw_hash.clone()); + } + "effects" => { + captures.insert("effects".to_string(), sha256(b"[]")); + } + "state" => {} + _ => unreachable!("checkpoint captures were validated"), + } + } + checkpoints.insert(frame.to_string(), captures); + } + } + + validate_stats(&stats)?; + let state_bytes = phase_state_bytes(&stats)?; + let state_hash = sha256(&state_bytes); + if let Some(final_checkpoint) = checkpoints.get_mut(&(scenario.frames - 1).to_string()) { + if scenario + .checkpoints + .iter() + .find(|item| item.frame == scenario.frames - 1) + .is_some_and(|item| item.capture.iter().any(|capture| capture == "state")) + { + final_checkpoint.insert("state".to_string(), state_hash.clone()); + } + } + let record = CorrectnessRecord { + schema_version: 1, + event: "correctness", + scenario_id: scenario.id.clone(), + framebuffer_trace_hash: hex_digest(trace.finalize().as_slice()), + final_framebuffer_hash: sha256(&fixture.framebuffer), + draw_list_hash: draw_hash(&fixture.words), + state_hash, + effect_hash: sha256(b"[]"), + checkpoints, + phase_stats: stats, + }; + println!( + "{DAMAGE_PREFIX}{}", + serde_json::to_string(&record).map_err(|error| error.to_string())? + ); + Ok(()) +} + +fn run_measurement(scenario: &Scenario) -> Result<(), String> { + let boot_started = Instant::now(); + let mut fixture = Fixture::new()?; + let boot_wall_time_ns = u64::try_from(boot_started.elapsed().as_nanos()) + .map_err(|_| "native boot time overflowed u64".to_string())?; + let mut timings = Vec::with_capacity(scenario.phases.len()); + for phase in &scenario.phases { + let started = Instant::now(); + for frame in phase.start_frame..phase.end_frame { + fixture.render_frame(&phase.name, frame - phase.start_frame)?; + } + let wall_time_ns = u64::try_from(started.elapsed().as_nanos()) + .map_err(|_| format!("phase {} time overflowed u64", phase.name))?; + timings.push(PhaseTiming { + name: phase.name.clone(), + start_frame: phase.start_frame, + end_frame: phase.end_frame, + wall_time_ns, + }); + } + fixture.verify_full_render()?; + let record = MeasurementRecord { + schema_version: 1, + event: "measurement", + scenario_id: scenario.id.clone(), + boot_wall_time_ns, + phases: timings, + final_framebuffer_hash: sha256(&fixture.framebuffer), + final_draw_list_hash: draw_hash(&fixture.words), + }; + println!( + "{DAMAGE_PREFIX}{}", + serde_json::to_string(&record).map_err(|error| error.to_string())? + ); + Ok(()) +} + +fn run_markers(scenario: &Scenario) -> Result<(), String> { + let mut fixture = Fixture::new()?; + for phase in &scenario.phases { + let id = phase_id(&scenario.id, &phase.name); + reset_allocation_phase(); + marker(MARKER_BEGIN, id)?; + for frame in phase.start_frame..phase.end_frame { + fixture.render_frame(&phase.name, frame - phase.start_frame)?; + } + let allocation = allocation_snapshot(); + marker(MARKER_END, id)?; + let record = GuestPhaseRecord { + schema_version: 1, + event: "phase", + scenario_id: scenario.id.clone(), + phase: phase.name.clone(), + phase_id: id, + iteration: 0, + alloc_calls: allocation.calls, + allocated_bytes: allocation.bytes, + current_bytes: allocation.current, + peak_bytes: allocation.peak.saturating_sub(allocation.baseline), + // This core-only fixture constructs no QuickJS runtime. + quickjs_live_bytes_after_gc: 0, + draw_list_hash: draw_hash(&fixture.words), + }; + println!( + "{GUEST_PREFIX}{}", + serde_json::to_string(&record).map_err(|error| error.to_string())? + ); + } + fixture.verify_full_render()?; + let final_draw_list_hash = draw_hash(&fixture.words); + let final_state_hash = fnv1a64( + fixture + .framebuffer + .iter() + .copied() + .chain(fixture.words.iter().flat_map(|word| word.to_le_bytes())), + ); + let record = GuestCompleteRecord { + schema_version: 1, + event: "complete", + scenario_id: scenario.id.clone(), + suite: scenario.suite.clone(), + framework: scenario.subject.framework.clone(), + final_draw_list_hash, + final_state_hash, + effect_hash: fnv1a64(std::iter::empty()), + }; + println!( + "{GUEST_PREFIX}{}", + serde_json::to_string(&record).map_err(|error| error.to_string())? + ); + Ok(()) +} + +fn main() { + let result = (|| { + let args = parse_args()?; + let scenario = read_scenario(&args.scenario)?; + match args.mode { + Mode::Correctness => run_correctness(&scenario), + Mode::Measurement => run_measurement(&scenario), + Mode::Markers => run_markers(&scenario), + } + })(); + if let Err(error) = result { + eprintln!("pocketjs-perf-damage: {error}"); + std::process::exit(1); + } +} diff --git a/tools/perf/executors/damage.ts b/tools/perf/executors/damage.ts new file mode 100644 index 00000000..88dd44b5 --- /dev/null +++ b/tools/perf/executors/damage.ts @@ -0,0 +1,392 @@ +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { ScenarioV1 } from "../core/index.ts"; +import type { NativeOkResult, NativeRunResult } from "../runner/native.ts"; + +export const DAMAGE_FIXTURE_PACKAGE = "pocketjs-perf-damage"; +export const DAMAGE_FIXTURE_BINARY = "pocketjs-perf-damage"; +export const DAMAGE_OUTPUT_PREFIX = "POCKETJS_PERF_DAMAGE "; + +const DEFAULT_HARNESS_ROOT = resolve(fileURLToPath(new URL("../../..", import.meta.url))); +const SHA256 = /^[a-f0-9]{64}$/; +const FNV1A64 = /^fnv1a64:[a-f0-9]{16}$/; + +export type DamageFixtureMode = "correctness" | "measurement" | "markers"; + +export interface MaterializeDamageFixtureOptions { + /** Revision whose engine/core implementation is under test. */ + readonly sourceRoot: string; + /** Revision containing this executor and fixture source. */ + readonly harnessRoot?: string; + /** Existing or new staging directory. */ + readonly destination: string; + /** Core path as seen by the compiler (for example /source in Docker). */ + readonly dependencyRoot?: string; +} + +export interface MaterializedDamageFixture { + readonly root: string; + readonly manifestPath: string; + readonly packageName: typeof DAMAGE_FIXTURE_PACKAGE; + readonly binaryName: typeof DAMAGE_FIXTURE_BINARY; +} + +export interface DamageCorrectnessRecordV1 { + readonly schemaVersion: 1; + readonly event: "correctness"; + readonly scenarioId: string; + readonly framebufferTraceHash: string; + readonly finalFramebufferHash: string; + readonly drawListHash: string; + readonly stateHash: string; + readonly effectHash: string; + readonly checkpoints: Readonly>>>; + readonly phaseStats: readonly { + readonly name: string; + readonly frames: number; + readonly fullRedrawFrames: number; + readonly emptyFrames: number; + readonly maxRegions: number; + readonly totalDamageArea: number; + }[]; +} + +export interface DamageMeasurementRecordV1 { + readonly schemaVersion: 1; + readonly event: "measurement"; + readonly scenarioId: string; + readonly bootWallTimeNs: number; + readonly phases: readonly { + readonly name: string; + readonly startFrame: number; + readonly endFrame: number; + readonly wallTimeNs: number; + }[]; + readonly finalFramebufferHash: string; + readonly finalDrawListHash: string; +} + +export interface RunNativeDamageOptions { + readonly sourceRoot: string; + readonly harnessRoot?: string; + readonly outDir?: string; + readonly cargoPath?: string; +} + +/** The discriminator shared by Native and QEMU suite dispatch. */ +export function isDamageScenario(scenario: ScenarioV1): boolean { + return scenario.subject.family === "core-lab" && + scenario.subject.framework === "core" && + scenario.executorRequirements.includes("fixture.core.damage"); +} + +/** + * Stage the current harness around a possibly older core checkout. The + * generated manifest is the important isolation boundary: a baseline never + * accidentally links the candidate's pocketjs-core. + */ +export function materializeDamageFixture( + options: MaterializeDamageFixtureOptions, +): MaterializedDamageFixture { + const sourceRoot = resolve(options.sourceRoot); + const harnessRoot = resolve(options.harnessRoot ?? DEFAULT_HARNESS_ROOT); + const destination = resolve(options.destination); + const fixtureRoot = join(harnessRoot, "tools", "perf", "damage-fixture"); + const fixtureSource = join(fixtureRoot, "src", "main.rs"); + const fixtureLock = join(fixtureRoot, "Cargo.lock"); + if (!existsSync(fixtureSource) || !existsSync(fixtureLock)) { + throw new Error(`damage fixture source is incomplete under ${fixtureRoot}`); + } + const dependencyRoot = options.dependencyRoot ?? sourceRoot; + const dependency = join(dependencyRoot, "engine", "core"); + if (options.dependencyRoot === undefined && !existsSync(join(dependency, "Cargo.toml"))) { + throw new Error(`source root has no engine/core/Cargo.toml: ${sourceRoot}`); + } + + mkdirSync(join(destination, "src"), { recursive: true }); + copyFileSync(fixtureSource, join(destination, "src", "main.rs")); + copyFileSync(fixtureLock, join(destination, "Cargo.lock")); + writeFileSync(join(destination, "Cargo.toml"), stagedManifest(dependency)); + return { + root: destination, + manifestPath: join(destination, "Cargo.toml"), + packageName: DAMAGE_FIXTURE_PACKAGE, + binaryName: DAMAGE_FIXTURE_BINARY, + }; +} + +export function damageFixtureArgs( + scenarioPath: string, + mode: DamageFixtureMode, +): readonly string[] { + return ["--scenario", scenarioPath, `--${mode}`]; +} + +/** Parse the correctness replay without trusting unrelated process output. */ +export function parseDamageCorrectnessOutput(output: string): DamageCorrectnessRecordV1 { + const value = parseDamageRecord(output, "correctness"); + string(value.scenarioId, "scenarioId"); + sha(value.framebufferTraceHash, "framebufferTraceHash"); + sha(value.finalFramebufferHash, "finalFramebufferHash"); + fnv(value.drawListHash, "drawListHash"); + sha(value.stateHash, "stateHash"); + sha(value.effectHash, "effectHash"); + if (!plainRecord(value.checkpoints)) throw new Error("damage checkpoints must be an object"); + for (const [frame, captures] of Object.entries(value.checkpoints)) { + if (!/^(0|[1-9][0-9]*)$/.test(frame) || !plainRecord(captures)) { + throw new Error(`damage checkpoint ${frame} is invalid`); + } + for (const [capture, digest] of Object.entries(captures)) { + if (capture === "drawList") fnv(digest, `checkpoints.${frame}.${capture}`); + else if (["framebuffer", "state", "effects"].includes(capture)) { + sha(digest, `checkpoints.${frame}.${capture}`); + } else { + throw new Error(`damage checkpoint capture ${capture} is unknown`); + } + } + } + if (!Array.isArray(value.phaseStats) || value.phaseStats.length !== 8) { + throw new Error("damage correctness must report eight phaseStats"); + } + for (const [index, entry] of value.phaseStats.entries()) { + if (!plainRecord(entry)) throw new Error(`damage phaseStats[${index}] must be an object`); + string(entry.name, `phaseStats[${index}].name`); + for (const key of ["frames", "fullRedrawFrames", "emptyFrames", "maxRegions", "totalDamageArea"]) { + uint(entry[key], `phaseStats[${index}].${key}`); + } + } + return value as unknown as DamageCorrectnessRecordV1; +} + +export function parseDamageMeasurementOutput(output: string): DamageMeasurementRecordV1 { + const value = parseDamageRecord(output, "measurement"); + string(value.scenarioId, "scenarioId"); + uint(value.bootWallTimeNs, "bootWallTimeNs"); + sha(value.finalFramebufferHash, "finalFramebufferHash"); + fnv(value.finalDrawListHash, "finalDrawListHash"); + if (!Array.isArray(value.phases) || value.phases.length !== 8) { + throw new Error("damage measurement must report eight phases"); + } + for (const [index, entry] of value.phases.entries()) { + if (!plainRecord(entry)) throw new Error(`damage phases[${index}] must be an object`); + string(entry.name, `phases[${index}].name`); + for (const key of ["startFrame", "endFrame", "wallTimeNs"]) { + uint(entry[key], `phases[${index}].${key}`); + } + if ((entry.endFrame as number) <= (entry.startFrame as number)) { + throw new Error(`damage phases[${index}] has an empty range`); + } + } + return value as unknown as DamageMeasurementRecordV1; +} + +/** Execute the core-only correctness and minimally observed measurement replays. */ +export async function runNativeDamageScenario( + scenario: ScenarioV1, + options: RunNativeDamageOptions, +): Promise { + if (!isDamageScenario(scenario)) { + return { + schemaVersion: 1, + kind: "pocketjs.perf.native-result", + status: "unsupported", + scenarioId: scenario.id, + executor: "native", + reasons: ["scenario is not the fixture.core.damage core-lab subject"], + }; + } + const sourceRoot = resolve(options.sourceRoot); + const staging = options.outDir + ? join(resolve(options.outDir), "damage-fixture") + : mkdtempSync(join(tmpdir(), "pocketjs-perf-damage-")); + const materialized = materializeDamageFixture({ + sourceRoot, + harnessRoot: options.harnessRoot, + destination: staging, + }); + const scenarioPath = join(staging, "scenario.json"); + writeFileSync(scenarioPath, `${JSON.stringify(scenario, null, 2)}\n`); + const targetDir = join(staging, "target"); + const cargo = options.cargoPath ?? "cargo"; + const build = Bun.spawnSync([ + cargo, + "build", + "--release", + "--locked", + "--manifest-path", + materialized.manifestPath, + "--target-dir", + targetDir, + ], { cwd: staging, stdout: "pipe", stderr: "pipe" }); + if (build.exitCode !== 0) { + throw commandFailure("building native damage fixture", build.exitCode, build.stderr); + } + const binary = join(targetDir, "release", DAMAGE_FIXTURE_BINARY); + if (!existsSync(binary) || !statSync(binary).isFile()) { + throw new Error(`cargo did not produce damage fixture binary: ${binary}`); + } + const correctness = parseDamageCorrectnessOutput( + runFixture(binary, damageFixtureArgs(scenarioPath, "correctness"), staging), + ); + const measurement = parseDamageMeasurementOutput( + runFixture(binary, damageFixtureArgs(scenarioPath, "measurement"), staging), + ); + if (correctness.scenarioId !== scenario.id || measurement.scenarioId !== scenario.id) { + throw new Error("damage fixture returned a mismatched scenarioId"); + } + if (correctness.finalFramebufferHash !== measurement.finalFramebufferHash) { + throw new Error("damage correctness and measurement framebuffers diverged"); + } + if (correctness.drawListHash !== measurement.finalDrawListHash) { + throw new Error("damage correctness and measurement DrawLists diverged"); + } + const expectedPhases = scenario.phases.filter((phase) => phase.collect); + if (measurement.phases.length !== expectedPhases.length || measurement.phases.some((phase, index) => { + const expected = expectedPhases[index]; + return !expected || phase.name !== expected.name || phase.startFrame !== expected.startFrame || + phase.endFrame !== expected.endFrame; + })) { + throw new Error("damage measurement phase sequence differs from the scenario"); + } + + const diagnosticMetrics: Record = { + "native.boot_wall_time_ns": { value: measurement.bootWallTimeNs, unit: "ns" }, + "native.measured_frames": { + value: expectedPhases.reduce((sum, phase) => sum + phase.endFrame - phase.startFrame, 0), + unit: "count", + }, + }; + for (const phase of measurement.phases) { + diagnosticMetrics[`native.phase.${phase.name}.wall_time_ns`] = { + value: phase.wallTimeNs, + unit: "ns", + }; + } + for (const phase of correctness.phaseStats) { + diagnosticMetrics[`native.damage.${phase.name}.full_redraw_frames`] = { + value: phase.fullRedrawFrames, + unit: "count", + }; + diagnosticMetrics[`native.damage.${phase.name}.empty_frames`] = { + value: phase.emptyFrames, + unit: "count", + }; + diagnosticMetrics[`native.damage.${phase.name}.max_regions`] = { + value: phase.maxRegions, + unit: "count", + }; + } + diagnosticMetrics["native.wall_time_ns"] = { + value: measurement.phases.reduce((sum, phase) => sum + phase.wallTimeNs, 0), + unit: "ns", + }; + const requestedGateMetrics = Array.isArray(scenario.params.gateMetrics) + ? scenario.params.gateMetrics.filter((metric): metric is string => typeof metric === "string") + : []; + const result: NativeOkResult = { + schemaVersion: 1, + kind: "pocketjs.perf.native-result", + status: "ok", + scenarioId: scenario.id, + executor: "native", + sourceRoot, + correctness: { + framebufferTraceHash: correctness.framebufferTraceHash, + finalFramebufferHash: correctness.finalFramebufferHash, + drawListHash: correctness.drawListHash, + stateHash: correctness.stateHash, + effectHash: correctness.effectHash, + checkpoints: correctness.checkpoints, + }, + measurement: { + bootWallTimeNs: measurement.bootWallTimeNs, + phases: measurement.phases, + finalFramebufferHash: measurement.finalFramebufferHash, + finalDrawListHash: measurement.finalDrawListHash, + }, + diagnosticMetrics, + exactMetrics: {}, + unsupportedMetrics: [...new Set(requestedGateMetrics)], + }; + if (options.outDir) { + mkdirSync(resolve(options.outDir), { recursive: true }); + const safeId = scenario.id.replace(/[^a-zA-Z0-9._-]+/g, "-"); + writeFileSync( + join(resolve(options.outDir), `${safeId}.native.json`), + `${JSON.stringify(result, null, 2)}\n`, + ); + } + return result; +} + +function stagedManifest(corePath: string): string { + return `[workspace]\n\n` + + `[package]\n` + + `name = ${JSON.stringify(DAMAGE_FIXTURE_PACKAGE)}\n` + + `version = "0.1.0"\n` + + `edition = "2024"\n` + + `publish = false\n\n` + + `[dependencies]\n` + + `libc = "0.2"\n` + + `pocketjs-core = { path = ${JSON.stringify(corePath)}, features = ["std"] }\n` + + `serde = { version = "1", features = ["derive"] }\n` + + `serde_json = "1"\n` + + `sha2 = "0.10"\n\n` + + `[profile.release]\n` + + `opt-level = 3\n` + + `lto = "thin"\n` + + `codegen-units = 1\n` + + `panic = "abort"\n` + + `strip = "debuginfo"\n`; +} + +function runFixture(binary: string, args: readonly string[], cwd: string): string { + const result = Bun.spawnSync([binary, ...args], { cwd, stdout: "pipe", stderr: "pipe" }); + if (result.exitCode !== 0) { + throw commandFailure(`running ${basename(binary)}`, result.exitCode, result.stderr); + } + return new TextDecoder().decode(result.stdout); +} + +function commandFailure(action: string, exitCode: number, stderr: Uint8Array): Error { + const detail = new TextDecoder().decode(stderr).trim(); + return new Error(`${action} failed (${exitCode})${detail ? `: ${detail}` : ""}`); +} + +function parseDamageRecord(output: string, event: "correctness" | "measurement"): Record { + const records = output.split(/\r?\n/) + .filter((line) => line.startsWith(DAMAGE_OUTPUT_PREFIX)) + .map((line) => JSON.parse(line.slice(DAMAGE_OUTPUT_PREFIX.length)) as unknown); + if (records.length !== 1 || !plainRecord(records[0])) { + throw new Error(`expected exactly one ${event} damage protocol record`); + } + const value = records[0]; + if (value.schemaVersion !== 1 || value.event !== event) { + throw new Error(`invalid damage ${event} protocol envelope`); + } + return value; +} + +function plainRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function string(value: unknown, path: string): asserts value is string { + if (typeof value !== "string" || value.length === 0) throw new Error(`damage ${path} must be a string`); +} + +function uint(value: unknown, path: string): asserts value is number { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + throw new Error(`damage ${path} must be a non-negative safe integer`); + } +} + +function sha(value: unknown, path: string): asserts value is string { + if (typeof value !== "string" || !SHA256.test(value)) throw new Error(`damage ${path} must be SHA-256`); +} + +function fnv(value: unknown, path: string): asserts value is string { + if (typeof value !== "string" || !FNV1A64.test(value)) throw new Error(`damage ${path} must be FNV-1a-64`); +} diff --git a/tools/perf/executors/qemu-worker.ts b/tools/perf/executors/qemu-worker.ts new file mode 100644 index 00000000..6bbb52ea --- /dev/null +++ b/tools/perf/executors/qemu-worker.ts @@ -0,0 +1,29 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { parseScenarioV1 } from "../core/index.ts"; +import { runVaporScenario } from "./vapor.ts"; + +export const QEMU_WORKER_OUTPUT_PREFIX = "POCKETJS_PERF_QEMU_WORKER "; + +function flag(name: string): string { + const index = process.argv.indexOf(name); + const value = index >= 0 ? process.argv[index + 1] : undefined; + if (!value) throw new Error(`${name} is required`); + return value; +} + +const scenario = parseScenarioV1(JSON.parse(readFileSync(resolve(flag("--scenario")), "utf8"))); +const executor = flag("--executor"); +if (executor !== "qemu-armv7-thumb2" && executor !== "qemu-aarch64") { + throw new Error(`unsupported QEMU worker executor ${JSON.stringify(executor)}`); +} +const result = await runVaporScenario({ + scenario, + executor, + sourceRoot: resolve(flag("--source-root")), + harnessRoot: resolve(flag("--harness-root")), + outDir: resolve(flag("--out-dir")), + image: flag("--image"), +}); +console.log(`${QEMU_WORKER_OUTPUT_PREFIX}${JSON.stringify(result)}`); +if (result.status === "invalid") process.exitCode = 2; diff --git a/tools/perf/executors/qemu.ts b/tools/perf/executors/qemu.ts new file mode 100644 index 00000000..8f651b38 --- /dev/null +++ b/tools/perf/executors/qemu.ts @@ -0,0 +1,1263 @@ +import { createHash } from "node:crypto"; +import { + copyFileSync, + cpSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { basename, join, relative, resolve } from "node:path"; +import { + artifactBuildVariantKey, + buildRenderConfig, + parseReceiptV1, + rgbaFramebufferByteLength, + type ReceiptV1, + type ScenarioV1, +} from "../core/index.ts"; +import { + createQemuReceipts, + parseGuestOutput, + sha256Json, + type ArtifactMetrics, + type GuestProtocolResult, + type QemuTarget, + type ReceiptEnvironmentV1, +} from "../receipts/index.ts"; +import { NATIVE_RUN_OUTPUT_PREFIX, parseNativeResult } from "../receipts/native-protocol.ts"; +import { estimatedSuiteSeconds, expandSuiteFrameworks, loadScenarioSuite } from "../runner/suite.ts"; +import type { NativeOkResult, NativeRunResult } from "../runner/native.ts"; +import { nativeProvenance } from "../cli/receipts.ts"; +import { runCommand } from "../cli/process.ts"; +import type { QemuBridgeOptions } from "../cli/types.ts"; +import { + damageFixtureArgs, + isDamageScenario, + materializeDamageFixture, + parseDamageCorrectnessOutput, + type DamageCorrectnessRecordV1, +} from "./damage.ts"; +import type { VaporQemuResult } from "./vapor.ts"; + +const DEFAULT_IMAGE = "pocketjs-perf-qemu:11.0.3"; +const QEMU_PLUGIN = "/opt/pocketjs-perf-qemu/build/pocketjs-perf-counter.so"; +const CARGO_REGISTRY_VOLUME = "pocketjs-perf-cargo-registry-v1"; +const QEMU_WORKER_OUTPUT_PREFIX = "POCKETJS_PERF_QEMU_WORKER "; +export const QEMU_ENTROPY_PROFILE = "seed-1+guest-shim-v1"; + +const ADAPTER_CAPABILITIES: Readonly>> = Object.freeze({ + "guest-app": new Set([ + "guest.frame", + "core.ui", + "renderer.framebuffer", + "assets.pak", + "input.buttons", + "input.analog", + "input.touch", + "correctness.framebuffer", + "correctness.draw-list", + "correctness.effects", + "correctness.state-final", + ]), + "core-lab": new Set([ + "fixture.core.damage", + "correctness.framebuffer", + "correctness.draw-list", + ]), + vapor: new Set([ + "fixture.vapor.generated-c", + "input.buttons", + "input.relative-axis", + "correctness.framebuffer", + "correctness.draw-list", + "correctness.effects", + "correctness.state-final", + ]), +}); + +interface TargetSpec { + readonly executor: QemuBridgeOptions["executor"]; + readonly rustTarget: string; + readonly qemuTarget: QemuTarget; + readonly compiler: string; + readonly sizeTool: string; + readonly emulator: string; + readonly cpuArgs: readonly string[]; + readonly emulatorArgs: readonly string[]; + readonly sysroot: string; + readonly rustFlags: readonly string[]; + readonly cFlags: readonly string[]; + readonly cargoEnvironment: Readonly>; +} + +const TARGETS: Readonly> = { + "qemu-armv7-thumb2": { + executor: "qemu-armv7-thumb2", + rustTarget: "armv7-unknown-linux-gnueabihf", + qemuTarget: "arm", + compiler: "arm-linux-gnueabihf-gcc", + sizeTool: "arm-linux-gnueabihf-size", + emulator: "/opt/qemu/bin/qemu-arm", + cpuArgs: ["-cpu", "cortex-a9,neon=off,vfp-d32=off"], + emulatorArgs: ["-seed", "1"], + sysroot: "/usr/arm-linux-gnueabihf", + rustFlags: ["-C", "target-feature=+thumb-mode"], + cFlags: ["-mthumb", "-march=armv7-a", "-mfpu=vfpv3-d16", "-mfloat-abi=hard"], + cargoEnvironment: { + CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER: "arm-linux-gnueabihf-gcc", + CC_armv7_unknown_linux_gnueabihf: "arm-linux-gnueabihf-gcc", + CFLAGS_armv7_unknown_linux_gnueabihf: + "-mthumb -march=armv7-a -mfpu=vfpv3-d16 -mfloat-abi=hard", + RUSTFLAGS: "-C target-feature=+thumb-mode", + }, + }, + "qemu-aarch64": { + executor: "qemu-aarch64", + rustTarget: "aarch64-unknown-linux-gnu", + qemuTarget: "aarch64", + compiler: "aarch64-linux-gnu-gcc", + sizeTool: "aarch64-linux-gnu-size", + emulator: "/opt/qemu/bin/qemu-aarch64", + cpuArgs: ["-cpu", "cortex-a53"], + emulatorArgs: ["-seed", "1"], + sysroot: "/usr/aarch64-linux-gnu", + rustFlags: [], + cFlags: ["-march=armv8-a"], + cargoEnvironment: { + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: "aarch64-linux-gnu-gcc", + CC_aarch64_unknown_linux_gnu: "aarch64-linux-gnu-gcc", + CFLAGS_aarch64_unknown_linux_gnu: "-march=armv8-a", + }, + }, +}; + +export function qemuInvocationProfile(executor: QemuBridgeOptions["executor"]): { + readonly cpuArgs: readonly string[]; + readonly emulatorArgs: readonly string[]; + readonly entropyProfile: typeof QEMU_ENTROPY_PROFILE; +} { + const target = TARGETS[executor]; + return { + cpuArgs: [...target.cpuArgs], + emulatorArgs: [...target.emulatorArgs], + entropyProfile: QEMU_ENTROPY_PROFILE, + }; +} + +interface CommandOutput { + readonly exitCode: number; + readonly stdout: string; + readonly stderr: string; + readonly combined: string; +} + +interface SuiteContext { + readonly options: QemuBridgeOptions; + readonly sourceRoot: string; + readonly harnessRoot: string; + readonly outDir: string; + readonly workDir: string; + readonly image: string; + readonly imageIdentity: string; + readonly target: TargetSpec; + readonly toolchain: ReceiptEnvironmentV1["toolchain"]; + readonly fingerprint: string; + readonly hostPlatform: string; + readonly hostArch: string; +} + +export interface GuestArtifacts { + readonly bundle: string; + readonly pak: string | null; +} + +interface QemuRunArtifacts { + readonly binary: string; + readonly correctnessOutput: string; + readonly correctnessFramebufferHash: string; + readonly checkpointFramebufferHashes: Readonly>; + readonly correctnessStateHash: string; + readonly correctnessEffectHash: string; + readonly measurementOutput: string; + readonly artifactMetrics: ArtifactMetrics; +} + +export interface QemuSuiteResult { + readonly receipts: readonly ReceiptV1[]; + readonly invalidReasons: readonly string[]; +} + +function text(bytes: Uint8Array): string { + return new TextDecoder().decode(bytes); +} + +function command(argv: readonly string[], cwd: string): CommandOutput { + const result = runCommand(argv, { cwd }); + const stdout = text(result.stdout); + const stderr = text(result.stderr); + return { + exitCode: result.exitCode, + stdout, + stderr, + combined: `${stdout}${stdout && stderr ? "\n" : ""}${stderr}`, + }; +} + +function failure(action: string, result: CommandOutput): Error { + const detail = (result.stderr.trim() || result.stdout.trim()).slice(-4_000); + return new Error(`${action} failed (${result.exitCode})${detail ? `: ${detail}` : ""}`); +} + +function safeName(scenario: ScenarioV1): string { + return `${scenario.id}.${scenario.subject.framework}`.replace(/[^a-zA-Z0-9._-]+/g, "-"); +} + +export function qemuScenarioCapabilityReasons(scenario: ScenarioV1): readonly string[] { + const capabilities = ADAPTER_CAPABILITIES[scenario.subject.family]; + if (!capabilities) { + return [`${scenario.id}: no QEMU capability declaration for ${scenario.subject.family}`]; + } + return scenario.executorRequirements + .filter((requirement) => !capabilities.has(requirement)) + .map((requirement) => ( + `${scenario.id}: ${scenario.subject.family} QEMU adapter does not provide ` + + `executor requirement ${JSON.stringify(requirement)}` + )); +} + +function dockerMount(path: string, target: string, readonly = false): readonly string[] { + const mode = readonly ? ",readonly" : ""; + return ["--mount", `type=bind,source=${realpathSync(path)},target=${target}${mode}`]; +} + +function dockerRun( + context: SuiteContext, + argv: readonly string[], + options: { readonly network?: boolean; readonly environment?: Readonly> } = {}, +): CommandOutput { + const environment = { + LC_ALL: "C", + TZ: "UTC", + ...options.environment, + }; + const dockerArgs = [ + "docker", "run", "--rm", + ...dockerMount(context.sourceRoot, "/source", true), + ...dockerMount(context.workDir, "/work"), + ...dockerMount(context.outDir, "/output"), + "--workdir", "/work", + ...(options.network === false ? ["--network", "none"] : []), + ...Object.entries(environment).flatMap(([name, value]) => ["--env", `${name}=${value}`]), + context.image, + ...argv, + ]; + return command(dockerArgs, context.outDir); +} + +function inspectImage(image: string, cwd: string): string { + const result = command([ + "docker", "image", "inspect", image, + "--format", "{{.Id}} {{.Os}}/{{.Architecture}} {{json .RepoDigests}}", + ], cwd); + if (result.exitCode !== 0) { + throw new Error( + `pinned QEMU image ${image} is unavailable; run tools/perf/qemu/docker.sh build`, + ); + } + const identity = result.stdout.trim(); + if (!/^sha256:[a-f0-9]{64}\s+linux\/(?:amd64|arm64)\s+/.test(identity)) { + throw new Error(`cannot identify pinned QEMU image ${image}: ${JSON.stringify(identity)}`); + } + return identity; +} + +function containerVersion(image: string, cwd: string, executable: string, ...args: string[]): string { + const result = command(["docker", "run", "--rm", "--entrypoint", executable, image, ...args], cwd); + if (result.exitCode !== 0) throw failure(`reading ${executable} version`, result); + return (result.stdout || result.stderr).split(/\r?\n/, 1)[0]!.trim(); +} + +export function qemuHarnessFingerprint( + harnessRoot: string, + imageIdentity: string, + executor: QemuBridgeOptions["executor"], + bunVersion: string, + hostPlatform: string = process.platform, + hostArch: string = process.arch, +): string { + const target = TARGETS[executor]; + const hash = createHash("sha256"); + hash.update("pocketjs.perf.qemu-executor.v1\0").update(imageIdentity).update("\0"); + hash.update(JSON.stringify({ + executor: target.executor, + target: target.rustTarget, + rustFlags: target.rustFlags, + cFlags: target.cFlags, + qemuCpuArgs: target.cpuArgs, + qemuEmulatorArgs: target.emulatorArgs, + entropyProfile: QEMU_ENTROPY_PROFILE, + markerPlugin: QEMU_PLUGIN, + hostBun: bunVersion, + hostPlatform, + hostArch, + })).update("\0"); + for (const relativePath of [ + "bun.lock", + "tools/perf/core/render-config.ts", + "tools/perf/executors/qemu.ts", + "tools/perf/executors/qemu-worker.ts", + "tools/perf/executors/damage.ts", + "tools/perf/executors/vapor.ts", + "tools/perf/runner/native.ts", + "tools/perf/runner/native-cli.ts", + "tools/perf/runner/native-world.ts", + "tools/perf/runner/input.ts", + "tools/perf/apps/idle-fixture-main.tsx", + "tools/perf/apps/list-fixture-main.tsx", + "tools/perf/apps/keyed-list-model.ts", + "tools/perf/guest/Cargo.toml", + "tools/perf/guest/src/main.rs", + "tools/perf/damage-fixture/Cargo.toml", + "tools/perf/damage-fixture/Cargo.lock", + "tools/perf/damage-fixture/src/main.rs", + "tools/perf/qemu/guest_marker.h", + "tools/perf/qemu/perf_counter.c", + "tools/perf/receipts/protocol.ts", + "tools/perf/receipts/factory.ts", + "tools/perf/receipts/hash.ts", + "tools/perf/receipts/native-protocol.ts", + "hosts/web/wasm-ops.js", + "framework/src/touch.ts", + ]) { + const path = join(harnessRoot, relativePath); + if (!existsSync(path)) continue; + const bytes = readFileSync(path); + hash.update(relativePath).update("\0").update(String(bytes.byteLength)).update("\0").update(bytes); + } + return hash.digest("hex"); +} + +function makeContext(options: QemuBridgeOptions): SuiteContext { + const sourceRoot = realpathSync(resolve(options.sourceRoot)); + const harnessRoot = realpathSync(resolve(options.harnessRoot)); + const outDir = resolve(options.outDir); + mkdirSync(outDir, { recursive: true }); + const workDir = mkdtempSync(join(outDir, ".qemu-work-")); + const image = process.env.POCKETJS_QEMU_IMAGE || DEFAULT_IMAGE; + try { + const imageIdentity = inspectImage(image, outDir); + const target = TARGETS[options.executor]; + const rustc = containerVersion(image, outDir, "rustc", "--version"); + const cCompiler = containerVersion(image, outDir, target.compiler, "--version"); + const qemu = containerVersion(image, outDir, target.emulator, "--version"); + if (!/\bversion 11\.0\.3\b/.test(qemu)) { + throw new Error(`expected QEMU 11.0.3, got ${JSON.stringify(qemu)}`); + } + const rustSysroot = containerVersion(image, outDir, "rustc", "--print", "sysroot"); + const bunVersion = `Bun ${Bun.version}`; + return { + options, + sourceRoot, + harnessRoot, + outDir, + workDir, + image, + imageIdentity, + target, + toolchain: { + rustc, + cCompiler, + sysroot: `rust=${rustSysroot};guest=${target.sysroot}`, + qemu, + bun: bunVersion, + }, + fingerprint: qemuHarnessFingerprint( + harnessRoot, + imageIdentity, + options.executor, + bunVersion, + process.platform, + process.arch, + ), + hostPlatform: process.platform, + hostArch: process.arch, + }; + } catch (error) { + cleanupWorkDirectory(outDir, workDir, image); + throw error; + } +} + +export function qemuCleanupFallbackArgs(image: string, workDir: string): readonly string[] { + return [ + "docker", "run", "--rm", + "--network", "none", + "--read-only", + "--cap-drop", "ALL", + "--cap-add", "DAC_OVERRIDE", + "--security-opt", "no-new-privileges", + ...dockerMount(workDir, "/work"), + "--entrypoint", "find", + image, + "/work", "-mindepth", "1", "-delete", + ]; +} + +function isPermissionError(error: unknown): boolean { + if (typeof error !== "object" || error === null || !("code" in error)) return false; + return error.code === "EACCES" || error.code === "EPERM"; +} + +function cleanupWorkDirectory(outDir: string, workDir: string, image: string): void { + if (!existsSync(workDir)) return; + const resolvedWork = realpathSync(workDir); + if (resolve(resolvedWork, "..") !== realpathSync(outDir) || + !basename(resolvedWork).startsWith(".qemu-work-")) { + throw new Error(`refusing to clean unexpected QEMU work directory ${resolvedWork}`); + } + try { + rmSync(resolvedWork, { recursive: true, force: true }); + return; + } catch (error) { + if (!isPermissionError(error) || !existsSync(resolvedWork)) throw error; + } + + // Docker builds run as container root so Cargo target directories can be + // unreadable to an unprivileged Linux host. Limit the privileged fallback to + // the validated disposable bind mount. DAC_OVERRIDE is required to enter the + // host-owned mode-0700 mkdtemp root; every other capability remains dropped. + const result = command(qemuCleanupFallbackArgs(image, resolvedWork), outDir); + if (result.exitCode !== 0) throw failure("cleaning the QEMU work directory", result); + rmSync(resolvedWork, { recursive: true, force: true }); +} + +function quickJsLockTuples(lockPath: string): readonly string[] { + if (!existsSync(lockPath)) throw new Error(`source root has no engine/Cargo.lock: ${lockPath}`); + const wanted = new Set(["rquickjs", "rquickjs-core", "rquickjs-sys"]); + const tuples: string[] = []; + for (const block of readFileSync(lockPath, "utf8").split("[[package]]").slice(1)) { + const name = /^\s*name\s*=\s*"([^"]+)"/m.exec(block)?.[1]; + if (!name || !wanted.has(name)) continue; + const version = /^\s*version\s*=\s*"([^"]+)"/m.exec(block)?.[1]; + const source = /^\s*source\s*=\s*"([^"]+)"/m.exec(block)?.[1]; + const checksum = /^\s*checksum\s*=\s*"([^"]+)"/m.exec(block)?.[1]; + if (!version || !source || !checksum) throw new Error(`${lockPath}: incomplete ${name} lock entry`); + tuples.push(`${name}@${version}\0${source}\0${checksum}`); + } + tuples.sort(); + if (tuples.length !== wanted.size || new Set(tuples.map((tuple) => tuple.split("@", 1)[0])).size !== wanted.size) { + throw new Error(`${lockPath}: expected one locked rquickjs, rquickjs-core, and rquickjs-sys tuple`); + } + return tuples; +} + +function cargoDockerArgs( + context: SuiteContext, + manifest: string, + targetDir: string, +): readonly string[] { + const manifestInWork = `/work/${relative(context.workDir, manifest)}`; + const targetInWork = `/work/${relative(context.workDir, targetDir)}`; + return [ + "docker", "run", "--rm", + ...dockerMount(context.sourceRoot, "/source", true), + ...dockerMount(context.workDir, "/work"), + "--mount", `type=volume,source=${CARGO_REGISTRY_VOLUME},target=/opt/rust/cargo/registry`, + "--workdir", "/work", + "--env", "LC_ALL=C", + "--env", "TZ=UTC", + ...Object.entries(context.target.cargoEnvironment) + .flatMap(([name, value]) => ["--env", `${name}=${value}`]), + context.image, + "cargo", "build", "--release", "--locked", + "--manifest-path", manifestInWork, + "--target-dir", targetInWork, + "--target", context.target.rustTarget, + ]; +} + +function buildCargoFixture( + context: SuiteContext, + manifest: string, + targetDir: string, + binaryName: string, +): string { + mkdirSync(targetDir, { recursive: true }); + const result = command(cargoDockerArgs(context, manifest, targetDir), context.outDir); + if (result.exitCode !== 0) throw failure(`cross-building ${binaryName}`, result); + const binary = join(targetDir, context.target.rustTarget, "release", binaryName); + if (!existsSync(binary) || !statSync(binary).isFile()) { + throw new Error(`cross build did not produce ${binary}`); + } + return binary; +} + +function materializeGuestHarness(context: SuiteContext): { manifest: string; targetDir: string } { + const source = join(context.harnessRoot, "tools/perf/guest"); + const destination = join(context.workDir, "guest"); + mkdirSync(join(destination, "src"), { recursive: true }); + cpSync(join(source, "src"), join(destination, "src"), { recursive: true }); + const sourceLock = readFileSync(join(context.sourceRoot, "engine/Cargo.lock"), "utf8"); + const guestPackage = [ + "", + "[[package]]", + 'name = "pocketjs-perf-guest"', + 'version = "0.1.0"', + "dependencies = [", + ' "anyhow",', + ' "libc",', + ' "pocket-mod",', + ' "pocket-ui-surface",', + ' "pocketjs-core",', + ' "rquickjs",', + ' "serde",', + ' "serde_json",', + "]", + "", + ].join("\n"); + if (sourceLock.includes('name = "pocketjs-perf-guest"')) { + throw new Error("source engine lock unexpectedly already contains pocketjs-perf-guest"); + } + writeFileSync(join(destination, "Cargo.lock"), `${sourceLock.trimEnd()}${guestPackage}`); + let manifest = readFileSync(join(source, "Cargo.toml"), "utf8"); + const quickJsVersion = lockedPackageVersion(sourceLock, "rquickjs"); + const versionedQuickJs = manifest.replace( + /rquickjs = \{ version = "[^"]+", features = \["rust-alloc"\] \}/, + `rquickjs = { version = "=${quickJsVersion}", features = ["rust-alloc"] }`, + ); + if (versionedQuickJs === manifest) { + throw new Error("guest manifest no longer has the expected rquickjs allocator dependency"); + } + manifest = versionedQuickJs; + const replacements: Readonly> = { + "../../../engine/crates/pocket-mod": "/source/engine/crates/pocket-mod", + "../../../engine/crates/pocket-ui-surface": "/source/engine/crates/pocket-ui-surface", + "../../../engine/core": "/source/engine/core", + }; + for (const [from, to] of Object.entries(replacements)) manifest = manifest.replaceAll(from, to); + if (/path\s*=\s*"\.\./.test(manifest)) { + throw new Error("guest manifest contains an unstaged relative dependency"); + } + const manifestPath = join(destination, "Cargo.toml"); + writeFileSync(manifestPath, manifest); + resolveGuestLock(context, manifestPath); + return { manifest: manifestPath, targetDir: join(destination, "target") }; +} + +function lockedPackageVersion(lock: string, packageName: string): string { + const versions: string[] = []; + for (const block of lock.split("[[package]]").slice(1)) { + const name = /^\s*name\s*=\s*"([^"]+)"/m.exec(block)?.[1]; + if (name !== packageName) continue; + const version = /^\s*version\s*=\s*"([^"]+)"/m.exec(block)?.[1]; + if (!version) throw new Error(`source lock has an incomplete ${packageName} entry`); + versions.push(version); + } + if (versions.length !== 1) { + throw new Error(`source lock must contain exactly one ${packageName} version; got ${versions.length}`); + } + return versions[0]!; +} + +function resolveGuestLock(context: SuiteContext, manifestPath: string): void { + const result = command([ + "docker", "run", "--rm", + ...dockerMount(context.sourceRoot, "/source", true), + ...dockerMount(context.workDir, "/work"), + "--mount", `type=volume,source=${CARGO_REGISTRY_VOLUME},target=/opt/rust/cargo/registry`, + "--workdir", "/work", + "--env", "LC_ALL=C", + "--env", "TZ=UTC", + context.image, + "cargo", "metadata", + "--manifest-path", `/work/${relative(context.workDir, manifestPath)}`, + "--format-version", "1", + ], context.outDir); + if (result.exitCode !== 0) throw failure("resolving the source-seeded guest lock", result); + const expected = quickJsLockTuples(join(context.sourceRoot, "engine/Cargo.lock")); + const actual = quickJsLockTuples(join(context.workDir, "guest/Cargo.lock")); + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + throw new Error( + `staged guest QuickJS lock differs from measured source: ` + + `expected ${expected.join(", ")}; got ${actual.join(", ")}`, + ); + } +} + +function buildWasm(sourceRoot: string): void { + const result = command([process.execPath, join(sourceRoot, "tools/wasm.ts")], sourceRoot); + if (result.exitCode !== 0) throw failure("building Native correctness WASM", result); +} + +export function snapshotGuestArtifacts( + sourceBundle: string, + sourcePak: string | null, + workDir: string, + cacheKey: string, +): GuestArtifacts { + const artifactDirectory = join( + workDir, + "artifacts", + createHash("sha256").update(cacheKey).digest("hex"), + ); + mkdirSync(artifactDirectory, { recursive: true }); + const frozenBundle = join(artifactDirectory, "bundle.js"); + copyFileSync(sourceBundle, frozenBundle); + let frozenPak: string | null = null; + if (sourcePak) { + frozenPak = join(artifactDirectory, "bundle.pak"); + copyFileSync(sourcePak, frozenPak); + } + return { bundle: frozenBundle, pak: frozenPak }; +} + +function buildGuestArtifacts(context: SuiteContext, scenario: ScenarioV1): GuestArtifacts { + const sourceRoot = context.sourceRoot; + const renderContract = qemuScenarioRenderContract(scenario); + const result = command([ + process.execPath, + join(sourceRoot, "tools/build.ts"), + scenario.subject.id, + `--framework=${scenario.subject.framework}`, + renderContract.densityArgument, + `--outdir=${join(sourceRoot, "dist")}`, + ], sourceRoot); + if (result.exitCode !== 0) throw failure(`building ${scenario.subject.id}`, result); + const bundle = join(sourceRoot, "dist", `${scenario.subject.entry}.js`); + const pakPath = join(sourceRoot, "dist", `${scenario.subject.entry}.pak`); + if (!existsSync(bundle)) throw new Error(`${scenario.id}: build did not produce ${bundle}`); + const pak = existsSync(pakPath) ? pakPath : null; + if (scenario.executorRequirements.includes("assets.pak") && !pak) { + throw new Error(`${scenario.id}: build did not produce required ${pakPath}`); + } + return snapshotGuestArtifacts( + bundle, + pak, + context.workDir, + renderContract.artifactCacheKey, + ); +} + +function runIsolatedNative( + context: SuiteContext, + scenario: ScenarioV1, + artifacts: GuestArtifacts, + directory: string, +): NativeRunResult { + mkdirSync(directory, { recursive: true }); + const scenarioPath = join(directory, "scenario.json"); + const sourceRoot = join(directory, "source"); + mkdirSync(join(sourceRoot, "hosts/web"), { recursive: true }); + mkdirSync(join(sourceRoot, "dist"), { recursive: true }); + copyFileSync( + join(context.sourceRoot, "hosts/web/pocketjs.wasm"), + join(sourceRoot, "hosts/web/pocketjs.wasm"), + ); + copyFileSync(artifacts.bundle, join(sourceRoot, "dist", `${scenario.subject.entry}.js`)); + if (artifacts.pak) { + copyFileSync(artifacts.pak, join(sourceRoot, "dist", `${scenario.subject.entry}.pak`)); + } + writeFileSync(scenarioPath, `${JSON.stringify(scenario, null, 2)}\n`); + const result = command([ + process.execPath, + join(context.harnessRoot, "tools/perf/runner/native-cli.ts"), + scenarioPath, + "--source-root", sourceRoot, + "--harness-root", context.harnessRoot, + "--out-dir", directory, + ], context.harnessRoot); + const records = result.stdout.split(/\r?\n/) + .filter((line) => line.startsWith(NATIVE_RUN_OUTPUT_PREFIX)); + if (records.length !== 1) throw failure(`${scenario.id}: isolated Native correctness replay`, result); + const parsed = parseNativeResult(JSON.parse(records[0]!.slice(NATIVE_RUN_OUTPUT_PREFIX.length))); + if (parsed.success === false) { + throw new Error(`${scenario.id}: invalid Native replay: ${parsed.reasons.join("; ")}`); + } + if (result.exitCode !== 0 && !(result.exitCode === 2 && parsed.data.status === "unsupported")) { + throw failure(`${scenario.id}: isolated Native correctness replay`, result); + } + return parsed.data; +} + +function containerWorkPath(context: SuiteContext, hostPath: string): string { + const workPath = relative(context.workDir, hostPath); + if (!workPath.startsWith("..")) return `/work/${workPath}`; + const outputPath = relative(context.outDir, hostPath); + if (!outputPath.startsWith("..")) return `/output/${outputPath}`; + throw new Error(`${hostPath} is outside QEMU work and output directories`); +} + +function guestArguments( + context: SuiteContext, + binary: string, + scenarioPath: string, + artifacts: GuestArtifacts, + mode: "correctness" | "measurement", + framebufferPath?: string, + framebufferDir?: string, +): readonly string[] { + const args = [ + context.target.emulator, + ...context.target.cpuArgs, + ...context.target.emulatorArgs, + "-L", context.target.sysroot, + ...(mode === "measurement" + ? ["-d", "plugin", "-plugin", QEMU_PLUGIN] + : []), + containerWorkPath(context, binary), + "--scenario", containerWorkPath(context, scenarioPath), + "--bundle", containerWorkPath(context, artifacts.bundle), + ...(artifacts.pak ? ["--pak", containerWorkPath(context, artifacts.pak)] : []), + ...(mode === "correctness" + ? [ + "--correctness", + ...(framebufferPath + ? ["--framebuffer-out", containerWorkPath(context, framebufferPath)] + : []), + ...(framebufferDir + ? ["--framebuffer-dir", containerWorkPath(context, framebufferDir)] + : []), + ] + : ["--markers"]), + ]; + return args; +} + +function runQuickJsGuest( + context: SuiteContext, + binary: string, + scenario: ScenarioV1, + artifacts: GuestArtifacts, + directory: string, +): QemuRunArtifacts { + mkdirSync(directory, { recursive: true }); + const scenarioPath = join(directory, "scenario.json"); + const framebufferPath = join(directory, "correctness.rgba"); + const framebufferDir = join(directory, "correctness-frames"); + writeFileSync(scenarioPath, `${JSON.stringify(scenario, null, 2)}\n`); + const correctness = dockerRun( + context, + guestArguments( + context, + binary, + scenarioPath, + artifacts, + "correctness", + framebufferPath, + framebufferDir, + ), + { network: false }, + ); + writeFileSync(join(directory, "correctness.log"), correctness.combined); + if (correctness.exitCode !== 0) throw failure(`${scenario.id}: QEMU correctness replay`, correctness); + const expectedFramebufferBytes = framebufferByteLength(scenario); + if (!existsSync(framebufferPath) || statSync(framebufferPath).size !== expectedFramebufferBytes) { + throw new Error( + `${scenario.id}: correctness replay did not emit the expected ` + + `${expectedFramebufferBytes}-byte RGBA framebuffer`, + ); + } + const correctnessFramebufferHash = createHash("sha256") + .update(readFileSync(framebufferPath)) + .digest("hex"); + const checkpointFramebufferHashes: Record = {}; + for (const checkpoint of scenario.checkpoints) { + if (!checkpoint.capture.includes("framebuffer")) continue; + const path = join(framebufferDir, `${checkpoint.frame}.rgba`); + if (!existsSync(path) || statSync(path).size !== expectedFramebufferBytes) { + throw new Error(`${scenario.id}: missing correctness framebuffer checkpoint ${checkpoint.frame}`); + } + checkpointFramebufferHashes[String(checkpoint.frame)] = createHash("sha256") + .update(readFileSync(path)) + .digest("hex"); + } + const stateBytes = readFileSync(join(framebufferDir, "state.json")); + const effectBytes = readFileSync(join(framebufferDir, "effects.json")); + const correctnessStateHash = sha256Json(JSON.parse(text(stateBytes))); + const correctnessEffectHash = sha256Json(JSON.parse(text(effectBytes))); + const measurement = dockerRun( + context, + guestArguments(context, binary, scenarioPath, artifacts, "measurement"), + { network: false }, + ); + writeFileSync(join(directory, "measurement.log"), measurement.combined); + if (measurement.exitCode !== 0) { + throw failure(`${scenario.id}: QEMU measurement replay`, measurement); + } + const metrics: ArtifactMetrics = { + "artifact.bundle_bytes": statSync(artifacts.bundle).size, + ...(artifacts.pak ? { "artifact.pak_bytes": statSync(artifacts.pak).size } : {}), + "artifact.elf_text_rodata_bytes": elfTextRodata(context, binary), + }; + return { + binary, + correctnessOutput: correctness.combined, + correctnessFramebufferHash, + checkpointFramebufferHashes, + correctnessStateHash, + correctnessEffectHash, + measurementOutput: measurement.combined, + artifactMetrics: metrics, + }; +} + +export function qemuScenarioRenderContract(scenario: ScenarioV1): { + readonly artifactCacheKey: string; + readonly densityArgument: string; + readonly framebufferByteLength: number; +} { + const render = buildRenderConfig(scenario.params); + return { + artifactCacheKey: artifactBuildVariantKey(scenario), + densityArgument: `--density=${render.rasterDensity}`, + framebufferByteLength: rgbaFramebufferByteLength(render), + }; +} + +function framebufferByteLength(scenario: ScenarioV1): number { + return qemuScenarioRenderContract(scenario).framebufferByteLength; +} + +function elfTextRodata(context: SuiteContext, binary: string): number { + const result = dockerRun(context, [ + context.target.sizeTool, + "-A", "-d", containerWorkPath(context, binary), + ], { network: false }); + if (result.exitCode !== 0) throw failure(`inspecting ${basename(binary)} ELF sections`, result); + let total = 0; + for (const line of result.stdout.split(/\r?\n/)) { + const match = /^\s*(\.text(?:\.[^\s]+)?|\.rodata(?:\.[^\s]+)?)\s+(\d+)\b/.exec(line); + if (match) total += Number(match[2]); + } + if (!Number.isSafeInteger(total) || total <= 0) { + throw new Error(`${basename(binary)} has no measurable .text/.rodata sections`); + } + return total; +} + +function binaryHash(paths: readonly { readonly tag: string; readonly path: string | null }[]): string { + const hash = createHash("sha256"); + for (const item of paths) { + hash.update(item.tag).update("\0"); + if (!item.path) { + hash.update("absent\0"); + continue; + } + const bytes = readFileSync(item.path); + hash.update(String(bytes.byteLength)).update("\0").update(bytes); + } + return hash.digest("hex"); +} + +function environment( + context: SuiteContext, + scenario: ScenarioV1, + binarySha256: string, + profile: string, + build: { + readonly target?: string; + readonly rustFlags?: readonly string[]; + readonly cFlags?: readonly string[]; + readonly linkerFlags?: readonly string[]; + } = {}, +): ReceiptEnvironmentV1 { + const native = nativeProvenance(context.sourceRoot, scenario); + return { + source: native.source, + toolchain: context.toolchain, + build: { + target: build.target ?? context.target.rustTarget, + profile, + rustFlags: build.rustFlags ?? context.target.rustFlags, + cFlags: build.cFlags ?? context.target.cFlags, + linkerFlags: build.linkerFlags ?? [], + }, + executor: { + id: context.target.executor, + version: "QEMU 11.0.3 linux-user / plugin API 6", + profile: `deterministic-linux-user;host=${context.hostPlatform}/${context.hostArch};` + + `cpu=${context.target.cpuArgs.join(" ")};` + + `emulator=${context.target.emulatorArgs.join(" ")};` + + `entropy=${QEMU_ENTROPY_PROFILE};${context.imageIdentity}`, + fingerprint: context.fingerprint, + }, + binary: { sha256: binarySha256 }, + }; +} + +export function qemuQuickJsReplayReasons( + scenario: ScenarioV1, + correctnessOutput: string, + measurementOutput: string, + expectedFramebufferTraceHash?: string, +): { readonly correctness: GuestProtocolResult; readonly measurement: GuestProtocolResult; readonly reasons: string[] } { + const correctness = parseGuestOutput(correctnessOutput, { framebufferTraceHash: "required" }); + const measurement = parseGuestOutput(measurementOutput, { framebufferTraceHash: "forbidden" }); + const reasons = [ + ...(correctness.status === "invalid" ? correctness.reasons.map((reason) => `correctness: ${reason}`) : []), + ...(measurement.status === "invalid" ? measurement.reasons.map((reason) => `measurement: ${reason}`) : []), + ]; + if (correctness.complete) { + if (correctness.complete.scenarioId !== scenario.id) reasons.push("correctness scenarioId mismatch"); + if (correctness.complete.framework !== scenario.subject.framework) reasons.push("correctness framework mismatch"); + if (correctness.complete.framebufferTraceHash && expectedFramebufferTraceHash !== undefined && + correctness.complete.framebufferTraceHash !== expectedFramebufferTraceHash) { + reasons.push("QEMU correctness framebuffer trace differs from Native/WASM correctness replay"); + } + } + if (measurement.complete) { + if (measurement.complete.scenarioId !== scenario.id) reasons.push("measurement scenarioId mismatch"); + if (measurement.complete.framework !== scenario.subject.framework) reasons.push("measurement framework mismatch"); + } + if (correctness.phases.length !== measurement.phases.length) { + reasons.push("correctness and measurement emitted different phase counts"); + } + for (let index = 0; index < Math.max(correctness.phases.length, measurement.phases.length); index += 1) { + const left = correctness.phases[index]; + const right = measurement.phases[index]; + if (!left || !right) continue; + if (left.phase !== right.phase || left.phaseId !== right.phaseId || left.iteration !== right.iteration) { + reasons.push(`correctness/measurement phase ${index} identity differs`); + } + if (left.drawListHash !== right.drawListHash) { + reasons.push(`correctness/measurement DrawList differs after phase ${left.phase}`); + } + } + if (correctness.complete && measurement.complete && + correctness.complete.finalDrawListHash !== measurement.complete.finalDrawListHash) { + reasons.push("correctness/measurement final DrawList differs"); + } + return { correctness, measurement, reasons }; +} + +function invalidate(receipts: readonly ReceiptV1[], reasons: readonly string[]): readonly ReceiptV1[] { + if (reasons.length === 0) return receipts; + return receipts.map((receipt) => parseReceiptV1({ + ...receipt, + status: "invalid", + invalidReasons: [...new Set([...receipt.invalidReasons, ...reasons])], + })); +} + +function nativeCorrectness(result: NativeRunResult, scenario: ScenarioV1): NativeOkResult["correctness"] { + if (result.status !== "ok") { + throw new Error(`${scenario.id}: Native correctness replay unsupported: ${result.reasons.join("; ")}`); + } + return result.correctness; +} + +function applyCorrectness( + receipts: readonly ReceiptV1[], + correctness: NativeOkResult["correctness"] | DamageCorrectnessRecordV1, +): readonly ReceiptV1[] { + return receipts.map((receipt) => { + if (!receipt.correctness) return receipt; + return parseReceiptV1({ + ...receipt, + correctness: { + ...receipt.correctness, + stateHash: correctness.stateHash, + effectHash: correctness.effectHash, + }, + }); + }); +} + +function quickJsReceipts( + context: SuiteContext, + scenario: ScenarioV1, + artifacts: GuestArtifacts, + run: QemuRunArtifacts, + native: NativeOkResult["correctness"], +): readonly ReceiptV1[] { + const replay = qemuQuickJsReplayReasons( + scenario, + run.correctnessOutput, + run.measurementOutput, + native.framebufferTraceHash, + ); + const reasons = [...replay.reasons]; + if (replay.correctness.complete && replay.correctness.complete.finalDrawListHash !== native.drawListHash) { + reasons.push("QEMU correctness DrawList differs from Native/WASM correctness replay"); + } + if (run.correctnessFramebufferHash !== native.finalFramebufferHash) { + reasons.push("QEMU correctness framebuffer differs from Native/WASM correctness replay"); + } + for (const [frame, hash] of Object.entries(run.checkpointFramebufferHashes)) { + const expected = native.checkpoints[frame]?.framebuffer; + if (!expected) reasons.push(`Native/WASM correctness replay has no framebuffer checkpoint ${frame}`); + else if (hash !== expected) reasons.push(`QEMU framebuffer differs at correctness checkpoint ${frame}`); + } + if (run.correctnessStateHash !== native.stateHash) { + reasons.push("QEMU correctness state tree differs from Native/WASM correctness replay"); + } + if (run.correctnessEffectHash !== native.effectHash) { + reasons.push("QEMU correctness effect trace differs from Native/WASM correctness replay"); + } + const provenance = environment( + context, + scenario, + binaryHash([ + { tag: "elf", path: run.binary }, + { tag: "bundle", path: artifacts.bundle }, + { tag: "pak", path: artifacts.pak }, + ]), + "cargo-release-perf-guest", + ); + const receipts = createQemuReceipts(scenario, run.measurementOutput, { + provenance, + target: context.target.qemuTarget, + correctnessGuestOutput: run.correctnessOutput, + framebufferHash: native.framebufferTraceHash, + artifactMetrics: run.artifactMetrics, + createdAt: new Date().toISOString(), + }); + return invalidate(applyCorrectness(receipts, native), reasons); +} + +function runDamage( + context: SuiteContext, + scenario: ScenarioV1, + directory: string, +): readonly ReceiptV1[] { + const fixtureDir = join(context.workDir, `damage-${safeName(scenario)}`); + const fixture = materializeDamageFixture({ + sourceRoot: context.sourceRoot, + harnessRoot: context.harnessRoot, + destination: fixtureDir, + dependencyRoot: "/source", + }); + const targetDir = join(fixtureDir, "target"); + const binary = buildCargoFixture(context, fixture.manifestPath, targetDir, fixture.binaryName); + mkdirSync(directory, { recursive: true }); + const scenarioPath = join(context.workDir, `damage-${safeName(scenario)}.json`); + writeFileSync(scenarioPath, `${JSON.stringify(scenario, null, 2)}\n`); + const base = [ + context.target.emulator, + ...context.target.cpuArgs, + ...context.target.emulatorArgs, + "-L", context.target.sysroot, + containerWorkPath(context, binary), + ...damageFixtureArgs(containerWorkPath(context, scenarioPath), "correctness"), + ]; + const correctnessRun = dockerRun(context, base, { network: false }); + writeFileSync(join(directory, "correctness.log"), correctnessRun.combined); + if (correctnessRun.exitCode !== 0) throw failure(`${scenario.id}: damage correctness replay`, correctnessRun); + const correctness = parseDamageCorrectnessOutput(correctnessRun.combined); + const measurementArgs = [ + context.target.emulator, + ...context.target.cpuArgs, + ...context.target.emulatorArgs, + "-L", context.target.sysroot, + "-d", "plugin", "-plugin", QEMU_PLUGIN, + containerWorkPath(context, binary), + ...damageFixtureArgs(containerWorkPath(context, scenarioPath), "markers"), + ]; + const measurement = dockerRun(context, measurementArgs, { network: false }); + writeFileSync(join(directory, "measurement.log"), measurement.combined); + const parsed = parseGuestOutput(measurement.combined); + const reasons: string[] = []; + if (measurement.exitCode !== 0) reasons.push(`${scenario.id}: damage measurement exited ${measurement.exitCode}`); + if (correctness.scenarioId !== scenario.id) reasons.push("damage correctness scenarioId mismatch"); + if (parsed.complete && parsed.complete.finalDrawListHash !== correctness.drawListHash) { + reasons.push("damage correctness/measurement final DrawList differs"); + } + const artifactMetrics: ArtifactMetrics = { + "artifact.elf_text_rodata_bytes": elfTextRodata(context, binary), + }; + const provenance = environment( + context, + scenario, + binaryHash([{ tag: "elf", path: binary }]), + "cargo-release-perf-damage", + ); + const receipts = createQemuReceipts(scenario, measurement.combined, { + provenance, + target: context.target.qemuTarget, + framebufferHash: correctness.framebufferTraceHash, + artifactMetrics, + createdAt: new Date().toISOString(), + }); + return invalidate(applyCorrectness(receipts, correctness), reasons); +} + +function parseVaporWorker(result: CommandOutput): VaporQemuResult | { status: "invalid"; reasons: readonly string[] } { + if (result.exitCode !== 0) throw failure("isolated Vapor QEMU adapter", result); + const records = result.stdout.split(/\r?\n/) + .filter((line) => line.startsWith(QEMU_WORKER_OUTPUT_PREFIX)); + if (records.length !== 1) throw failure("isolated Vapor QEMU adapter", result); + const value = JSON.parse(records[0]!.slice(QEMU_WORKER_OUTPUT_PREFIX.length)) as Record; + if (value.status === "invalid") { + const reasons = Array.isArray(value.reasons) ? value.reasons.map(String) : ["Vapor adapter returned invalid"]; + return { status: "invalid", reasons }; + } + if (value.status !== "ok" || typeof value.combinedOutput !== "string" || + typeof value.framebufferHash !== "string" || typeof value.elfPath !== "string" || + typeof value.stateHash !== "string" || typeof value.effectHash !== "string" || + typeof value.finalDrawListHash !== "string" || + (value.executor !== "qemu-armv7-thumb2" && value.executor !== "qemu-aarch64") || + typeof value.artifactMetrics !== "object" || value.artifactMetrics === null || + typeof value.build !== "object" || value.build === null) { + throw new Error("isolated Vapor QEMU adapter returned an invalid result"); + } + return value as unknown as VaporQemuResult; +} + +function runVapor( + context: SuiteContext, + scenario: ScenarioV1, + directory: string, +): readonly ReceiptV1[] { + mkdirSync(directory, { recursive: true }); + const scenarioPath = join(directory, "scenario.json"); + writeFileSync(scenarioPath, `${JSON.stringify(scenario, null, 2)}\n`); + const worker = command([ + process.execPath, + join(context.harnessRoot, "tools/perf/executors/qemu-worker.ts"), + "--scenario", scenarioPath, + "--executor", context.target.executor, + "--source-root", context.sourceRoot, + "--harness-root", context.harnessRoot, + "--out-dir", directory, + "--image", context.image, + ], context.harnessRoot); + const result = parseVaporWorker(worker); + if (result.status === "invalid") throw new Error(result.reasons.join("; ")); + if (result.executor !== context.target.executor) { + throw new Error(`Vapor adapter returned executor ${result.executor} for ${context.target.executor}`); + } + writeFileSync(join(directory, "measurement.log"), result.combinedOutput); + const elfPath = resolve(result.elfPath); + if (!existsSync(elfPath)) throw new Error(`Vapor adapter ELF is missing: ${elfPath}`); + const build = result.build; + if (build.qemuTarget !== context.target.qemuTarget || + JSON.stringify(build.cpuArgs) !== JSON.stringify(context.target.cpuArgs) || + JSON.stringify(build.emulatorArgs) !== JSON.stringify(context.target.emulatorArgs)) { + throw new Error(`Vapor adapter QEMU target/invocation profile differs from ${context.target.executor}`); + } + const provenance = environment( + context, + scenario, + binaryHash([{ tag: "elf", path: elfPath }]), + "generated-c-release", + { + target: context.target.rustTarget, + rustFlags: [], + cFlags: build.cFlags, + linkerFlags: build.linkerFlags, + }, + ); + const receipts = createQemuReceipts(scenario, result.combinedOutput, { + provenance, + target: build.qemuTarget, + framebufferHash: result.framebufferHash, + artifactMetrics: result.artifactMetrics, + createdAt: new Date().toISOString(), + }); + return receipts.map((receipt) => receipt.correctness + ? parseReceiptV1({ + ...receipt, + correctness: { + ...receipt.correctness, + stateHash: result.stateHash, + effectHash: result.effectHash, + }, + }) + : receipt); +} + +/** Build and run one deterministic local suite under pinned QEMU linux-user. */ +export async function runQemuSuite(options: QemuBridgeOptions): Promise { + let scenarios: ScenarioV1[]; + try { + scenarios = expandSuiteFrameworks(loadScenarioSuite(options.suite, options.scenarioDir)); + if (scenarios.length === 0) { + return { receipts: [], invalidReasons: [`no scenarios found for suite ${JSON.stringify(options.suite)}`] }; + } + const capabilityReasons = scenarios.flatMap(qemuScenarioCapabilityReasons); + if (capabilityReasons.length > 0) { + return { receipts: [], invalidReasons: capabilityReasons }; + } + const estimate = estimatedSuiteSeconds(scenarios); + if (estimate > options.maxEstimatedSeconds) { + return { + receipts: [], + invalidReasons: [ + `${options.suite} suite estimate ${estimate}s exceeds the ${options.maxEstimatedSeconds}s limit`, + ], + }; + } + } catch (error) { + return { receipts: [], invalidReasons: [error instanceof Error ? error.message : String(error)] }; + } + + let context: SuiteContext; + try { + context = makeContext(options); + } catch (error) { + return { receipts: [], invalidReasons: [error instanceof Error ? error.message : String(error)] }; + } + try { + const receipts: ReceiptV1[] = []; + const invalidReasons: string[] = []; + let guestBinary: string | null = null; + let wasmBuilt = false; + const builtApps = new Map(); + + for (const scenario of scenarios) { + const directory = join(context.outDir, "raw", safeName(scenario)); + try { + let next: readonly ReceiptV1[]; + if (scenario.subject.family === "guest-app") { + if (!wasmBuilt) { + buildWasm(context.sourceRoot); + wasmBuilt = true; + } + if (!guestBinary) { + const guest = materializeGuestHarness(context); + guestBinary = buildCargoFixture( + context, + guest.manifest, + guest.targetDir, + "pocketjs-perf-guest", + ); + } + const appKey = qemuScenarioRenderContract(scenario).artifactCacheKey; + let artifacts = builtApps.get(appKey); + if (!artifacts) { + artifacts = buildGuestArtifacts(context, scenario); + builtApps.set(appKey, artifacts); + } + const native = nativeCorrectness( + runIsolatedNative(context, scenario, artifacts, join(directory, "native")), + scenario, + ); + const run = runQuickJsGuest(context, guestBinary, scenario, artifacts, directory); + next = quickJsReceipts(context, scenario, artifacts, run, native); + } else if (isDamageScenario(scenario)) { + next = runDamage(context, scenario, directory); + } else if (scenario.subject.family === "vapor") { + next = runVapor(context, scenario, directory); + } else { + throw new Error(`no QEMU adapter for subject family ${JSON.stringify(scenario.subject.family)}`); + } + receipts.push(...next); + for (const receipt of next) { + if (receipt.status === "invalid") { + invalidReasons.push(...receipt.invalidReasons.map((reason) => `${scenario.id}: ${reason}`)); + } + } + } catch (error) { + invalidReasons.push(`${scenario.id}: ${error instanceof Error ? error.message : String(error)}`); + } + } + return { receipts, invalidReasons: [...new Set(invalidReasons)] }; + } finally { + cleanupWorkDirectory(context.outDir, context.workDir, context.image); + } +} diff --git a/tools/perf/executors/vapor.ts b/tools/perf/executors/vapor.ts new file mode 100644 index 00000000..f4308cd8 --- /dev/null +++ b/tools/perf/executors/vapor.ts @@ -0,0 +1,2246 @@ +import { createHash } from "node:crypto"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import type { ScenarioV1 } from "../core/types.ts"; +import { scenarioPhaseId } from "../receipts/hash.ts"; +import { + GUEST_OUTPUT_PREFIX, + parseGuestOutput, + parseQemuOutput, + QEMU_OUTPUT_PREFIX, +} from "../receipts/protocol.ts"; +import type { + GuestProtocolResult, + QemuProtocolResult, +} from "../receipts/protocol.ts"; +import type { NativeOkResult, NativeRunResult } from "../runner/native.ts"; + +type VaporExecutor = "native" | "qemu-armv7-thumb2" | "qemu-aarch64"; +type VaporTargetName = "gba" | "playdate"; + +interface VaporGrid { + readonly chars: readonly string[]; + readonly pals: readonly (readonly number[])[]; +} + +interface VaporNodeLike { + readonly tag?: string; + readonly text?: string; + readonly attrs?: ReadonlyMap; + readonly children?: readonly VaporNodeLike[]; + readonly nodeType?: number; +} + +interface CompiledVaporAppLike { + readonly c: string; + readonly styles: unknown; + readonly debugSlots: readonly VaporDebugSlot[]; + readonly relativeAxesUsed: readonly number[]; + readonly buttonsUsed: readonly number[]; +} + +interface VaporDebugSlot { + readonly name: string; + readonly offset: number; + readonly size: number; + readonly kind: "num" | "bool" | "str" | "listLen"; +} + +interface VaporOracleLike { + readonly root: VaporNodeLike; + press(button: number): Promise; + axisDelta(axis: number, delta: number): Promise; + grid(): VaporGrid; + unmount(): void; +} + +type VaporDebugValue = number | boolean | string; + +interface VaporEvent { + readonly frame: number; + readonly kind: "button" | "relative-axis"; + readonly control: number; + readonly value: number; +} + +interface TargetProfile { + readonly target: VaporTargetName; + readonly width: number; + readonly height: number; + readonly poolCap: number; + readonly strCap: number; + readonly benchmarkEntry: string; +} + +export interface RunVaporScenarioOptions { + readonly scenario: ScenarioV1; + readonly executor: VaporExecutor; + readonly sourceRoot: string; + readonly harnessRoot: string; + readonly outDir?: string; + readonly image?: string; +} + +export interface RunNativeVaporScenarioOptions { + readonly sourceRoot: string; + readonly harnessRoot: string; + readonly outDir?: string; +} + +export interface VaporInvalidResult { + readonly status: "invalid"; + readonly executor: VaporExecutor; + readonly reasons: readonly string[]; + readonly combinedOutput?: string; +} + +export interface VaporNativeResult { + readonly status: "ok"; + readonly executor: "native"; + /** SHA-256 over every canonical oracle grid in frame order. */ + readonly framebufferHash: string; + /** SHA-256 of the final canonical oracle grid. */ + readonly finalFramebufferHash: string; + /** FNV-1a-64 over final grid chars followed by palette bytes. */ + readonly finalDrawListHash: string; + /** SHA-256 over the final real Vue Vapor micro-DOM. */ + readonly stateHash: string; + /** SHA-256 over the hardware-neutral events actually delivered. */ + readonly effectHash: string; + /** Guest-compatible state digest; the C lane uses app_debug_state instead. */ + readonly finalStateDigest: string; + /** Guest-compatible app_debug_state digests at declared state checkpoints. */ + readonly checkpointStateDigests: Readonly>; + /** Guest-compatible digest of delivered input/effect events. */ + readonly finalEffectDigest: string; + readonly phaseDrawListHashes: Readonly>; + readonly axisEventsDelivered: number; + readonly axisEventsObserved: number; + readonly compiledRelativeAxesUsed: readonly number[]; + readonly compiledButtonsUsed: readonly number[]; + readonly target: VaporTargetName; + readonly width: number; + readonly height: number; +} + +export interface VaporQemuResult { + readonly status: "ok"; + readonly executor: Exclude; + /** Interleaved-protocol-compatible text consumed by createQemuReceipts. */ + readonly combinedOutput: string; + /** Generated-C correctness presentation-buffer trace, after Vue oracle parity. */ + readonly framebufferHash: string; + /** Native DOM hash retained for receipts only after generated-C debug-state parity succeeds. */ + readonly stateHash: string; + readonly effectHash: string; + readonly finalDrawListHash: string; + readonly elfPath: string; + readonly artifactMetrics: Readonly<{ + "artifact.elf_text_rodata_bytes": number; + }>; + readonly build: VaporQemuBuildSpec; +} + +export type VaporScenarioResult = VaporInvalidResult | VaporNativeResult | VaporQemuResult; + +export interface VaporQemuBuildSpec { + readonly compiler: string; + readonly emulator: string; + readonly cpuArgs: readonly string[]; + readonly emulatorArgs: readonly string[]; + readonly qemuTarget: "arm" | "aarch64"; + readonly cFlags: readonly string[]; + readonly linkerFlags: readonly string[]; +} + +export const VAPOR_QEMU_BUILD_SPECS: Readonly< + Record, VaporQemuBuildSpec> +> = Object.freeze({ + "qemu-armv7-thumb2": Object.freeze({ + compiler: "arm-linux-gnueabihf-gcc", + emulator: "/opt/qemu/bin/qemu-arm", + cpuArgs: Object.freeze(["-cpu", "cortex-a9,neon=off,vfp-d32=off"]), + emulatorArgs: Object.freeze(["-seed", "1"]), + qemuTarget: "arm", + cFlags: Object.freeze([ + "-std=c11", + "-O2", + "-g0", + "-march=armv7-a", + "-mthumb", + "-mfpu=vfpv3-d16", + "-mfloat-abi=hard", + "-ffreestanding", + "-fno-builtin", + "-fno-ident", + "-fno-stack-protector", + "-fno-asynchronous-unwind-tables", + "-fno-unwind-tables", + "-Wall", + "-Wextra", + "-Werror", + ]), + linkerFlags: Object.freeze(["-nostdlib", "-static", "-Wl,-e,_start", "-Wl,--build-id=none", "-lgcc"]), + }), + "qemu-aarch64": Object.freeze({ + compiler: "aarch64-linux-gnu-gcc", + emulator: "/opt/qemu/bin/qemu-aarch64", + cpuArgs: Object.freeze(["-cpu", "cortex-a53"]), + emulatorArgs: Object.freeze(["-seed", "1"]), + qemuTarget: "aarch64", + cFlags: Object.freeze([ + "-std=c11", + "-O2", + "-g0", + "-march=armv8-a", + "-ffreestanding", + "-fno-builtin", + "-fno-ident", + "-fno-stack-protector", + "-fno-asynchronous-unwind-tables", + "-fno-unwind-tables", + "-Wall", + "-Wextra", + "-Werror", + ]), + linkerFlags: Object.freeze(["-nostdlib", "-static", "-Wl,-e,_start", "-Wl,--build-id=none", "-lgcc"]), + }), +}); + +const BUTTONS: Readonly> = Object.freeze({ + primary: 0, + secondary: 1, + select: 2, + start: 3, + right: 4, + left: 5, + up: 6, + down: 7, + "shoulder-right": 8, + "shoulder-left": 9, +}); + +const RELATIVE_AXES: Readonly> = Object.freeze({ + primary: 0, + secondary: 1, +}); + +const FNV_OFFSET = 0xcbf29ce484222325n; +const FNV_PRIME = 0x100000001b3n; +const U64_MASK = 0xffffffffffffffffn; + +function unique(values: readonly string[]): string[] { + return [...new Set(values.filter(Boolean))]; +} + +function invalid( + executor: VaporExecutor, + reasons: readonly string[], + combinedOutput?: string, +): VaporInvalidResult { + return { status: "invalid", executor, reasons: unique(reasons), ...(combinedOutput ? { combinedOutput } : {}) }; +} + +function validateScenario(scenario: ScenarioV1): string[] { + const reasons: string[] = []; + if (scenario.subject.family !== "vapor") { + reasons.push(`Vapor adapter requires subject.family=vapor, got ${JSON.stringify(scenario.subject.family)}`); + } + if (scenario.subject.framework !== "core") { + reasons.push(`Vapor generated-C receipts require subject.framework=core`); + } + if (scenario.frames !== scenario.tape.frames) { + reasons.push(`scenario frames ${scenario.frames} do not match tape frames ${scenario.tape.frames}`); + } + if (!scenario.phases.some((phase) => phase.collect)) reasons.push("scenario has no collected phase"); + for (const track of scenario.tape.tracks) { + if (track.kind !== "button" && track.kind !== "relative-axis") { + reasons.push(`Vapor adapter does not support ${track.kind} track ${JSON.stringify("control" in track ? track.control : track.effect)}`); + } + } + return reasons; +} + +function profileForEntry(sourceRoot: string, entry: string): TargetProfile | null { + const normalized = entry.replaceAll("\\", "/"); + if (normalized.endsWith("vapor/examples/todo/todo.tsx") || + normalized.endsWith("vapor/examples/todo/todo.playdate.tsx")) { + return { + target: "playdate", + width: 50, + height: 30, + poolCap: 32, + strCap: 24, + benchmarkEntry: join(sourceRoot, "vapor/examples/todo/todo.playdate.tsx"), + }; + } + return null; +} + +/** + * The performance fixture stays derived from the real crank-driven Todo, but + * adds one monotonic reactive row. The scenario's +44999,+1,-45000 tape ends + * with zero net motion, so the normal cursor would return to its initial row; + * this counter makes every RelativeAxis delivery observable at phase end. + */ +function benchmarkFixtureSource(profile: TargetProfile): string { + let source = readFileSync(profile.benchmarkEntry, "utf8"); + const edits: readonly [string, string][] = [ + [ + " const crankRemainder = ref(0);", + " const crankRemainder = ref(0);\n const axisEvents = ref(0);", + ], + [ + " onAxisDelta(RelativeAxis.Primary, (delta) => {\n", + " onAxisDelta(RelativeAxis.Primary, (delta) => {\n axisEvents.value += 1;\n", + ], + [ + " \n" + + " {\" AXIS EVENTS \"}\n" + + " {axisEvents.value}\n" + + " \n" + + " { + if (track.kind === "button") { + if (BUTTONS[track.control] === undefined) { + reasons.push(`Vapor has no button mapping for ${JSON.stringify(track.control)}`); + return; + } + track.samples.forEach((sample, sampleIndex) => raw.push({ + frame: sample.frame, + track: trackIndex, + sample: sampleIndex, + kind: "button-sample", + control: track.control, + value: sample.pressed ? 1 : 0, + })); + } else if (track.kind === "relative-axis") { + const axis = RELATIVE_AXES[track.control]; + if (axis === undefined) { + reasons.push(`Vapor has no relative-axis mapping for ${JSON.stringify(track.control)}`); + return; + } + track.samples.forEach((sample, sampleIndex) => { + if (!Number.isInteger(sample.delta) || sample.delta === 0 || sample.delta < -0x80000000 || sample.delta > 0x7fffffff) { + reasons.push( + `relative-axis ${JSON.stringify(track.control)} frame ${sample.frame} delta must be a non-zero signed 32-bit integer`, + ); + return; + } + raw.push({ + frame: sample.frame, + track: trackIndex, + sample: sampleIndex, + kind: "relative-axis", + control: track.control, + value: sample.delta, + }); + }); + } + }); + raw.sort((a, b) => a.frame - b.frame || a.track - b.track || a.sample - b.sample); + const pressed = new Map(); + const events: VaporEvent[] = []; + for (const event of raw) { + if (event.kind === "button-sample") { + const wasPressed = pressed.get(event.control) ?? false; + const isPressed = event.value === 1; + pressed.set(event.control, isPressed); + if (isPressed && !wasPressed) { + events.push({ frame: event.frame, kind: "button", control: BUTTONS[event.control]!, value: 1 }); + } + } else { + events.push({ + frame: event.frame, + kind: "relative-axis", + control: RELATIVE_AXES[event.control]!, + value: event.value, + }); + } + } + return { events, reasons }; +} + +function fnvBytes(bytes: Iterable, seed = FNV_OFFSET): bigint { + let hash = seed; + for (const byte of bytes) { + hash ^= BigInt(byte & 0xff); + hash = (hash * FNV_PRIME) & U64_MASK; + } + return hash; +} + +function taggedFnv(hash: bigint): string { + return `fnv1a64:${hash.toString(16).padStart(16, "0")}`; +} + +function gridBytes(grid: VaporGrid, width: number, height: number): Uint8Array { + if (grid.chars.length !== height || grid.pals.length !== height) { + throw new Error(`oracle grid geometry mismatch: expected ${width}x${height}`); + } + const bytes = new Uint8Array(width * height * 2); + let at = 0; + for (let y = 0; y < height; y += 1) { + const row = grid.chars[y]!; + if (row.length !== width || grid.pals[y]!.length !== width) { + throw new Error(`oracle grid row ${y} does not match width ${width}`); + } + for (let x = 0; x < width; x += 1) bytes[at++] = row.charCodeAt(x) & 0xff; + } + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) bytes[at++] = grid.pals[y]![x]! & 0xff; + } + return bytes; +} + +function observedAxisEvents(grid: VaporGrid, profile: TargetProfile): number { + const line = grid.chars[profile.height - 2]?.trim() ?? ""; + const match = /^AXIS EVENTS (\d+)$/.exec(line); + if (!match) throw new Error(`Vapor fixture axis row is missing or malformed: ${JSON.stringify(line)}`); + return Number(match[1]); +} + +function assertAxisEvents( + grid: VaporGrid, + profile: TargetProfile, + events: readonly VaporEvent[], +): number { + const expected = events.filter((event) => event.kind === "relative-axis").length; + const observed = observedAxisEvents(grid, profile); + if (observed !== expected) { + throw new Error(`Vapor fixture observed ${observed} relative-axis events; expected ${expected}`); + } + return observed; +} + +function eventBytes(events: readonly VaporEvent[]): Uint8Array { + const bytes: number[] = []; + const u32 = (value: number) => { + const unsigned = value >>> 0; + bytes.push(unsigned & 0xff, (unsigned >>> 8) & 0xff, (unsigned >>> 16) & 0xff, (unsigned >>> 24) & 0xff); + }; + for (const event of events) { + bytes.push(event.kind === "button" ? 1 : 2); + u32(event.frame); + bytes.push(event.control); + if (event.kind === "relative-axis") u32(event.value); + } + return Uint8Array.from(bytes); +} + +function canonicalNode(node: VaporNodeLike): unknown { + if (node.nodeType === 3) return ["text", node.text ?? ""]; + if (node.nodeType === 8) return ["comment", node.text ?? ""]; + return [ + "element", + node.tag ?? "", + Object.fromEntries([...(node.attrs ?? new Map())].sort(([a], [b]) => a.localeCompare(b))), + (node.children ?? []).map(canonicalNode), + ]; +} + +function canonicalJson(value: unknown): string { + if (value === null || typeof value === "boolean" || typeof value === "string" || typeof value === "number") { + return JSON.stringify(value); + } + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object") { + const record = value as Record; + return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(",")}}`; + } + throw new Error(`cannot canonicalize ${typeof value}`); +} + +function sha256(value: string | Uint8Array): string { + return createHash("sha256").update(value).digest("hex"); +} + +function oracleStateFixtureSource( + source: string, + slots: readonly VaporDebugSlot[], + hookName: string, +): string { + const anchor = "\n return (\n"; + const at = source.lastIndexOf(anchor); + if (at < 0) throw new Error("Vapor benchmark fixture has no component return anchor"); + const values = slots.map((slot) => { + const value = slot.kind === "listLen" + ? `${slot.name}.value.length` + : `${slot.name}.value`; + return ` ${JSON.stringify(slot.name)}: ${value},`; + }).join("\n"); + const hook = [ + "", + ` (globalThis as Record)[${JSON.stringify(hookName)}] = () => ({`, + values, + " });", + ].join("\n"); + return `${source.slice(0, at)}${hook}${source.slice(at)}`; +} + +function debugStateBytes( + slots: readonly VaporDebugSlot[], + values: Readonly>, +): Uint8Array { + const bytes: number[] = []; + const names = new Set(slots.map((slot) => slot.name)); + const unknown = Object.keys(values).filter((name) => !names.has(name)); + if (unknown.length > 0) { + throw new Error(`Vapor oracle state hook returned unknown slots: ${unknown.join(", ")}`); + } + for (const slot of slots) { + if (!Object.hasOwn(values, slot.name)) { + throw new Error(`Vapor oracle state hook omitted ${slot.name}`); + } + const value = values[slot.name]!; + if (slot.kind === "str") { + if (typeof value !== "string") throw new Error(`Vapor state slot ${slot.name} is not a string`); + if (value.length > slot.size - 1) { + throw new Error(`Vapor state slot ${slot.name} exceeds its ${slot.size - 1}-byte capacity`); + } + bytes.push(value.length); + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code > 0x7f) throw new Error(`Vapor state slot ${slot.name} is not ASCII`); + bytes.push(code); + } + continue; + } + const normalized = slot.kind === "bool" + ? value === true ? 1 : value === false ? 0 : value + : value; + if (typeof normalized !== "number" || !Number.isInteger(normalized) || + normalized < -0x80000000 || normalized > 0x7fffffff) { + throw new Error(`Vapor state slot ${slot.name} is not a signed 32-bit integer`); + } + const unsigned = normalized >>> 0; + bytes.push( + unsigned & 0xff, + (unsigned >>> 8) & 0xff, + (unsigned >>> 16) & 0xff, + (unsigned >>> 24) & 0xff, + ); + } + return Uint8Array.from(bytes); +} + +async function withSourceDependencies( + sourceRoot: string, + harnessRoot: string, + body: () => Promise, +): Promise { + const sourceModules = join(sourceRoot, "node_modules"); + const harnessModules = join(harnessRoot, "node_modules"); + let linked = false; + if (!existsSync(sourceModules) && existsSync(harnessModules)) { + symlinkSync(harnessModules, sourceModules, "dir"); + linked = true; + } + try { + return await body(); + } finally { + if (linked) unlinkSync(sourceModules); + } +} + +let oracleLeaseTail: Promise = Promise.resolve(); + +async function acquireOracleLease(): Promise<() => void> { + const previous = oracleLeaseTail; + let releaseCurrent!: () => void; + const current = new Promise((resolve) => { releaseCurrent = resolve; }); + oracleLeaseTail = previous.then(() => current); + await previous; + let released = false; + return () => { + if (released) return; + released = true; + releaseCurrent(); + }; +} + +interface OracleSession { + readonly app: CompiledVaporAppLike; + readonly oracle: VaporOracleLike; + debugStateBytes(): Uint8Array; + cleanup(): void; +} + +function stageOracleModule(stageRoot: string, sourceRoot: string): string { + const sourceUrl = pathToFileURL(join(sourceRoot, "vapor/oracle/boot.ts")); + sourceUrl.searchParams.set("pocket-perf-stage", basename(stageRoot)); + const wrapperPath = join(stageRoot, "oracle-stage.ts"); + writeFileSync( + wrapperPath, + `export { bootOracle } from ${JSON.stringify(sourceUrl.href)};\n`, + ); + return wrapperPath; +} + +async function compileAndBoot( + _scenario: ScenarioV1, + sourceRoot: string, + harnessRoot: string, + profile: TargetProfile, +): Promise { + const releaseLease = await acquireOracleLease(); + const stageRoot = mkdtempSync(join(tmpdir(), "pocketjs-perf-vapor-oracle-")); + const stateHookName = `__pocketPerfVaporState_${basename(stageRoot).replace(/[^a-zA-Z0-9_]/g, "_")}`; + let handedOff = false; + try { + const session = await withSourceDependencies(sourceRoot, harnessRoot, async () => { + const compilerUrl = pathToFileURL(join(sourceRoot, "vapor/compiler/compile.ts")); + compilerUrl.searchParams.set("pocket-perf-source", basename(sourceRoot)); + const oracleUrl = pathToFileURL(stageOracleModule(stageRoot, sourceRoot)); + const compiler = await import(compilerUrl.href) as { + compileVaporApp( + fileName: string, + source: string, + title: string, + target: VaporTargetName, + ): CompiledVaporAppLike; + }; + const oracleModule = await import(oracleUrl.href) as { + bootOracle(options: { + width: number; + height: number; + styles: unknown; + entry: string; + }): Promise; + }; + const source = benchmarkFixtureSource(profile); + const app = compiler.compileVaporApp(profile.benchmarkEntry, source, "PERF AXIS", profile.target); + const modules = join(harnessRoot, "node_modules"); + if (existsSync(modules)) symlinkSync(modules, join(stageRoot, "node_modules"), "dir"); + const inputPath = join(sourceRoot, "vapor/host/input.ts"); + const screenPath = join(sourceRoot, "vapor/host/screen.ts"); + const oracleApp = oracleStateFixtureSource(source, app.debugSlots, stateHookName) + .replace('from "../../host/input.ts"', `from ${JSON.stringify(inputPath)}`) + .replace('from "../../host/screen.ts"', `from ${JSON.stringify(screenPath)}`); + const appPath = join(stageRoot, "axis-app.tsx"); + const entryPath = join(stageRoot, "axis-entry.ts"); + writeFileSync(appPath, oracleApp); + writeFileSync(entryPath, ` +import { createVaporApp, nextTick } from "vue"; +import AxisApp from "./axis-app.tsx"; +import { __dispatchAxisDelta, __dispatchButton, __resetButtons } from ${JSON.stringify(inputPath)}; +type AnyApp = { mount(container: unknown): void; unmount(): void }; +const hooks = globalThis as Record; +hooks.__vaporBoot = (container: unknown): AnyApp => { + __resetButtons(); + const app = (createVaporApp as unknown as (comp: unknown) => AnyApp)({ setup: () => (AxisApp as () => unknown)() }); + app.mount(container); + return app; +}; +hooks.__vaporPress = (button: number): void => { __dispatchButton(button); }; +hooks.__vaporAxisDelta = (axis: number, delta: number): void => { __dispatchAxisDelta(axis as 0 | 1, delta); }; +hooks.__vaporTick = (): Promise => nextTick(); +`); + const oracle = await oracleModule.bootOracle({ + width: profile.width, + height: profile.height, + styles: app.styles, + entry: entryPath, + }); + const stateHook = (globalThis as Record)[stateHookName]; + if (typeof stateHook !== "function") { + throw new Error("Vapor benchmark fixture did not install its state hook"); + } + return { + app, + oracle, + debugStateBytes: () => debugStateBytes( + app.debugSlots, + (stateHook as () => Readonly>)(), + ), + }; + }); + handedOff = true; + let cleaned = false; + return { + ...session, + cleanup() { + if (cleaned) return; + cleaned = true; + try { + delete (globalThis as Record)[stateHookName]; + rmSync(stageRoot, { recursive: true, force: true }); + } finally { + releaseLease(); + } + }, + }; + } finally { + if (!handedOff) { + try { + delete (globalThis as Record)[stateHookName]; + rmSync(stageRoot, { recursive: true, force: true }); + } finally { + releaseLease(); + } + } + } +} + +async function runNativeInProcess( + options: RunVaporScenarioOptions, + events: readonly VaporEvent[], + profile: TargetProfile, +): Promise { + let app: CompiledVaporAppLike; + let oracle: VaporOracleLike; + let captureDebugState: () => Uint8Array = () => new Uint8Array(); + let cleanup: () => void = () => {}; + try { + ({ app, oracle, debugStateBytes: captureDebugState, cleanup } = await compileAndBoot( + options.scenario, + resolve(options.sourceRoot), + resolve(options.harnessRoot), + profile, + )); + } catch (error) { + return invalid(options.executor, [error instanceof Error ? error.message : String(error)]); + } + + try { + const byFrame = new Map(); + for (const event of events) { + const current = byFrame.get(event.frame); + if (current) current.push(event); + else byFrame.set(event.frame, [event]); + } + const trace = createHash("sha256"); + const phaseEnds = new Map(); + for (const phase of options.scenario.phases) { + if (!phase.collect) continue; + const current = phaseEnds.get(phase.endFrame - 1); + if (current) current.push(phase.name); + else phaseEnds.set(phase.endFrame - 1, [phase.name]); + } + const phaseDrawListHashes: Record = {}; + const stateCheckpointFrames = new Set( + options.scenario.checkpoints + .filter((checkpoint) => checkpoint.capture.includes("state")) + .map((checkpoint) => checkpoint.frame), + ); + const checkpointStateDigests: Record = {}; + let finalBytes: Uint8Array = new Uint8Array(); + let finalGrid: VaporGrid | null = null; + let finalDrawListHash = taggedFnv(FNV_OFFSET); + let axisEventsDelivered = 0; + for (let frame = 0; frame < options.scenario.frames; frame += 1) { + for (const event of byFrame.get(frame) ?? []) { + if (event.kind === "button") await oracle.press(event.control); + else { + await oracle.axisDelta(event.control, event.value); + axisEventsDelivered += 1; + } + } + finalGrid = oracle.grid(); + finalBytes = gridBytes(finalGrid, profile.width, profile.height); + finalDrawListHash = taggedFnv(fnvBytes(finalBytes)); + trace.update(finalBytes); + for (const name of phaseEnds.get(frame) ?? []) phaseDrawListHashes[name] = finalDrawListHash; + if (stateCheckpointFrames.has(frame)) { + checkpointStateDigests[String(frame)] = taggedFnv(fnvBytes(captureDebugState())); + } + } + const nodeJson = canonicalJson(canonicalNode(oracle.root)); + const effects = eventBytes(events); + if (!finalGrid) throw new Error(`${options.scenario.id}: Vapor fixture produced no grid`); + const axisEventsObserved = assertAxisEvents(finalGrid, profile, events); + return { + status: "ok", + executor: "native", + framebufferHash: trace.digest("hex"), + finalFramebufferHash: sha256(finalBytes), + finalDrawListHash, + stateHash: sha256(nodeJson), + effectHash: sha256(effects), + finalStateDigest: taggedFnv(fnvBytes(captureDebugState())), + checkpointStateDigests, + finalEffectDigest: taggedFnv(fnvBytes(effects)), + phaseDrawListHashes, + axisEventsDelivered, + axisEventsObserved, + compiledRelativeAxesUsed: [...app.relativeAxesUsed], + compiledButtonsUsed: [...app.buttonsUsed], + target: profile.target, + width: profile.width, + height: profile.height, + }; + } catch (error) { + return invalid(options.executor, [error instanceof Error ? error.message : String(error)]); + } finally { + try { + oracle.unmount(); + } finally { + cleanup(); + } + } +} + +function safeNanoseconds(value: bigint): number { + const result = Number(value); + if (!Number.isSafeInteger(result) || result < 0) { + throw new Error(`native Vapor timing exceeds the safe integer range: ${value}`); + } + return result; +} + +function nativeResultFile( + result: T, + outDir: string | undefined, +): T { + if (!outDir) return result; + mkdirSync(outDir, { recursive: true }); + const safeId = result.scenarioId.replace(/[^a-zA-Z0-9._-]+/g, "-"); + writeFileSync(join(outDir, `${safeId}.native.json`), `${JSON.stringify(result, null, 2)}\n`); + return result; +} + +function nativeUnsupported( + scenario: ScenarioV1, + reasons: readonly string[], + outDir: string | undefined, +): NativeRunResult { + return nativeResultFile({ + schemaVersion: 1, + kind: "pocketjs.perf.native-result", + status: "unsupported", + scenarioId: scenario.id, + executor: "native", + reasons: unique(reasons), + }, outDir); +} + +async function vaporCorrectnessReplayInProcess( + scenario: ScenarioV1, + options: RunNativeVaporScenarioOptions, + events: readonly VaporEvent[], + profile: TargetProfile, +): Promise { + const { oracle, cleanup } = await compileAndBoot( + scenario, + resolve(options.sourceRoot), + resolve(options.harnessRoot), + profile, + ); + try { + const byFrame = new Map(); + for (const event of events) { + const current = byFrame.get(event.frame); + if (current) current.push(event); + else byFrame.set(event.frame, [event]); + } + const checkpoints = new Map(scenario.checkpoints.map((checkpoint) => [checkpoint.frame, checkpoint])); + const captured: Record> = {}; + const trace = createHash("sha256"); + let finalBytes = new Uint8Array(0); + let finalDrawListHash = taggedFnv(FNV_OFFSET); + for (let frame = 0; frame < scenario.frames; frame += 1) { + for (const event of byFrame.get(frame) ?? []) { + if (event.kind === "button") await oracle.press(event.control); + else await oracle.axisDelta(event.control, event.value); + } + finalBytes = gridBytes(oracle.grid(), profile.width, profile.height); + finalDrawListHash = taggedFnv(fnvBytes(finalBytes)); + trace.update(finalBytes); + const checkpoint = checkpoints.get(frame); + if (!checkpoint) continue; + const values: Record = {}; + for (const capture of checkpoint.capture) { + if (capture === "framebuffer") values.framebuffer = sha256(finalBytes); + else if (capture === "drawList") values.drawList = finalDrawListHash; + else if (capture === "state") { + values.state = sha256(canonicalJson(canonicalNode(oracle.root))); + } else { + values.effects = sha256(eventBytes(events.filter((event) => event.frame <= frame))); + } + } + captured[String(frame)] = values; + } + assertAxisEvents(oracle.grid(), profile, events); + const finalState = canonicalJson(canonicalNode(oracle.root)); + return { + framebufferTraceHash: trace.digest("hex"), + finalFramebufferHash: sha256(finalBytes), + drawListHash: finalDrawListHash, + stateHash: sha256(finalState), + effectHash: sha256(eventBytes(events)), + checkpoints: captured, + }; + } finally { + try { + oracle.unmount(); + } finally { + cleanup(); + } + } +} + +async function vaporMeasurementReplayInProcess( + scenario: ScenarioV1, + options: RunNativeVaporScenarioOptions, + events: readonly VaporEvent[], + profile: TargetProfile, +): Promise { + const bootStarted = process.hrtime.bigint(); + const { oracle, cleanup } = await compileAndBoot( + scenario, + resolve(options.sourceRoot), + resolve(options.harnessRoot), + profile, + ); + const bootWallTimeNs = safeNanoseconds(process.hrtime.bigint() - bootStarted); + try { + const byFrame = new Map(); + for (const event of events) { + const current = byFrame.get(event.frame); + if (current) current.push(event); + else byFrame.set(event.frame, [event]); + } + const starts = new Map(); + const ends = new Map(); + for (const phase of scenario.phases) { + if (!phase.collect) continue; + const atStart = starts.get(phase.startFrame) ?? []; + starts.set(phase.startFrame, [...atStart, phase]); + const atEnd = ends.get(phase.endFrame - 1) ?? []; + ends.set(phase.endFrame - 1, [...atEnd, phase]); + } + const started = new Map(); + const timings: NativeOkResult["measurement"]["phases"][number][] = []; + for (let frame = 0; frame < scenario.frames; frame += 1) { + for (const phase of starts.get(frame) ?? []) started.set(phase.name, process.hrtime.bigint()); + for (const event of byFrame.get(frame) ?? []) { + if (event.kind === "button") await oracle.press(event.control); + else await oracle.axisDelta(event.control, event.value); + } + for (const phase of ends.get(frame) ?? []) { + const start = started.get(phase.name); + if (start === undefined) throw new Error(`Vapor phase ${phase.name} never started`); + timings.push({ + name: phase.name, + startFrame: phase.startFrame, + endFrame: phase.endFrame, + wallTimeNs: safeNanoseconds(process.hrtime.bigint() - start), + }); + } + } + // Both fingerprints are correctness work and deliberately happen after + // every measured phase has ended. + const finalGrid = oracle.grid(); + assertAxisEvents(finalGrid, profile, events); + const finalBytes = gridBytes(finalGrid, profile.width, profile.height); + return { + bootWallTimeNs, + phases: timings, + finalFramebufferHash: sha256(finalBytes), + finalDrawListHash: taggedFnv(fnvBytes(finalBytes)), + }; + } finally { + try { + oracle.unmount(); + } finally { + cleanup(); + } + } +} + +type VaporOracleReplayKind = "native" | "correctness" | "measurement"; + +interface VaporOracleReplayRequest { + readonly schemaVersion: 1; + readonly kind: VaporOracleReplayKind; + readonly scenario: ScenarioV1; + readonly sourceRoot: string; + readonly harnessRoot: string; + readonly events: readonly VaporEvent[]; + readonly profile: TargetProfile; +} + +type VaporOracleReplayResponse = + | { + readonly schemaVersion: 1; + readonly kind: VaporOracleReplayKind; + readonly status: "ok"; + readonly result: unknown; + } + | { + readonly schemaVersion: 1; + readonly kind: VaporOracleReplayKind; + readonly status: "error"; + readonly reason: string; + }; + +/** Internal entry used by the per-replay staged Bun module. */ +export async function runVaporOracleReplayChild( + requestPath: string, + responsePath: string, +): Promise { + let kind: VaporOracleReplayKind = "native"; + let response: VaporOracleReplayResponse; + try { + const request = JSON.parse(readFileSync(requestPath, "utf8")) as VaporOracleReplayRequest; + if (request.schemaVersion !== 1) throw new Error("unsupported Vapor oracle replay request schema"); + if (request.kind !== "native" && request.kind !== "correctness" && request.kind !== "measurement") { + throw new Error(`unsupported Vapor oracle replay kind ${JSON.stringify(request.kind)}`); + } + kind = request.kind; + const options: RunNativeVaporScenarioOptions = { + sourceRoot: request.sourceRoot, + harnessRoot: request.harnessRoot, + }; + const result = request.kind === "native" + ? await runNativeInProcess({ + scenario: request.scenario, + executor: "native", + sourceRoot: request.sourceRoot, + harnessRoot: request.harnessRoot, + }, request.events, request.profile) + : request.kind === "correctness" + ? await vaporCorrectnessReplayInProcess(request.scenario, options, request.events, request.profile) + : await vaporMeasurementReplayInProcess(request.scenario, options, request.events, request.profile); + response = { schemaVersion: 1, kind, status: "ok", result }; + } catch (error) { + response = { + schemaVersion: 1, + kind, + status: "error", + reason: error instanceof Error ? error.message : String(error), + }; + } + writeFileSync(responsePath, `${JSON.stringify(response)}\n`); +} + +async function runIsolatedOracleReplay( + kind: VaporOracleReplayKind, + scenario: ScenarioV1, + options: RunNativeVaporScenarioOptions, + events: readonly VaporEvent[], + profile: TargetProfile, +): Promise { + const stageRoot = mkdtempSync(join(tmpdir(), "pocketjs-perf-vapor-replay-")); + const requestPath = join(stageRoot, "request.json"); + const responsePath = join(stageRoot, "response.json"); + const runnerPath = join(stageRoot, "oracle-replay.ts"); + try { + const request: VaporOracleReplayRequest = { + schemaVersion: 1, + kind, + scenario, + sourceRoot: resolve(options.sourceRoot), + harnessRoot: resolve(options.harnessRoot), + events, + profile, + }; + const adapterUrl = new URL(import.meta.url); + adapterUrl.searchParams.set("pocket-perf-oracle-child", basename(stageRoot)); + writeFileSync(requestPath, `${JSON.stringify(request)}\n`); + writeFileSync(runnerPath, ` +import { runVaporOracleReplayChild } from ${JSON.stringify(adapterUrl.href)}; +const requestPath = process.argv[2]; +const responsePath = process.argv[3]; +if (!requestPath || !responsePath) throw new Error("Vapor oracle child paths are required"); +await runVaporOracleReplayChild(requestPath, responsePath); +`); + + const child = Bun.spawn([process.execPath, runnerPath, requestPath, responsePath], { + cwd: resolve(options.harnessRoot), + stdout: "pipe", + stderr: "pipe", + }); + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]); + const diagnostics = [stdout.trim(), stderr.trim()].filter(Boolean).join("\n"); + if (exitCode !== 0) { + throw new Error( + `Vapor ${kind} oracle child exited ${exitCode}${diagnostics ? `: ${diagnostics}` : ""}`, + ); + } + if (!existsSync(responsePath)) { + throw new Error( + `Vapor ${kind} oracle child produced no terminal response${diagnostics ? `: ${diagnostics}` : ""}`, + ); + } + let response: VaporOracleReplayResponse; + try { + response = JSON.parse(readFileSync(responsePath, "utf8")) as VaporOracleReplayResponse; + } catch (error) { + throw new Error( + `Vapor ${kind} oracle child produced a malformed terminal response: ` + + `${error instanceof Error ? error.message : String(error)}`, + ); + } + if (response.schemaVersion !== 1 || response.kind !== kind) { + throw new Error(`Vapor ${kind} oracle child produced an incompatible terminal response`); + } + if (response.status === "error") { + if (typeof response.reason !== "string" || response.reason.length === 0) { + throw new Error(`Vapor ${kind} oracle child produced an incompatible error response`); + } + throw new Error(response.reason); + } + if (response.status !== "ok" || + !Object.prototype.hasOwnProperty.call(response, "result") || + response.result === null || typeof response.result !== "object") { + throw new Error(`Vapor ${kind} oracle child produced an incomplete terminal response`); + } + return response.result as T; + } finally { + rmSync(stageRoot, { recursive: true, force: true }); + } +} + +async function runNative( + options: RunVaporScenarioOptions, + events: readonly VaporEvent[], + profile: TargetProfile, +): Promise { + try { + return await runIsolatedOracleReplay( + "native", + options.scenario, + options, + events, + profile, + ); + } catch (error) { + return invalid(options.executor, [error instanceof Error ? error.message : String(error)]); + } +} + +/** + * Native-runner bridge for the Vapor family. It uses two independent real + * Vue Vapor boots, matching the generic runner's correctness/measurement + * split while keeping all grid and state hashing outside measured phases. + */ +export async function runNativeVaporScenario( + scenario: ScenarioV1, + options: RunNativeVaporScenarioOptions, +): Promise { + const reasons = validateScenario(scenario); + const sourceRoot = resolve(options.sourceRoot); + const profile = profileForEntry(sourceRoot, scenario.subject.entry); + if (!profile) reasons.push(`Vapor oracle has no real Vue entry adapter for ${scenario.subject.entry}`); + else if (!existsSync(profile.benchmarkEntry)) reasons.push(`missing Vapor benchmark source ${profile.benchmarkEntry}`); + for (const path of [ + join(sourceRoot, scenario.subject.entry), + join(sourceRoot, "vapor/compiler/compile.ts"), + join(sourceRoot, "vapor/oracle/boot.ts"), + ]) { + if (!existsSync(path)) reasons.push(`missing Vapor source input ${path}`); + } + const built = buildEvents(scenario); + reasons.push(...built.reasons); + if (reasons.length > 0 || !profile) return nativeUnsupported(scenario, reasons, options.outDir); + + try { + const correctness = await runIsolatedOracleReplay( + "correctness", + scenario, + options, + built.events, + profile, + ); + const measurement = await runIsolatedOracleReplay( + "measurement", + scenario, + options, + built.events, + profile, + ); + if (correctness.finalFramebufferHash !== measurement.finalFramebufferHash) { + throw new Error( + `${scenario.id}: Vapor correctness/measurement final grid diverged: ` + + `${correctness.finalFramebufferHash} != ${measurement.finalFramebufferHash}`, + ); + } + if (correctness.drawListHash !== measurement.finalDrawListHash) { + throw new Error( + `${scenario.id}: Vapor correctness/measurement DrawList diverged: ` + + `${correctness.drawListHash} != ${measurement.finalDrawListHash}`, + ); + } + const diagnosticMetrics: Record = { + "native.boot_wall_time_ns": { value: measurement.bootWallTimeNs, unit: "ns" }, + "native.measured_frames": { + value: scenario.phases + .filter((phase) => phase.collect) + .reduce((sum, phase) => sum + phase.endFrame - phase.startFrame, 0), + unit: "count", + }, + "native.wall_time_ns": { + value: measurement.phases.reduce((sum, phase) => sum + phase.wallTimeNs, 0), + unit: "ns", + }, + }; + for (const phase of measurement.phases) { + diagnosticMetrics[`native.phase.${phase.name}.wall_time_ns`] = { + value: phase.wallTimeNs, + unit: "ns", + }; + } + const requestedGateMetrics = Array.isArray(scenario.params.gateMetrics) + ? scenario.params.gateMetrics.filter((metric): metric is string => typeof metric === "string") + : []; + return nativeResultFile({ + schemaVersion: 1, + kind: "pocketjs.perf.native-result", + status: "ok", + scenarioId: scenario.id, + executor: "native", + sourceRoot, + correctness, + measurement, + diagnosticMetrics, + exactMetrics: {}, + unsupportedMetrics: requestedGateMetrics, + }, options.outDir); + } catch (error) { + return nativeUnsupported( + scenario, + [error instanceof Error ? error.message : String(error)], + options.outDir, + ); + } +} + +function cString(value: string): string { + return JSON.stringify(value).slice(1, -1).replace(/\\u([0-9a-fA-F]{4})/g, "\\u$1"); +} + +function patchedRuntimeHeader(sourceRoot: string): string { + const original = readFileSync(join(sourceRoot, "vapor/runtime/vapor.h"), "utf8"); + const typedefs = [ + "typedef unsigned char u8;", + "typedef unsigned short u16;", + "typedef unsigned long u32;", + "typedef signed char s8;", + "typedef signed short s16;", + "typedef signed long s32;", + ].join("\n"); + if (!original.includes(typedefs)) { + throw new Error("vapor/runtime/vapor.h integer contract changed; update the Linux perf header adaptation"); + } + return original.replace(typedefs, [ + "#include ", + "typedef uint8_t u8;", + "typedef uint16_t u16;", + "typedef uint32_t u32;", + "typedef int8_t s8;", + "typedef int16_t s16;", + "typedef int32_t s32;", + ].join("\n")); +} + +function eventInitializers(events: readonly VaporEvent[]): string { + if (events.length === 0) return " { 0, 0, 0, 0 }"; + return events.map((event) => + ` { ${event.frame}u, ${event.kind === "button" ? "1u" : "2u"}, ${event.control}u, ${event.value} }`, + ).join(",\n"); +} + +function phaseInitializers(scenario: ScenarioV1): string { + const stateFrames = new Set(stateCheckpointFrames(scenario)); + return scenario.phases.filter((phase) => phase.collect).map((phase) => + ` { "${cString(phase.name)}", ${phase.startFrame}u, ${phase.endFrame}u, ${scenarioPhaseId(scenario.id, phase.name)}u, ${stateFrames.has(phase.endFrame - 1) ? "1u" : "0u"} }`, + ).join(",\n"); +} + +function stateCheckpointFrames(scenario: ScenarioV1): number[] { + return scenario.checkpoints + .filter((checkpoint) => checkpoint.capture.includes("state")) + .map((checkpoint) => checkpoint.frame); +} + +function qemuStateCheckpointReasons(scenario: ScenarioV1): string[] { + const phaseEndCounts = new Map(); + for (const phase of scenario.phases.filter((phase) => phase.collect)) { + const frame = phase.endFrame - 1; + phaseEndCounts.set(frame, (phaseEndCounts.get(frame) ?? 0) + 1); + } + return stateCheckpointFrames(scenario).flatMap((frame) => { + const count = phaseEndCounts.get(frame) ?? 0; + return count === 1 ? [] : [ + `Vapor QEMU state checkpoint ${frame} must coincide with exactly one collected phase end so hashing stays outside markers`, + ]; + }); +} + +function debugStateLayoutLength(slots: readonly VaporDebugSlot[]): number { + let end = 0; + let previousEnd = 0; + for (const slot of slots) { + if (!Number.isInteger(slot.offset) || !Number.isInteger(slot.size) || + slot.offset < previousEnd || slot.size <= 0) { + throw new Error(`Vapor compiler returned an invalid debug slot ${slot.name}`); + } + if (slot.kind !== "str" && slot.size !== 4) { + throw new Error(`Vapor compiler returned a non-32-bit ${slot.kind} slot ${slot.name}`); + } + end = Math.max(end, slot.offset + slot.size); + previousEnd = slot.offset + slot.size; + } + const aligned = (end + 3) & ~3; + if (aligned > 4096) throw new Error(`Vapor debug state requires ${aligned} bytes; guest limit is 4096`); + return aligned; +} + +function cDebugStateHashBody(slots: readonly VaporDebugSlot[]): string { + const statements: string[] = []; + slots.forEach((slot, slotIndex) => { + if (slot.kind === "str") { + const capacity = slot.size - 1; + statements.push( + ` { u8 n = state.bytes[${slot.offset}]; u8 j;`, + ` if (n > ${capacity}u) { perf_state_valid = 0; n = ${capacity}u; }`, + ` hash = fnv_byte(hash, n);`, + ` for (j = 0; j < n; j++) hash = fnv_byte(hash, state.bytes[${slot.offset + 1}u + j]);`, + " }", + ); + } else { + statements.push( + ` { u8 j; for (j = 0; j < 4u; j++) hash = fnv_byte(hash, state.bytes[${slot.offset}u + j]); }`, + ); + } + if (slotIndex === slots.length - 1) statements.push(""); + }); + return statements.join("\n").trimEnd(); +} + +function linuxHarnessSource( + scenario: ScenarioV1, + events: readonly VaporEvent[], + debugSlots: readonly VaporDebugSlot[], +): string { + const eventCount = events.length; + const phases = scenario.phases.filter((phase) => phase.collect); + const debugStateLength = debugStateLayoutLength(debugSlots); + const debugStateHashBody = cDebugStateHashBody(debugSlots); + return `/* Generated local performance fixture. */ +#include "vapor.h" +#include "guest_marker.h" + +typedef unsigned long long perf_u64; +typedef unsigned long perf_word; + +u8 vp_grid_ch[VP_GRID_H][VP_GRID_W]; +u8 vp_grid_pal[VP_GRID_H][VP_GRID_W]; + +typedef struct { u32 frame; u8 kind; u8 control; s32 value; } perf_event; +typedef struct { const char *name; u32 start_frame; u32 end_frame; u32 id; u8 capture_state; } perf_phase; + +static const perf_event PERF_EVENTS[${Math.max(1, eventCount)}] = { +${eventInitializers(events)} +}; +static const perf_phase PERF_PHASES[${phases.length}] = { +${phaseInitializers(scenario)} +}; + +static char out_buf[2048]; +static u32 out_len; +static u8 perf_state_valid = 1; +static u8 perf_correctness; + +static long raw_write(const char *buf, u32 len) { +#if defined(__aarch64__) + register long x0 __asm__("x0") = 1; + register const char *x1 __asm__("x1") = buf; + register u32 x2 __asm__("x2") = len; + register long x8 __asm__("x8") = 64; + __asm__ volatile("svc #0" : "+r"(x0) : "r"(x1), "r"(x2), "r"(x8) : "memory", "cc"); + return x0; +#else + register long r0 __asm__("r0") = 1; + register const char *r1 __asm__("r1") = buf; + register u32 r2 __asm__("r2") = len; + register long r7 __asm__("r7") = 4; + __asm__ volatile("svc #0" : "+r"(r0) : "r"(r1), "r"(r2), "r"(r7) : "memory", "cc"); + return r0; +#endif +} + +static __attribute__((noreturn)) void raw_exit(long code) { +#if defined(__aarch64__) + register long x0 __asm__("x0") = code; + register long x8 __asm__("x8") = 93; + __asm__ volatile("svc #0" : : "r"(x0), "r"(x8) : "memory", "cc"); +#else + register long r0 __asm__("r0") = code; + register long r7 __asm__("r7") = 1; + __asm__ volatile("svc #0" : : "r"(r0), "r"(r7) : "memory", "cc"); +#endif + __builtin_unreachable(); +} + +static void out_reset(void) { out_len = 0; } +static void out_char(char c) { if (out_len < sizeof(out_buf)) out_buf[out_len++] = c; } +static void out_text(const char *s) { while (*s) out_char(*s++); } +static void out_u32(u32 value) { + char digits[10]; u8 n = 0; + do { digits[n++] = (char)('0' + value % 10u); value /= 10u; } while (value && n < 10u); + while (n) out_char(digits[--n]); +} +static void out_hex64(perf_u64 value) { + static const char HEX[] = "0123456789abcdef"; s8 shift; + for (shift = 60; shift >= 0; shift -= 4) out_char(HEX[(value >> (u8)shift) & 15u]); +} +static void out_hex_bytes(const u8 *bytes, u32 len) { + static const char HEX[] = "0123456789abcdef"; u32 i; + for (i = 0; i < len; i++) { out_char(HEX[bytes[i] >> 4]); out_char(HEX[bytes[i] & 15u]); } +} +static void out_flush(void) { + u32 sent = 0; + while (sent < out_len) { long n = raw_write(out_buf + sent, out_len - sent); if (n <= 0) raw_exit(70); sent += (u32)n; } +} + +static perf_u64 fnv_byte(perf_u64 hash, u8 byte) { + return (hash ^ (perf_u64)byte) * 0x100000001b3ULL; +} +static perf_u64 fnv_u32(perf_u64 hash, u32 value) { + u8 i; for (i = 0; i < 4; i++) { hash = fnv_byte(hash, (u8)value); value >>= 8; } return hash; +} + +/* Dependency-free SHA-256, used only by the observational correctness replay. */ +typedef struct { + u32 state[8]; + u8 block[64]; + u32 block_len; + perf_u64 byte_len; +} perf_sha256; + +static const u32 PERF_SHA256_K[64] = { + 0x428a2f98u, 0x71374491u, 0xb5c0fbcfu, 0xe9b5dba5u, + 0x3956c25bu, 0x59f111f1u, 0x923f82a4u, 0xab1c5ed5u, + 0xd807aa98u, 0x12835b01u, 0x243185beu, 0x550c7dc3u, + 0x72be5d74u, 0x80deb1feu, 0x9bdc06a7u, 0xc19bf174u, + 0xe49b69c1u, 0xefbe4786u, 0x0fc19dc6u, 0x240ca1ccu, + 0x2de92c6fu, 0x4a7484aau, 0x5cb0a9dcu, 0x76f988dau, + 0x983e5152u, 0xa831c66du, 0xb00327c8u, 0xbf597fc7u, + 0xc6e00bf3u, 0xd5a79147u, 0x06ca6351u, 0x14292967u, + 0x27b70a85u, 0x2e1b2138u, 0x4d2c6dfcu, 0x53380d13u, + 0x650a7354u, 0x766a0abbu, 0x81c2c92eu, 0x92722c85u, + 0xa2bfe8a1u, 0xa81a664bu, 0xc24b8b70u, 0xc76c51a3u, + 0xd192e819u, 0xd6990624u, 0xf40e3585u, 0x106aa070u, + 0x19a4c116u, 0x1e376c08u, 0x2748774cu, 0x34b0bcb5u, + 0x391c0cb3u, 0x4ed8aa4au, 0x5b9cca4fu, 0x682e6ff3u, + 0x748f82eeu, 0x78a5636fu, 0x84c87814u, 0x8cc70208u, + 0x90befffau, 0xa4506cebu, 0xbef9a3f7u, 0xc67178f2u +}; + +static u32 sha_rotr(u32 value, u8 bits) { + return (value >> bits) | (value << (32u - bits)); +} + +static void sha256_compress(perf_sha256 *sha) { + u32 words[64]; u32 i; + u32 a, b, c, d, e, f, g, h; + for (i = 0; i < 16u; i++) { + u32 at = i * 4u; + words[i] = ((u32)sha->block[at] << 24) | ((u32)sha->block[at + 1u] << 16) | + ((u32)sha->block[at + 2u] << 8) | (u32)sha->block[at + 3u]; + } + for (i = 16u; i < 64u; i++) { + u32 s0 = sha_rotr(words[i - 15u], 7) ^ sha_rotr(words[i - 15u], 18) ^ + (words[i - 15u] >> 3); + u32 s1 = sha_rotr(words[i - 2u], 17) ^ sha_rotr(words[i - 2u], 19) ^ + (words[i - 2u] >> 10); + words[i] = words[i - 16u] + s0 + words[i - 7u] + s1; + } + a = sha->state[0]; b = sha->state[1]; c = sha->state[2]; d = sha->state[3]; + e = sha->state[4]; f = sha->state[5]; g = sha->state[6]; h = sha->state[7]; + for (i = 0; i < 64u; i++) { + u32 choice = (e & f) ^ ((~e) & g); + u32 majority = (a & b) ^ (a & c) ^ (b & c); + u32 sum1 = sha_rotr(e, 6) ^ sha_rotr(e, 11) ^ sha_rotr(e, 25); + u32 sum0 = sha_rotr(a, 2) ^ sha_rotr(a, 13) ^ sha_rotr(a, 22); + u32 temp1 = h + sum1 + choice + PERF_SHA256_K[i] + words[i]; + u32 temp2 = sum0 + majority; + h = g; g = f; f = e; e = d + temp1; d = c; c = b; b = a; a = temp1 + temp2; + } + sha->state[0] += a; sha->state[1] += b; sha->state[2] += c; sha->state[3] += d; + sha->state[4] += e; sha->state[5] += f; sha->state[6] += g; sha->state[7] += h; +} + +static void sha256_init(perf_sha256 *sha) { + u8 i; + static const u32 INITIAL[8] = { + 0x6a09e667u, 0xbb67ae85u, 0x3c6ef372u, 0xa54ff53au, + 0x510e527fu, 0x9b05688cu, 0x1f83d9abu, 0x5be0cd19u + }; + for (i = 0; i < 8u; i++) sha->state[i] = INITIAL[i]; + for (i = 0; i < 64u; i++) sha->block[i] = 0; + sha->block_len = 0; sha->byte_len = 0; +} + +static void sha256_update(perf_sha256 *sha, const u8 *bytes, u32 len) { + u32 at = 0; + sha->byte_len += (perf_u64)len; + while (at < len) { + u32 room = 64u - sha->block_len; + u32 take = len - at < room ? len - at : room; + u32 i; + for (i = 0; i < take; i++) sha->block[sha->block_len + i] = bytes[at + i]; + sha->block_len += take; at += take; + if (sha->block_len == 64u) { sha256_compress(sha); sha->block_len = 0; } + } +} + +static void sha256_finish(perf_sha256 *sha, u8 digest[32]) { + perf_u64 bit_len = sha->byte_len * 8u; u32 i; + sha->block[sha->block_len++] = 0x80u; + if (sha->block_len > 56u) { + while (sha->block_len < 64u) sha->block[sha->block_len++] = 0; + sha256_compress(sha); sha->block_len = 0; + } + while (sha->block_len < 56u) sha->block[sha->block_len++] = 0; + for (i = 0; i < 8u; i++) sha->block[63u - i] = (u8)(bit_len >> (i * 8u)); + sha256_compress(sha); + for (i = 0; i < 8u; i++) { + digest[i * 4u] = (u8)(sha->state[i] >> 24); + digest[i * 4u + 1u] = (u8)(sha->state[i] >> 16); + digest[i * 4u + 2u] = (u8)(sha->state[i] >> 8); + digest[i * 4u + 3u] = (u8)sha->state[i]; + } +} + +static u8 sha256_self_test(void) { + static const u8 EXPECTED[32] = { + 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, + 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, + 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, + 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad + }; + static const u8 ABC[3] = { (u8)'a', (u8)'b', (u8)'c' }; + perf_sha256 sha; u8 digest[32]; u8 i; + sha256_init(&sha); sha256_update(&sha, ABC, 3u); sha256_finish(&sha, digest); + for (i = 0; i < 32u; i++) if (digest[i] != EXPECTED[i]) return 0; + return 1; +} + +static perf_sha256 perf_framebuffer_trace; +static u8 perf_framebuffer_trace_digest[32]; + +static perf_u64 grid_hash(void) { + perf_u64 hash = 0xcbf29ce484222325ULL; u16 i; + for (i = 0; i < (u16)(VP_GRID_W * VP_GRID_H); i++) hash = fnv_byte(hash, ((u8 *)vp_grid_ch)[i]); + for (i = 0; i < (u16)(VP_GRID_W * VP_GRID_H); i++) hash = fnv_byte(hash, ((u8 *)vp_grid_pal)[i]); + return hash; +} +static perf_u64 state_hash(void) { + union { perf_u64 align; u8 bytes[4096]; } state; perf_u64 hash = 0xcbf29ce484222325ULL; + u16 len = app_debug_state(state.bytes); + if (len != ${debugStateLength}u) { perf_state_valid = 0; return hash; } +${debugStateHashBody} + return hash; +} +static perf_u64 events_hash(void) { + perf_u64 hash = 0xcbf29ce484222325ULL; u32 i; + for (i = 0; i < ${eventCount}u; i++) { + const perf_event *event = &PERF_EVENTS[i]; + hash = fnv_byte(hash, event->kind); + hash = fnv_u32(hash, event->frame); + hash = fnv_byte(hash, event->control); + if (event->kind == 2u) hash = fnv_u32(hash, (u32)event->value); + } + return hash; +} + +static void emit_phase(const perf_phase *phase) { + out_reset(); + out_text("POCKETJS_PERF_GUEST {\\\"schemaVersion\\\":1,\\\"event\\\":\\\"phase\\\",\\\"scenarioId\\\":\\\"${cString(scenario.id)}\\\",\\\"phase\\\":\\\""); + out_text(phase->name); + out_text("\\\",\\\"phaseId\\\":"); out_u32(phase->id); + out_text(",\\\"iteration\\\":0,\\\"allocCalls\\\":0,\\\"allocatedBytes\\\":0,\\\"currentBytes\\\":0,\\\"peakBytes\\\":0,\\\"quickjsLiveBytesAfterGc\\\":0,\\\"drawListHash\\\":\\\"fnv1a64:"); + out_hex64(grid_hash()); out_text("\\\"}\\n"); out_flush(); +} + +static void emit_state_checkpoint(u32 frame) { + out_reset(); + out_text("POCKETJS_PERF_VAPOR {\\\"schemaVersion\\\":1,\\\"event\\\":\\\"state-checkpoint\\\",\\\"scenarioId\\\":\\\"${cString(scenario.id)}\\\",\\\"frame\\\":"); + out_u32(frame); + out_text(",\\\"stateHash\\\":\\\"fnv1a64:"); out_hex64(state_hash()); + out_text("\\\"}\\n"); out_flush(); +} + +static void emit_complete(void) { + out_reset(); + out_text("POCKETJS_PERF_GUEST {\\\"schemaVersion\\\":1,\\\"event\\\":\\\"complete\\\",\\\"scenarioId\\\":\\\"${cString(scenario.id)}\\\",\\\"suite\\\":\\\"${cString(scenario.suite)}\\\",\\\"framework\\\":\\\"core\\\",\\\"finalDrawListHash\\\":\\\"fnv1a64:"); + out_hex64(grid_hash()); out_text("\\\",\\\"finalStateHash\\\":\\\"fnv1a64:"); + out_hex64(state_hash()); out_text("\\\",\\\"effectHash\\\":\\\"fnv1a64:"); + out_hex64(events_hash()); + if (perf_correctness) { + out_text("\\\",\\\"framebufferTraceHash\\\":\\\""); + out_hex_bytes(perf_framebuffer_trace_digest, 32u); + } + out_text("\\\"}\\n"); out_flush(); +} + +static int perf_main(void) { + u32 frame, event_at = 0, phase_at = 0; u16 cell; + for (cell = 0; cell < (u16)(VP_GRID_W * VP_GRID_H); cell++) { + ((u8 *)vp_grid_ch)[cell] = (u8)' '; ((u8 *)vp_grid_pal)[cell] = 0; + } + if (perf_correctness) { + if (!sha256_self_test()) return 78; + sha256_init(&perf_framebuffer_trace); + } + app_init(); + for (frame = 0; frame < ${scenario.frames}u; frame++) { + if (phase_at < ${phases.length}u && PERF_PHASES[phase_at].start_frame == frame) { + long marker_result = pocketjs_perf_begin(PERF_PHASES[phase_at].id, 0); + if (perf_correctness) { + if (marker_result != -38) return 76; + } else if (marker_result != 0) return 71; + } + while (event_at < ${eventCount}u && PERF_EVENTS[event_at].frame == frame) { + const perf_event *event = &PERF_EVENTS[event_at++]; + if (event->kind == 1u) app_on_button(event->control); + else app_on_axis_delta(event->control, event->value); + } + (void)app_flush(); + if (perf_correctness) { + sha256_update(&perf_framebuffer_trace, (const u8 *)vp_grid_ch, (u32)(VP_GRID_W * VP_GRID_H)); + sha256_update(&perf_framebuffer_trace, (const u8 *)vp_grid_pal, (u32)(VP_GRID_W * VP_GRID_H)); + } + if (phase_at < ${phases.length}u && PERF_PHASES[phase_at].end_frame == frame + 1u) { + const perf_phase *phase = &PERF_PHASES[phase_at]; + if (!perf_correctness && pocketjs_perf_end(phase->id, 0) != 0) return 72; + emit_phase(phase); + if (phase->capture_state) emit_state_checkpoint(frame); + phase_at++; + } + } + if (event_at != ${eventCount}u || phase_at != ${phases.length}u) return 73; + if (perf_correctness) sha256_finish(&perf_framebuffer_trace, perf_framebuffer_trace_digest); + emit_complete(); + if (vp_tripwires != 0) return 74; + return perf_state_valid ? 0 : 75; +} + +static u8 text_equal(const char *left, const char *right) { + if (!left || !right) return 0; + while (*left && *right && *left == *right) { left++; right++; } + return (u8)(*left == *right); +} + +static __attribute__((used, noreturn, noinline)) void perf_start(const perf_word *stack) { + u32 argc = (u32)stack[0]; + if (argc == 2u && text_equal((const char *)(perf_word)stack[2], "--correctness")) { + perf_correctness = 1; + } else if (argc != 1u) { + raw_exit(77); + } + raw_exit(perf_main()); +} + +#if defined(__aarch64__) +__asm__( + ".pushsection .text.start,\\\"ax\\\",%progbits\\n" + ".align 2\\n" + ".global _start\\n" + ".type _start, %function\\n" + "_start:\\n" + "mov x0, sp\\n" + "b perf_start\\n" + ".size _start, . - _start\\n" + ".popsection\\n" +); +#else +__attribute__((naked, noreturn)) void _start(void) { + __asm__("mov r0, sp\\n\\tb perf_start"); +} +#endif +`; +} + +export interface PreparedVaporQemuFixture { + readonly directory: string; + readonly generatedApp: string; + readonly runtimeCore: string; + readonly runtimeHeader: string; + readonly guestHarness: string; + readonly elfPath: string; + readonly build: VaporQemuBuildSpec; + readonly profile: TargetProfile; +} + +/** Materialize only deterministic generated inputs; compilation stays in the pinned image. */ +export async function prepareVaporQemuFixture( + options: RunVaporScenarioOptions, + eventsInput?: readonly VaporEvent[], +): Promise { + if (options.executor === "native") throw new Error("native has no QEMU fixture"); + const sourceRoot = resolve(options.sourceRoot); + const harnessRoot = resolve(options.harnessRoot); + const profile = profileForEntry(sourceRoot, options.scenario.subject.entry); + if (!profile) throw new Error(`Vapor oracle has no real Vue entry adapter for ${options.scenario.subject.entry}`); + const eventsResult = eventsInput ? { events: [...eventsInput], reasons: [] } : buildEvents(options.scenario); + if (eventsResult.reasons.length > 0) throw new Error(eventsResult.reasons.join("; ")); + const checkpointReasons = qemuStateCheckpointReasons(options.scenario); + if (checkpointReasons.length > 0) throw new Error(checkpointReasons.join("; ")); + const directory = resolve(options.outDir ?? mkdtempSync(join(tmpdir(), "pocketjs-perf-vapor-"))); + mkdirSync(directory, { recursive: true }); + + const compiled = await withSourceDependencies(sourceRoot, harnessRoot, async () => { + const compilerUrl = pathToFileURL(join(sourceRoot, "vapor/compiler/compile.ts")); + compilerUrl.searchParams.set("pocket-perf-qemu", basename(sourceRoot)); + const compiler = await import(compilerUrl.href) as { + compileVaporApp(fileName: string, source: string, title: string, target: VaporTargetName): CompiledVaporAppLike; + }; + return compiler.compileVaporApp( + profile.benchmarkEntry, + benchmarkFixtureSource(profile), + "PERF AXIS", + profile.target, + ); + }); + + const generatedApp = join(directory, "gen_app.c"); + const runtimeCore = join(directory, "vapor_core.c"); + const runtimeHeader = join(directory, "vapor.h"); + const guestHarness = join(directory, "vapor_perf_guest.c"); + const elfPath = join(directory, `vapor-${options.executor}.elf`); + writeFileSync(generatedApp, compiled.c); + writeFileSync(runtimeCore, readFileSync(join(sourceRoot, "vapor/runtime/vapor_core.c"))); + writeFileSync(runtimeHeader, patchedRuntimeHeader(sourceRoot)); + writeFileSync(guestHarness, linuxHarnessSource( + options.scenario, + eventsResult.events, + compiled.debugSlots, + )); + return { + directory, + generatedApp, + runtimeCore, + runtimeHeader, + guestHarness, + elfPath, + build: VAPOR_QEMU_BUILD_SPECS[options.executor], + profile, + }; +} + +interface ProcessResult { + readonly exitCode: number; + readonly stdout: string; + readonly stderr: string; +} + +function command(argv: readonly string[], cwd: string): ProcessResult { + const child = Bun.spawnSync(argv as string[], { cwd, stdout: "pipe", stderr: "pipe" }); + return { + exitCode: child.exitCode, + stdout: new TextDecoder().decode(child.stdout), + stderr: new TextDecoder().decode(child.stderr), + }; +} + +function dockerCommand(image: string, directory: string, argv: readonly string[]): ProcessResult { + return command([ + "docker", "run", "--rm", + "--volume", `${directory}:/work`, + "--workdir", "/work", + image, + ...argv, + ], directory); +} + +function combinedProcessOutput(result: ProcessResult): string { + return `${result.stdout}${result.stdout && result.stderr ? "\n" : ""}${result.stderr}`; +} + +function elfTextRodata(sizeOutput: string): number { + let total = 0; + for (const line of sizeOutput.split(/\r?\n/)) { + const match = /^\s*(\.text(?:\.[^\s]+)?|\.rodata(?:\.[^\s]+)?)\s+(\d+)\b/.exec(line); + if (match) total += Number(match[2]); + } + if (!Number.isSafeInteger(total) || total <= 0) throw new Error("ELF size output has no .text/.rodata bytes"); + return total; +} + +const VAPOR_STATE_OUTPUT_PREFIX = "POCKETJS_PERF_VAPOR "; +const FNV1A64_DIGEST = /^fnv1a64:[a-f0-9]{16}$/; + +interface VaporStateCheckpointRecord { + readonly schemaVersion: 1; + readonly event: "state-checkpoint"; + readonly scenarioId: string; + readonly frame: number; + readonly stateHash: string; +} + +function parseVaporStateCheckpoints(output: string): { + readonly records: readonly VaporStateCheckpointRecord[]; + readonly reasons: readonly string[]; +} { + const records: VaporStateCheckpointRecord[] = []; + const reasons: string[] = []; + for (const [index, line] of output.split(/\r?\n/u).entries()) { + if (!line.startsWith(VAPOR_STATE_OUTPUT_PREFIX)) continue; + const label = `Vapor state line ${index + 1}`; + let value: unknown; + try { + value = JSON.parse(line.slice(VAPOR_STATE_OUTPUT_PREFIX.length)); + } catch (error) { + reasons.push(`${label}: invalid JSON (${error instanceof Error ? error.message : String(error)})`); + continue; + } + if (typeof value !== "object" || value === null || Array.isArray(value)) { + reasons.push(`${label}: protocol value is not an object`); + continue; + } + const record = value as Record; + const required = ["schemaVersion", "event", "scenarioId", "frame", "stateHash"] as const; + const unknown = Object.keys(record).filter((key) => !required.includes(key as typeof required[number])); + const missing = required.filter((key) => !Object.hasOwn(record, key)); + if (unknown.length > 0 || missing.length > 0) { + if (unknown.length > 0) reasons.push(`${label}: unknown properties: ${unknown.join(", ")}`); + if (missing.length > 0) reasons.push(`${label}: missing properties: ${missing.join(", ")}`); + continue; + } + if (record.schemaVersion !== 1 || record.event !== "state-checkpoint" || + typeof record.scenarioId !== "string" || record.scenarioId.length === 0 || + typeof record.frame !== "number" || !Number.isSafeInteger(record.frame) || record.frame < 0 || + typeof record.stateHash !== "string" || !FNV1A64_DIGEST.test(record.stateHash)) { + reasons.push(`${label}: invalid state-checkpoint record`); + continue; + } + records.push(record as unknown as VaporStateCheckpointRecord); + } + return { records, reasons }; +} + +function generatedCReplayShapeReasons( + label: "correctness" | "measurement", + scenario: ScenarioV1, + guest: GuestProtocolResult, + states: ReturnType, +): string[] { + const reasons = states.reasons.map((reason) => `${label}: ${reason}`); + const expectedPhases = scenario.phases.filter((phase) => phase.collect); + if (guest.phases.length !== expectedPhases.length) { + reasons.push(`${label} emitted ${guest.phases.length} phases; expected ${expectedPhases.length}`); + } + guest.phases.forEach((phase, index) => { + const expected = expectedPhases[index]; + if (!expected || phase.scenarioId !== scenario.id || phase.phase !== expected.name || + phase.phaseId !== scenarioPhaseId(scenario.id, expected.name) || phase.iteration !== 0) { + reasons.push(`${label} phase ${index} identity differs from scenario`); + } + }); + if (guest.complete && + (guest.complete.scenarioId !== scenario.id || guest.complete.suite !== scenario.suite || + guest.complete.framework !== scenario.subject.framework)) { + reasons.push(`${label} complete identity differs from scenario`); + } + const expectedStateFrames = stateCheckpointFrames(scenario); + if (states.records.length !== expectedStateFrames.length) { + reasons.push( + `${label} emitted ${states.records.length} state checkpoints; expected ${expectedStateFrames.length}`, + ); + } + states.records.forEach((record, index) => { + if (record.scenarioId !== scenario.id || record.frame !== expectedStateFrames[index]) { + reasons.push(`${label} state checkpoint ${index} identity differs from scenario`); + } + }); + const finalPhaseIndex = expectedPhases.findIndex((phase) => phase.endFrame === scenario.frames); + if (finalPhaseIndex >= 0 && guest.complete && guest.phases[finalPhaseIndex] && + guest.phases[finalPhaseIndex]!.drawListHash !== guest.complete.finalDrawListHash) { + reasons.push(`${label} final phase DrawList differs from complete`); + } + return reasons; +} + +/** + * Validate the two generated-C executions before measurement diagnostics are + * discarded. The correctness run owns all observational hashes and must be + * plugin-free; the measurement run owns only the QEMU counter stream. + */ +export function vaporQemuReplayReasons( + scenario: ScenarioV1, + correctnessOutput: string, + measurementOutput: string, +): { + readonly correctness: GuestProtocolResult; + readonly measurement: GuestProtocolResult; + readonly qemu: QemuProtocolResult; + readonly reasons: string[]; +} { + const correctness = parseGuestOutput(correctnessOutput, { framebufferTraceHash: "required" }); + const measurement = parseGuestOutput(measurementOutput, { framebufferTraceHash: "forbidden" }); + const qemu = parseQemuOutput(measurementOutput); + const correctnessStates = parseVaporStateCheckpoints(correctnessOutput); + const measurementStates = parseVaporStateCheckpoints(measurementOutput); + const reasons: string[] = [ + ...(correctness.status === "invalid" + ? correctness.reasons.map((reason) => `correctness: ${reason}`) + : []), + ...(measurement.status === "invalid" + ? measurement.reasons.map((reason) => `measurement: ${reason}`) + : []), + ...(qemu.status === "invalid" ? qemu.reasons.map((reason) => `measurement: ${reason}`) : []), + ...generatedCReplayShapeReasons("correctness", scenario, correctness, correctnessStates), + ...generatedCReplayShapeReasons("measurement", scenario, measurement, measurementStates), + ]; + + if (correctnessOutput.split(/\r?\n/u).some((line) => line.startsWith(QEMU_OUTPUT_PREFIX))) { + reasons.push("correctness replay emitted QEMU plugin records"); + } + if (correctness.phases.length !== measurement.phases.length) { + reasons.push("correctness and measurement emitted different phase counts"); + } + for (let index = 0; index < Math.max(correctness.phases.length, measurement.phases.length); index += 1) { + const left = correctness.phases[index]; + const right = measurement.phases[index]; + if (!left || !right) continue; + if (left.scenarioId !== right.scenarioId || left.phase !== right.phase || + left.phaseId !== right.phaseId || left.iteration !== right.iteration) { + reasons.push(`correctness/measurement phase ${index} identity differs`); + } + if (left.drawListHash !== right.drawListHash) { + reasons.push(`correctness/measurement DrawList differs after phase ${left.phase}`); + } + } + + if (correctnessStates.records.length !== measurementStates.records.length) { + reasons.push("correctness and measurement emitted different state checkpoint counts"); + } + for (let index = 0; + index < Math.max(correctnessStates.records.length, measurementStates.records.length); + index += 1) { + const left = correctnessStates.records[index]; + const right = measurementStates.records[index]; + if (!left || !right) continue; + if (left.scenarioId !== right.scenarioId || left.frame !== right.frame) { + reasons.push(`correctness/measurement state checkpoint ${index} identity differs`); + } + if (left.stateHash !== right.stateHash) { + reasons.push(`correctness/measurement state differs at checkpoint ${left.frame}`); + } + } + + if (correctness.complete && measurement.complete) { + if (correctness.complete.scenarioId !== measurement.complete.scenarioId || + correctness.complete.suite !== measurement.complete.suite || + correctness.complete.framework !== measurement.complete.framework) { + reasons.push("correctness/measurement complete identity differs"); + } + if (correctness.complete.finalDrawListHash !== measurement.complete.finalDrawListHash) { + reasons.push("correctness/measurement final DrawList differs"); + } + if (correctness.complete.finalStateHash !== measurement.complete.finalStateHash) { + reasons.push("correctness/measurement final state differs"); + } + if (correctness.complete.effectHash !== measurement.complete.effectHash) { + reasons.push("correctness/measurement effects differ"); + } + } + + if (qemu.measurements.length !== measurement.phases.length) { + reasons.push( + `measurement QEMU emitted ${qemu.measurements.length} phases; ` + + `guest emitted ${measurement.phases.length}`, + ); + } + for (let index = 0; index < Math.max(qemu.measurements.length, measurement.phases.length); index += 1) { + const counter = qemu.measurements[index]; + const phase = measurement.phases[index]; + if (!counter || !phase) continue; + if (counter.phase_id !== phase.phaseId || counter.iteration !== phase.iteration) { + reasons.push(`measurement QEMU phase ${index} identity differs from guest phase`); + } + if (counter.vcpu !== 0) reasons.push(`measurement QEMU phase ${index} used unexpected vCPU ${counter.vcpu}`); + if (qemu.terminal?.event === "complete" && counter.target !== qemu.terminal.target) { + reasons.push(`measurement QEMU phase ${index} target differs from terminal`); + } + } + return { correctness, measurement, qemu, reasons: unique(reasons) }; +} + +/** Strictly bind generated-C state observations to the independent Vue Vapor oracle. */ +export function vaporGuestStateParityReasons( + scenario: ScenarioV1, + output: string, + guestFinalStateHash: string | null, + native: Pick, +): string[] { + const parsed = parseVaporStateCheckpoints(output); + const reasons = [...parsed.reasons]; + const expectedFrames = stateCheckpointFrames(scenario); + if (parsed.records.length !== expectedFrames.length) { + reasons.push( + `generated-C emitted ${parsed.records.length} state checkpoints; expected ${expectedFrames.length}`, + ); + } + const seen = new Set(); + parsed.records.forEach((record, index) => { + const expectedFrame = expectedFrames[index]; + if (record.scenarioId !== scenario.id) { + reasons.push(`generated-C state checkpoint ${record.frame} has a different scenarioId`); + } + if (record.frame >= scenario.frames) { + reasons.push(`generated-C state checkpoint ${record.frame} is outside the scenario`); + } + if (seen.has(record.frame)) reasons.push(`generated-C state checkpoint ${record.frame} is duplicated`); + seen.add(record.frame); + if (expectedFrame !== undefined && record.frame !== expectedFrame) { + reasons.push( + `generated-C state checkpoint ${index} is frame ${record.frame}; expected ${expectedFrame}`, + ); + } + const expectedHash = native.checkpointStateDigests[String(record.frame)]; + if (!expectedHash) { + reasons.push(`Vue Vapor oracle has no state checkpoint ${record.frame}`); + } else if (record.stateHash !== expectedHash) { + reasons.push(`generated-C state differs from Vue Vapor oracle at checkpoint ${record.frame}`); + } + }); + for (const frame of expectedFrames) { + if (!Object.hasOwn(native.checkpointStateDigests, String(frame))) { + reasons.push(`Vue Vapor oracle omitted declared state checkpoint ${frame}`); + } + } + if (!guestFinalStateHash) { + reasons.push("generated-C did not emit a final state hash"); + } else if (guestFinalStateHash !== native.finalStateDigest) { + reasons.push("generated-C final state differs from Vue Vapor oracle"); + } + const finalCheckpoint = parsed.records.find((record) => record.frame === scenario.frames - 1); + if (finalCheckpoint && guestFinalStateHash && finalCheckpoint.stateHash !== guestFinalStateHash) { + reasons.push("generated-C final state differs between checkpoint and complete records"); + } + return unique(reasons); +} + +async function runQemu( + options: RunVaporScenarioOptions & { readonly executor: Exclude }, + events: readonly VaporEvent[], + profile: TargetProfile, +): Promise { + const checkpointReasons = qemuStateCheckpointReasons(options.scenario); + if (checkpointReasons.length > 0) return invalid(options.executor, checkpointReasons); + const native = await runNative({ ...options, executor: "native" }, events, profile); + if (native.status !== "ok") return invalid(options.executor, native.reasons); + + let fixture: PreparedVaporQemuFixture; + try { + fixture = await prepareVaporQemuFixture(options, events); + } catch (error) { + return invalid(options.executor, [error instanceof Error ? error.message : String(error)]); + } + const image = options.image ?? "pocketjs-perf-qemu:11.0.3"; + const defines = [ + `-DVP_GRID_W=${fixture.profile.width}`, + `-DVP_GRID_H=${fixture.profile.height}`, + `-DVP_STR_CAP=${fixture.profile.strCap}`, + `-DVP_VIEW_CAP=${fixture.profile.poolCap}`, + ]; + const compile = dockerCommand(image, fixture.directory, [ + fixture.build.compiler, + ...fixture.build.cFlags, + ...defines, + "-I/work", + "-I/opt/pocketjs-perf-qemu", + "/work/gen_app.c", + "/work/vapor_core.c", + "/work/vapor_perf_guest.c", + ...fixture.build.linkerFlags, + "-o", + `/work/${basename(fixture.elfPath)}`, + ]); + if (compile.exitCode !== 0) { + return invalid(options.executor, [ + `Vapor ${options.executor} compile failed (${compile.exitCode})`, + compile.stderr.trim() || compile.stdout.trim(), + ]); + } + + const correctnessRun = dockerCommand(image, fixture.directory, [ + fixture.build.emulator, + ...fixture.build.cpuArgs, + ...fixture.build.emulatorArgs, + `/work/${basename(fixture.elfPath)}`, + "--correctness", + ]); + const measurementRun = dockerCommand(image, fixture.directory, [ + fixture.build.emulator, + ...fixture.build.cpuArgs, + ...fixture.build.emulatorArgs, + "-d", "plugin", + "-plugin", "/opt/pocketjs-perf-qemu/build/pocketjs-perf-counter.so", + `/work/${basename(fixture.elfPath)}`, + ]); + const correctnessOutput = combinedProcessOutput(correctnessRun); + const measurementOutput = combinedProcessOutput(measurementRun); + writeFileSync(join(fixture.directory, "correctness.log"), correctnessOutput); + writeFileSync(join(fixture.directory, "measurement-replay.log"), measurementOutput); + const replay = vaporQemuReplayReasons(options.scenario, correctnessOutput, measurementOutput); + const reasons: string[] = []; + if (correctnessRun.exitCode !== 0) { + reasons.push(`Vapor ${options.executor} correctness guest failed (${correctnessRun.exitCode})`); + } + if (measurementRun.exitCode !== 0) { + reasons.push(`Vapor ${options.executor} measurement guest failed (${measurementRun.exitCode})`); + } + reasons.push(...replay.reasons); + for (const [label, output, guest] of [ + ["correctness", correctnessOutput, replay.correctness], + ["measurement", measurementOutput, replay.measurement], + ] as const) { + reasons.push(...vaporGuestStateParityReasons( + options.scenario, + output, + guest.complete?.finalStateHash ?? null, + native, + ).map((reason) => `${label}: ${reason}`)); + if (guest.complete) { + if (guest.complete.finalDrawListHash !== native.finalDrawListHash) { + reasons.push( + `${label}: generated-C final grid ${guest.complete.finalDrawListHash} ` + + `differs from Vue Vapor oracle ${native.finalDrawListHash}`, + ); + } + if (guest.complete.effectHash !== native.finalEffectDigest) { + reasons.push(`${label}: generated-C delivered-event digest differs from the Vue Vapor oracle`); + } + } + for (const phase of guest.phases) { + const expected = native.phaseDrawListHashes[phase.phase]; + if (!expected) { + reasons.push(`${label}: Vue Vapor oracle has no phase ${phase.phase}`); + } else if (phase.drawListHash !== expected) { + reasons.push(`${label}: generated-C grid differs from Vue Vapor oracle after phase ${phase.phase}`); + } + } + } + const correctnessTrace = replay.correctness.complete?.framebufferTraceHash; + if (correctnessTrace && correctnessTrace !== native.framebufferHash) { + reasons.push("generated-C correctness framebuffer trace differs from the Vue Vapor oracle"); + } + for (const measurement of replay.qemu.measurements) { + if (measurement.target !== fixture.build.qemuTarget) { + reasons.push(`measurement QEMU target ${measurement.target} differs from ${fixture.build.qemuTarget}`); + } + } + if (replay.qemu.terminal?.event === "complete" && + replay.qemu.terminal.target !== fixture.build.qemuTarget) { + reasons.push(`measurement QEMU terminal target differs from ${fixture.build.qemuTarget}`); + } + const diagnosticOutput = [ + "--- generated-C correctness replay ---", + correctnessOutput, + "--- generated-C measurement replay ---", + measurementOutput, + ].join("\n"); + if (reasons.length > 0) return invalid(options.executor, reasons, diagnosticOutput); + + const protocolOutput = [ + ...correctnessOutput.split(/\r?\n/u).filter((line) => + line.startsWith(GUEST_OUTPUT_PREFIX) || line.startsWith(VAPOR_STATE_OUTPUT_PREFIX) + ), + ...measurementOutput.split(/\r?\n/u).filter((line) => line.startsWith(QEMU_OUTPUT_PREFIX)), + ].join("\n") + "\n"; + + const sizeTool = fixture.build.compiler.replace(/gcc$/, "size"); + const size = dockerCommand(image, fixture.directory, [sizeTool, "-A", `/work/${basename(fixture.elfPath)}`]); + if (size.exitCode !== 0) { + return invalid(options.executor, [ + `cannot inspect Vapor ELF sections (${size.exitCode})`, + size.stderr.trim() || size.stdout.trim(), + ], diagnosticOutput); + } + let textRodata: number; + try { + textRodata = elfTextRodata(size.stdout); + } catch (error) { + return invalid(options.executor, [error instanceof Error ? error.message : String(error)], diagnosticOutput); + } + return { + status: "ok", + executor: options.executor, + combinedOutput: protocolOutput, + framebufferHash: correctnessTrace!, + // Receipts use the richer DOM hash; accepting it is safe only because the + // generated-C final/checkpoint debug state was matched above. + stateHash: native.stateHash, + effectHash: native.effectHash, + finalDrawListHash: replay.correctness.complete!.finalDrawListHash, + elfPath: fixture.elfPath, + artifactMetrics: { "artifact.elf_text_rodata_bytes": textRodata }, + build: fixture.build, + }; +} + +/** + * Run the real Vue Vapor correctness oracle or its allocation-free generated-C + * Linux guest. Relative-axis samples stay on RelativeAxis/onAxisDelta and are + * never translated into button presses. + */ +export async function runVaporScenario(options: RunVaporScenarioOptions): Promise { + const reasons = validateScenario(options.scenario); + const sourceRoot = resolve(options.sourceRoot); + const profile = profileForEntry(sourceRoot, options.scenario.subject.entry); + if (!profile) reasons.push(`Vapor oracle has no real Vue entry adapter for ${options.scenario.subject.entry}`); + else if (!existsSync(profile.benchmarkEntry)) reasons.push(`missing Vapor benchmark source ${profile.benchmarkEntry}`); + for (const path of [ + join(sourceRoot, options.scenario.subject.entry), + join(sourceRoot, "vapor/compiler/compile.ts"), + join(sourceRoot, "vapor/oracle/boot.ts"), + join(sourceRoot, "vapor/runtime/vapor.h"), + join(sourceRoot, "vapor/runtime/vapor_core.c"), + ]) { + if (!existsSync(path)) reasons.push(`missing Vapor source input ${path}`); + } + const built = buildEvents(options.scenario); + reasons.push(...built.reasons); + if (reasons.length > 0 || !profile) return invalid(options.executor, reasons); + if (options.executor === "native") return await runNative(options, built.events, profile); + return await runQemu( + options as RunVaporScenarioOptions & { readonly executor: Exclude }, + built.events, + profile, + ); +} diff --git a/tools/perf/guest/.gitignore b/tools/perf/guest/.gitignore new file mode 100644 index 00000000..b83d2226 --- /dev/null +++ b/tools/perf/guest/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/tools/perf/guest/Cargo.lock b/tools/perf/guest/Cargo.lock new file mode 100644 index 00000000..f289029e --- /dev/null +++ b/tools/perf/guest/Cargo.lock @@ -0,0 +1,286 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "cc" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "grid" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b40ca9252762c466af32d0b1002e91e4e1bc5398f77455e55474deb466355ff5" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "pocket-mod" +version = "0.1.0" +dependencies = [ + "anyhow", + "log", + "pocketjs-core", + "rquickjs", +] + +[[package]] +name = "pocket-ui-surface" +version = "0.1.0" +dependencies = [ + "anyhow", + "log", + "pocket-mod", + "pocketjs-core", +] + +[[package]] +name = "pocketjs-core" +version = "0.1.0" +dependencies = [ + "taffy", +] + +[[package]] +name = "pocketjs-perf-guest" +version = "0.1.0" +dependencies = [ + "anyhow", + "libc", + "pocket-mod", + "pocket-ui-surface", + "pocketjs-core", + "rquickjs", + "serde", + "serde_json", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "relative-path" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bca40a312222d8ba74837cb474edef44b37f561da5f773981007a10bbaa992b0" +dependencies = [ + "serde", +] + +[[package]] +name = "rquickjs" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0688f8b0192998cca685adefdfad3483da295fa40a0ec406b4c14ecd729e858" +dependencies = [ + "rquickjs-core", +] + +[[package]] +name = "rquickjs-core" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e4f499ac5b943d97ee6dbc44f23c2c10426f420f7d2f1793d6318911b6608c" +dependencies = [ + "hashbrown", + "relative-path", + "rquickjs-sys", +] + +[[package]] +name = "rquickjs-sys" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13ac243b86a74120814ef7e9e30ad5a2c1199b7b9963b1cf7c84e4cdc1cad99" +dependencies = [ + "cc", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slotmap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" +dependencies = [ + "version_check", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "taffy" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfde4e2f8595f222ceaae1fb16b4963952e9b33e358869dc4cd6316b0e0790cd" +dependencies = [ + "arrayvec", + "grid", + "serde", + "slotmap", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tools/perf/guest/Cargo.toml b/tools/perf/guest/Cargo.toml new file mode 100644 index 00000000..30c5a8a9 --- /dev/null +++ b/tools/perf/guest/Cargo.toml @@ -0,0 +1,27 @@ +[workspace] + +[package] +name = "pocketjs-perf-guest" +version = "0.1.0" +edition = "2024" +publish = false + +[dependencies] +anyhow = "1" +libc = "0.2" +pocket-mod = { path = "../../../engine/crates/pocket-mod" } +pocket-ui-surface = { path = "../../../engine/crates/pocket-ui-surface" } +pocketjs-core = { path = "../../../engine/core", features = ["std"] } +# Enable the production QuickJS crate's Rust allocator adapter so the global +# benchmark allocator observes both Rust and QuickJS allocations. The QEMU +# staging build resolves this dependency from the measured source lockfile. +rquickjs = { version = "0.12", features = ["rust-alloc"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[profile.release] +opt-level = 3 +lto = "thin" +codegen-units = 1 +panic = "abort" +strip = "debuginfo" diff --git a/tools/perf/guest/src/main.rs b/tools/perf/guest/src/main.rs new file mode 100644 index 00000000..3f0063d7 --- /dev/null +++ b/tools/perf/guest/src/main.rs @@ -0,0 +1,1344 @@ +use std::alloc::{GlobalAlloc, Layout, System}; +use std::collections::{BTreeMap, HashMap}; +use std::fs; +use std::mem::MaybeUninit; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use anyhow::{Context as _, Result, anyhow, bail}; +use pocket_mod::Guest; +use pocket_mod::qjs::{Array, CatchResultExt, Function}; +use pocket_ui_surface::UiSurface; +use pocketjs_core::{raster, spec::btn}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +const GUEST_PREFIX: &str = "POCKETJS_PERF_GUEST "; +const MARKER_SYSCALL: u32 = 4096; +const MARKER_MAGIC: u32 = 0x504a_424d; +const MARKER_VERSION: u32 = 1; +const MARKER_COOKIE: u32 = 0xc001_c0de; +const MARKER_BEGIN: u32 = 1; +const MARKER_END: u32 = 2; + +#[cfg(any(target_os = "linux", test))] +fn fill_deterministic_random(call: u64, bytes: &mut [u8]) { + for (index, byte) in bytes.iter_mut().enumerate() { + *byte = (call as u8) + .wrapping_mul(0x9d) + .wrapping_add(index as u8) + .wrapping_add(0x53); + } +} + +#[cfg(all(target_os = "linux", not(test)))] +static DETERMINISTIC_RANDOM_CALLS: AtomicU64 = AtomicU64::new(0); + +/// Rust's Linux `RandomState` initialization resolves this weak C symbol +/// before falling back to the guest `getrandom(2)` syscall. This symbol is +/// benchmark-only: it fixes benchmark entropy while preserving the workload's +/// inputs and logic. The guest is single-threaded, so the call-indexed stream +/// is reproducible. +#[cfg(all(target_os = "linux", not(test)))] +#[unsafe(no_mangle)] +unsafe extern "C" fn getrandom( + buffer: *mut libc::c_void, + length: libc::size_t, + _flags: libc::c_uint, +) -> libc::ssize_t { + if length == 0 { + return 0; + } + if buffer.is_null() || length > isize::MAX as usize { + return -1; + } + let call = DETERMINISTIC_RANDOM_CALLS.fetch_add(1, Ordering::Relaxed); + let bytes = unsafe { std::slice::from_raw_parts_mut(buffer.cast::(), length) }; + fill_deterministic_random(call, bytes); + length as libc::ssize_t +} + +const SHA256_INITIAL_STATE: [u32; 8] = [ + 0x6a09_e667, + 0xbb67_ae85, + 0x3c6e_f372, + 0xa54f_f53a, + 0x510e_527f, + 0x9b05_688c, + 0x1f83_d9ab, + 0x5be0_cd19, +]; + +const SHA256_ROUND_CONSTANTS: [u32; 64] = [ + 0x428a_2f98, + 0x7137_4491, + 0xb5c0_fbcf, + 0xe9b5_dba5, + 0x3956_c25b, + 0x59f1_11f1, + 0x923f_82a4, + 0xab1c_5ed5, + 0xd807_aa98, + 0x1283_5b01, + 0x2431_85be, + 0x550c_7dc3, + 0x72be_5d74, + 0x80de_b1fe, + 0x9bdc_06a7, + 0xc19b_f174, + 0xe49b_69c1, + 0xefbe_4786, + 0x0fc1_9dc6, + 0x240c_a1cc, + 0x2de9_2c6f, + 0x4a74_84aa, + 0x5cb0_a9dc, + 0x76f9_88da, + 0x983e_5152, + 0xa831_c66d, + 0xb003_27c8, + 0xbf59_7fc7, + 0xc6e0_0bf3, + 0xd5a7_9147, + 0x06ca_6351, + 0x1429_2967, + 0x27b7_0a85, + 0x2e1b_2138, + 0x4d2c_6dfc, + 0x5338_0d13, + 0x650a_7354, + 0x766a_0abb, + 0x81c2_c92e, + 0x9272_2c85, + 0xa2bf_e8a1, + 0xa81a_664b, + 0xc24b_8b70, + 0xc76c_51a3, + 0xd192_e819, + 0xd699_0624, + 0xf40e_3585, + 0x106a_a070, + 0x19a4_c116, + 0x1e37_6c08, + 0x2748_774c, + 0x34b0_bcb5, + 0x391c_0cb3, + 0x4ed8_aa4a, + 0x5b9c_ca4f, + 0x682e_6ff3, + 0x748f_82ee, + 0x78a5_636f, + 0x84c8_7814, + 0x8cc7_0208, + 0x90be_fffa, + 0xa450_6ceb, + 0xbef9_a3f7, + 0xc671_78f2, +]; + +/// Small, dependency-free SHA-256 used only by the observational correctness +/// replay. Keeping the implementation in this fingerprinted harness avoids a +/// separately resolved crypto dependency becoming part of the executor. +struct Sha256 { + state: [u32; 8], + block: [u8; 64], + block_len: usize, + byte_len: u64, +} + +impl Sha256 { + fn new() -> Self { + Self { + state: SHA256_INITIAL_STATE, + block: [0; 64], + block_len: 0, + byte_len: 0, + } + } + + fn digest(bytes: &[u8]) -> [u8; 32] { + let mut digest = Self::new(); + digest.update(bytes); + digest.finalize() + } + + fn update(&mut self, mut bytes: &[u8]) { + self.byte_len = self + .byte_len + .checked_add(bytes.len() as u64) + .expect("benchmark SHA-256 input length overflowed u64"); + + if self.block_len != 0 { + let needed = 64 - self.block_len; + let copied = needed.min(bytes.len()); + self.block[self.block_len..self.block_len + copied].copy_from_slice(&bytes[..copied]); + self.block_len += copied; + bytes = &bytes[copied..]; + if self.block_len == 64 { + Self::compress(&mut self.state, &self.block); + self.block_len = 0; + } else { + return; + } + } + + while bytes.len() >= 64 { + let block: &[u8; 64] = bytes[..64] + .try_into() + .expect("SHA-256 block has a fixed length"); + Self::compress(&mut self.state, block); + bytes = &bytes[64..]; + } + self.block[..bytes.len()].copy_from_slice(bytes); + self.block_len = bytes.len(); + } + + fn finalize(mut self) -> [u8; 32] { + let bit_len = self.byte_len.wrapping_mul(8); + self.block[self.block_len] = 0x80; + self.block_len += 1; + if self.block_len > 56 { + self.block[self.block_len..].fill(0); + Self::compress(&mut self.state, &self.block); + self.block = [0; 64]; + } else { + self.block[self.block_len..56].fill(0); + } + self.block[56..].copy_from_slice(&bit_len.to_be_bytes()); + Self::compress(&mut self.state, &self.block); + + let mut output = [0u8; 32]; + for (chunk, word) in output.chunks_exact_mut(4).zip(self.state) { + chunk.copy_from_slice(&word.to_be_bytes()); + } + output + } + + fn compress(state: &mut [u32; 8], block: &[u8; 64]) { + let mut schedule = [0u32; 64]; + for (index, bytes) in block.chunks_exact(4).enumerate() { + schedule[index] = + u32::from_be_bytes(bytes.try_into().expect("SHA-256 word is 4 bytes")); + } + for index in 16..64 { + let s0 = schedule[index - 15].rotate_right(7) + ^ schedule[index - 15].rotate_right(18) + ^ (schedule[index - 15] >> 3); + let s1 = schedule[index - 2].rotate_right(17) + ^ schedule[index - 2].rotate_right(19) + ^ (schedule[index - 2] >> 10); + schedule[index] = schedule[index - 16] + .wrapping_add(s0) + .wrapping_add(schedule[index - 7]) + .wrapping_add(s1); + } + + let [mut a, mut b, mut c, mut d, mut e, mut f, mut g, mut h] = *state; + for index in 0..64 { + let choice = (e & f) ^ ((!e) & g); + let majority = (a & b) ^ (a & c) ^ (b & c); + let sum1 = e.rotate_right(6) ^ e.rotate_right(11) ^ e.rotate_right(25); + let sum0 = a.rotate_right(2) ^ a.rotate_right(13) ^ a.rotate_right(22); + let temp1 = h + .wrapping_add(sum1) + .wrapping_add(choice) + .wrapping_add(SHA256_ROUND_CONSTANTS[index]) + .wrapping_add(schedule[index]); + let temp2 = sum0.wrapping_add(majority); + h = g; + g = f; + f = e; + e = d.wrapping_add(temp1); + d = c; + c = b; + b = a; + a = temp1.wrapping_add(temp2); + } + for (slot, value) in state.iter_mut().zip([a, b, c, d, e, f, g, h]) { + *slot = slot.wrapping_add(value); + } + } +} + +struct CountingGlobal; + +static ALLOC_CALLS: AtomicU64 = AtomicU64::new(0); +static ALLOC_BYTES: AtomicU64 = AtomicU64::new(0); +static CURRENT_BYTES: AtomicU64 = AtomicU64::new(0); +static PEAK_BYTES: AtomicU64 = AtomicU64::new(0); +static PHASE_BASELINE_BYTES: AtomicU64 = AtomicU64::new(0); + +#[global_allocator] +static GLOBAL: CountingGlobal = CountingGlobal; + +#[inline] +fn record_alloc(bytes: usize) { + let bytes = bytes as u64; + ALLOC_CALLS.fetch_add(1, Ordering::Relaxed); + ALLOC_BYTES.fetch_add(bytes, Ordering::Relaxed); + let current = CURRENT_BYTES.fetch_add(bytes, Ordering::Relaxed) + bytes; + let mut peak = PEAK_BYTES.load(Ordering::Relaxed); + while current > peak { + match PEAK_BYTES.compare_exchange_weak(peak, current, Ordering::Relaxed, Ordering::Relaxed) + { + Ok(_) => break, + Err(next) => peak = next, + } + } +} + +unsafe impl GlobalAlloc for CountingGlobal { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let ptr = unsafe { System.alloc(layout) }; + if !ptr.is_null() { + record_alloc(layout.size()); + } + ptr + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let ptr = unsafe { System.alloc_zeroed(layout) }; + if !ptr.is_null() { + record_alloc(layout.size()); + } + ptr + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + CURRENT_BYTES.fetch_sub(layout.size() as u64, Ordering::Relaxed); + unsafe { System.dealloc(ptr, layout) }; + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let next = unsafe { System.realloc(ptr, layout, new_size) }; + if !next.is_null() { + CURRENT_BYTES.fetch_sub(layout.size() as u64, Ordering::Relaxed); + record_alloc(new_size); + } + next + } +} + +#[derive(Clone, Copy)] +struct AllocationSnapshot { + calls: u64, + bytes: u64, + current: u64, + peak: u64, + baseline: u64, +} + +fn reset_allocation_phase() { + ALLOC_CALLS.store(0, Ordering::Relaxed); + ALLOC_BYTES.store(0, Ordering::Relaxed); + let current = CURRENT_BYTES.load(Ordering::Relaxed); + PHASE_BASELINE_BYTES.store(current, Ordering::Relaxed); + PEAK_BYTES.store(current, Ordering::Relaxed); +} + +fn allocation_snapshot() -> AllocationSnapshot { + AllocationSnapshot { + calls: ALLOC_CALLS.load(Ordering::Relaxed), + bytes: ALLOC_BYTES.load(Ordering::Relaxed), + current: CURRENT_BYTES.load(Ordering::Relaxed), + peak: PEAK_BYTES.load(Ordering::Relaxed), + baseline: PHASE_BASELINE_BYTES.load(Ordering::Relaxed), + } +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Scenario { + schema_version: u32, + kind: String, + id: String, + suite: String, + subject: Subject, + #[serde(rename = "executorRequirements")] + _executor_requirements: Vec, + frames: u32, + tape: InputTape, + phases: Vec, + checkpoints: Vec, + params: HashMap, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Subject { + id: String, + family: String, + framework: String, + entry: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct InputTape { + schema_version: u32, + kind: String, + id: String, + frames: u32, + tracks: Vec, +} + +#[derive(Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +enum InputTrack { + Button { + control: String, + samples: Vec, + }, + Analog { + control: String, + samples: Vec, + }, + Touch { + control: String, + samples: Vec, + }, + RelativeAxis { + control: String, + samples: Vec, + }, + Effect { + effect: String, + samples: Vec, + }, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ButtonSample { + frame: u32, + pressed: bool, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct AnalogSample { + frame: u32, + value: f64, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct TouchSample { + frame: u32, + phase: TouchPhase, + x: f64, + y: f64, +} + +#[derive(Clone, Copy, Deserialize)] +#[serde(rename_all = "lowercase")] +enum TouchPhase { + Start, + Move, + End, + Cancel, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct AxisSample { + frame: u32, + delta: f64, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct EffectSample { + frame: u32, + value: Value, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Phase { + name: String, + start_frame: u32, + end_frame: u32, + collect: bool, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Checkpoint { + frame: u32, + capture: Vec, +} + +#[derive(Clone, Default)] +struct FrameInput { + buttons: u32, + analog: u32, + touches: Vec, +} + +struct RenderConfig { + viewport: (f32, f32), + width: usize, + height: usize, + density: u32, + scale: u32, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct GuestPhaseRecord<'a> { + schema_version: u32, + event: &'static str, + scenario_id: &'a str, + phase: &'a str, + phase_id: u32, + iteration: u32, + alloc_calls: u64, + allocated_bytes: u64, + current_bytes: u64, + peak_bytes: u64, + quickjs_live_bytes_after_gc: u64, + draw_list_hash: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct GuestComplete<'a> { + schema_version: u32, + event: &'static str, + scenario_id: &'a str, + suite: &'a str, + framework: &'a str, + final_draw_list_hash: String, + final_state_hash: String, + effect_hash: String, + #[serde(skip_serializing_if = "Option::is_none")] + framebuffer_trace_hash: Option, +} + +struct Args { + scenario: PathBuf, + bundle: PathBuf, + pak: Option, + framebuffer_out: Option, + framebuffer_dir: Option, + markers: bool, + correctness: bool, +} + +fn parse_args() -> Result { + let mut scenario = None; + let mut bundle = None; + let mut pak = None; + let mut framebuffer_out = None; + let mut framebuffer_dir = None; + let mut markers = false; + let mut correctness = false; + let mut args = std::env::args().skip(1); + while let Some(arg) = args.next() { + match arg.as_str() { + "--scenario" => scenario = args.next().map(PathBuf::from), + "--bundle" => bundle = args.next().map(PathBuf::from), + "--pak" => pak = args.next().map(PathBuf::from), + "--framebuffer-out" => framebuffer_out = args.next().map(PathBuf::from), + "--framebuffer-dir" => framebuffer_dir = args.next().map(PathBuf::from), + "--markers" => markers = true, + "--correctness" => correctness = true, + "--help" | "-h" => { + println!( + "usage: pocketjs-perf-guest --scenario FILE --bundle FILE [--pak FILE] [--framebuffer-out FILE] [--framebuffer-dir DIR] [--markers] [--correctness]" + ); + std::process::exit(0); + } + _ => bail!("unknown argument {arg}"), + } + } + Ok(Args { + scenario: scenario.ok_or_else(|| anyhow!("--scenario is required"))?, + bundle: bundle.ok_or_else(|| anyhow!("--bundle is required"))?, + pak, + framebuffer_out, + framebuffer_dir, + markers, + correctness, + }) +} + +fn read_scenario(path: &Path) -> Result { + let bytes = fs::read(path).with_context(|| format!("reading {}", path.display()))?; + let scenario: Scenario = + serde_json::from_slice(&bytes).with_context(|| format!("parsing {}", path.display()))?; + if scenario.schema_version != 1 || scenario.kind != "pocketjs.perf.scenario" { + bail!("unsupported scenario schema"); + } + if scenario.tape.schema_version != 1 || scenario.tape.kind != "pocketjs.perf.input-tape" { + bail!("unsupported input tape schema"); + } + if scenario.frames != scenario.tape.frames { + bail!("scenario/tape frame count mismatch"); + } + if scenario.frames == 0 { + bail!("scenario must contain at least one frame"); + } + if scenario.subject.entry.is_empty() + || scenario.subject.id.is_empty() + || scenario.subject.family.is_empty() + || scenario.tape.id.is_empty() + { + bail!("scenario subject and tape identifiers must be non-empty"); + } + for checkpoint in &scenario.checkpoints { + if checkpoint.frame >= scenario.frames { + bail!( + "checkpoint frame {} is outside the scenario", + checkpoint.frame + ); + } + for capture in &checkpoint.capture { + if !matches!( + capture.as_str(), + "framebuffer" | "drawList" | "state" | "effects" + ) { + bail!("unsupported correctness capture {capture}"); + } + } + } + let mut previous_end = 0; + for phase in &scenario.phases { + if phase.name.is_empty() { + bail!("phase names must be non-empty"); + } + if phase.start_frame >= phase.end_frame || phase.end_frame > scenario.frames { + bail!( + "phase {} has invalid range {}..{} for {} frames", + phase.name, + phase.start_frame, + phase.end_frame, + scenario.frames + ); + } + if phase.start_frame < previous_end { + bail!("phase {} overlaps the preceding phase", phase.name); + } + previous_end = phase.end_frame; + } + Ok(scenario) +} + +fn render_config(scenario: &Scenario) -> Result { + let viewport = scenario.params.get("viewport"); + let field = |name: &str| viewport.and_then(|value| value.get(name)); + let dimension = |name: &str, fallback: u32| -> Result<(f32, usize)> { + let value = match field(name) { + Some(value) => value + .as_u64() + .and_then(|value| u32::try_from(value).ok()) + .ok_or_else(|| anyhow!("viewport.{name} must be an unsigned integer"))?, + None => fallback, + }; + if !(1..=32_000).contains(&value) { + bail!("viewport.{name} must be in 1..=32000"); + } + Ok((value as f32, value as usize)) + }; + let integer = |name: &str, fallback: u32, maximum: u32| -> Result { + let value = match field(name) { + Some(value) => value + .as_u64() + .and_then(|value| u32::try_from(value).ok()) + .ok_or_else(|| anyhow!("viewport.{name} must be an unsigned integer"))?, + None => fallback, + }; + if !(1..=maximum).contains(&value) { + bail!("viewport.{name} must be in 1..={maximum}"); + } + Ok(value) + }; + let (viewport_width, width) = dimension("width", 480)?; + let (viewport_height, height) = dimension("height", 272)?; + let density = integer("rasterDensity", 1, u8::MAX as u32)?; + let scale = integer("renderScale", 1, raster::MAX_RENDER_SCALE)?; + Ok(RenderConfig { + viewport: (viewport_width, viewport_height), + width, + height, + density, + scale, + }) +} + +fn button_mask(control: &str) -> Result { + Ok(match control { + "primary" | "circle" => btn::CIRCLE, + "secondary" | "triangle" => btn::TRIANGLE, + "tertiary" | "cross" => btn::CROSS, + "quaternary" | "square" => btn::SQUARE, + "select" => btn::SELECT, + "start" => btn::START, + "up" => btn::UP, + "right" => btn::RIGHT, + "down" => btn::DOWN, + "left" => btn::LEFT, + "shoulder-left" | "shoulderLeft" | "ltrigger" => btn::LTRIGGER, + "shoulder-right" | "shoulderRight" | "rtrigger" => btn::RTRIGGER, + _ => bail!("unknown logical button {control}"), + }) +} + +fn contact_id(control: &str) -> Result { + let digits = control + .strip_prefix("contact-") + .or_else(|| control.strip_prefix("touch-")) + .unwrap_or(control); + let id: u32 = digits + .parse() + .with_context(|| format!("touch control {control} must end in a numeric contact id"))?; + if id > 7 { + bail!("touch contact id {id} is outside 0..7"); + } + Ok(id) +} + +fn logical_coord(value: f64, name: &str) -> Result { + if !value.is_finite() || value < 0.0 || value > 511.0 { + bail!("{name} coordinate {value} is outside 0..511"); + } + Ok(value.round() as u32) +} + +fn analog_byte(value: f64) -> Result { + if !value.is_finite() || !(-1.0..=1.0).contains(&value) { + bail!("analog level {value} is outside -1..1"); + } + Ok(if value == 0.0 { + 128 + } else if value < 0.0 { + (128.0 + value * 128.0).round() as u32 + } else { + (128.0 + value * 127.0).round() as u32 + }) +} + +fn expand_tape(scenario: &Scenario) -> Result<(Vec, bool)> { + let mut button_events = vec![Vec::<(u32, bool)>::new(); scenario.frames as usize]; + let mut analog_events = vec![Vec::<(String, f64)>::new(); scenario.frames as usize]; + let mut touch_events = + vec![Vec::<(u32, TouchPhase, u32, u32)>::new(); scenario.frames as usize]; + let mut has_touch = false; + + for track in &scenario.tape.tracks { + match track { + InputTrack::Button { control, samples } => { + let mask = button_mask(control)?; + for sample in samples { + let at = button_events + .get_mut(sample.frame as usize) + .ok_or_else(|| { + anyhow!("button sample frame {} is outside the tape", sample.frame) + })?; + at.push((mask, sample.pressed)); + } + } + InputTrack::Analog { control, samples } => { + for sample in samples { + let at = analog_events + .get_mut(sample.frame as usize) + .ok_or_else(|| { + anyhow!("analog sample frame {} is outside the tape", sample.frame) + })?; + at.push((control.clone(), sample.value)); + } + } + InputTrack::Touch { control, samples } => { + has_touch = true; + let id = contact_id(control)?; + for sample in samples { + let x = logical_coord(sample.x, "touch x")?; + let y = logical_coord(sample.y, "touch y")?; + let at = touch_events.get_mut(sample.frame as usize).ok_or_else(|| { + anyhow!("touch sample frame {} is outside the tape", sample.frame) + })?; + let phase = match sample.phase { + TouchPhase::Start => TouchPhase::Start, + TouchPhase::Move => TouchPhase::Move, + TouchPhase::End => TouchPhase::End, + TouchPhase::Cancel => TouchPhase::Cancel, + }; + at.push((id, phase, x, y)); + } + } + InputTrack::RelativeAxis { control, samples } => { + if !samples.is_empty() { + let _sum = samples.iter().fold(0.0, |sum, sample| { + sum + sample.delta + f64::from(sample.frame) + }); + bail!("relative axis {control} requires the Vapor executor"); + } + } + InputTrack::Effect { effect, samples } => { + if !samples.is_empty() { + let _observed = samples + .iter() + .any(|sample| sample.frame < scenario.frames && !sample.value.is_null()); + bail!("effect delivery {effect} requires a configured effect adapter"); + } + } + } + } + + let mut buttons = 0u32; + let mut analog_x = 128u32; + let mut analog_y = 128u32; + let mut contacts = BTreeMap::::new(); + let mut frames = Vec::with_capacity(scenario.frames as usize); + for frame in 0..scenario.frames as usize { + for &(mask, pressed) in &button_events[frame] { + if pressed { + buttons |= mask; + } else { + buttons &= !mask; + } + } + for (control, value) in &analog_events[frame] { + if !value.is_finite() { + bail!("analog sample must be finite"); + } + match control.as_str() { + "x" | "horizontal" => analog_x = analog_byte(*value)?, + "y" | "vertical" => analog_y = analog_byte(*value)?, + "packed" => { + let packed = value.round().clamp(0.0, 65535.0) as u32; + analog_x = (packed >> 8) & 0xff; + analog_y = packed & 0xff; + } + _ => bail!("unknown analog control {control}"), + } + } + for (id, phase, x, y) in &touch_events[frame] { + match phase { + TouchPhase::Start | TouchPhase::Move => { + contacts.insert(*id, (*x, *y)); + } + TouchPhase::End | TouchPhase::Cancel => { + contacts.remove(id); + } + } + } + frames.push(FrameInput { + buttons, + analog: (analog_x << 8) | analog_y, + touches: contacts + .iter() + .map(|(id, (x, y))| (id << 18) | (y << 9) | x) + .collect(), + }); + } + Ok((frames, has_touch)) +} + +fn fnv1a64_bytes(bytes: impl IntoIterator) -> String { + let mut hash = 0xcbf2_9ce4_8422_2325u64; + for byte in bytes { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + format!("fnv1a64:{hash:016x}") +} + +fn lowercase_hex(bytes: &[u8]) -> String { + let mut output = String::with_capacity(bytes.len() * 2); + for byte in bytes { + use std::fmt::Write as _; + write!(&mut output, "{byte:02x}").expect("writing to String cannot fail"); + } + output +} + +fn draw_hash(surface: &UiSurface) -> String { + surface.with_ui(|ui| { + fnv1a64_bytes( + ui.current_draw_list() + .words + .iter() + .flat_map(|word| word.to_le_bytes()), + ) + }) +} + +fn frame_with_touch_hits( + guest: &Guest, + surface: &UiSurface, + buttons: u32, + analog: u32, + touches: &[u32], +) -> Result<()> { + // Match the native hosts' frame argument 4 contract: resolve each new + // contact against the committed bounds tree once, then let Core carry the + // hit fact until that contact lifts. Keeping this in the perf guest avoids + // changing production Guest/UiSurface APIs solely for the harness. + let mut hits = [0i32; 8]; + let hit_count = surface.with_ui(|ui| ui.touch_hits(touches, &mut hits)); + guest.with(|ctx| -> Result<()> { + let frame: Option = ctx.globals().get("frame").ok(); + if let Some(frame) = frame { + let touch_array = Array::new(ctx.clone()) + .map_err(|error| anyhow!("allocating benchmark touch array: {error}"))?; + for (index, touch) in touches.iter().enumerate() { + touch_array + .set(index, *touch) + .map_err(|error| anyhow!("setting benchmark touch {index}: {error}"))?; + } + let hit_array = Array::new(ctx.clone()) + .map_err(|error| anyhow!("allocating benchmark touch-hit array: {error}"))?; + for (index, hit) in hits[..hit_count].iter().enumerate() { + hit_array + .set(index, *hit) + .map_err(|error| anyhow!("setting benchmark touch hit {index}: {error}"))?; + } + frame + .call::<_, ()>((buttons, analog, touch_array, hit_array)) + .catch(&ctx) + .map_err(|error| anyhow!("benchmark frame() threw: {error}"))?; + } + Ok(()) + })?; + guest.drain_jobs(); + Ok(()) +} + +fn render_frame(surface: &UiSurface, framebuffer: &mut [u8], scale: u32) { + surface.with_ui(|ui| { + // `draw()` returns storage owned by `ui`; rasterization only reads the + // DrawList plus texture/font resources from that same Ui. This is the + // same single-threaded reborrow used by the WASM host renderer. + let draw_list: *const pocketjs_core::DrawList = ui.draw(); + let ui_ref = unsafe { &*(ui as *const pocketjs_core::Ui) }; + unsafe { + raster::render_scaled(ui_ref, &(*draw_list).words, framebuffer, scale); + } + }); +} + +fn write_checkpoint_framebuffer( + scenario: &Scenario, + frame: u32, + directory: Option<&Path>, + framebuffer: &[u8], +) -> Result<()> { + let Some(directory) = directory else { + return Ok(()); + }; + let captures_framebuffer = scenario.checkpoints.iter().any(|checkpoint| { + checkpoint.frame == frame + && checkpoint + .capture + .iter() + .any(|capture| capture == "framebuffer") + }); + if !captures_framebuffer { + return Ok(()); + } + let path = directory.join(format!("{frame}.rgba")); + fs::write(&path, framebuffer) + .with_context(|| format!("writing correctness checkpoint {}", path.display())) +} + +fn install_correctness_probes(guest: &Guest) -> Result<()> { + guest + .eval( + "pocketjs-perf-probes", + r#" +globalThis.__pocketPerfMessages = []; +globalThis.__pocketPerfInbox = []; +globalThis.__pocketDevtoolsTransport = { + send(line) { globalThis.__pocketPerfMessages.push(String(line)); }, + recv() { + return globalThis.__pocketPerfInbox.length > 0 + ? globalThis.__pocketPerfInbox.shift() + : null; + }, +}; +globalThis.__pocketPerfEffects = []; +globalThis.__pocketEffectTrace = (event) => { + globalThis.__pocketPerfEffects.push(event); +}; +"#, + ) + .context("installing benchmark correctness probes") +} + +fn effect_snapshot(guest: &Guest) -> Result<(String, String)> { + let effects = guest + .with(|ctx| ctx.eval::(b"JSON.stringify(globalThis.__pocketPerfEffects)")) + .context("serializing benchmark effect trace")?; + Ok((fnv1a64_bytes(effects.bytes()), effects)) +} + +fn capture_state_snapshot(guest: &Guest, surface: &UiSurface) -> Result<(String, String)> { + guest + .with(|ctx| { + ctx.eval::<(), _>( + b"void globalThis.__pocketPerfInbox.push(JSON.stringify({t:'getTree'}))", + ) + }) + .context("requesting final DevTools tree")?; + // Match the Native correctness adapter: service the request in one extra + // unmeasured frame, then advance Core once. This work is outside every + // marker and cannot affect performance counters. + guest.frame_with_analog(0, 0x8080)?; + surface.tick(); + let state = guest + .with(|ctx| { + ctx.eval::( + br#" +(() => { + const lines = globalThis.__pocketPerfMessages; + for (let i = lines.length - 1; i >= 0; i--) { + const message = JSON.parse(lines[i]); + if (message && message.t === 'tree') return JSON.stringify(message.root); + } + throw new Error('benchmark DevTools tree response is missing'); +})() +"#, + ) + }) + .context("reading final DevTools tree")?; + Ok((fnv1a64_bytes(state.bytes()), state)) +} + +fn phase_id(scenario: &str, phase: &str) -> u32 { + let mut hash = 0x811c_9dc5u32; + for byte in scenario.bytes().chain([0]).chain(phase.bytes()) { + hash ^= u32::from(byte); + hash = hash.wrapping_mul(0x0100_0193); + } + hash +} + +fn marker(enabled: bool, opcode: u32, id: u32) -> Result<()> { + if !enabled { + return Ok(()); + } + let packed = (MARKER_VERSION << 8) | opcode; + #[cfg(target_os = "linux")] + let result = unsafe { + libc::syscall( + MARKER_SYSCALL as libc::c_long, + MARKER_MAGIC as libc::c_long, + packed as libc::c_long, + id as libc::c_long, + 0 as libc::c_long, + MARKER_COOKIE as libc::c_long, + 0 as libc::c_long, + ) + } as i64; + #[cfg(not(target_os = "linux"))] + let result = unsafe { + libc::syscall( + MARKER_SYSCALL as libc::c_int, + MARKER_MAGIC as libc::c_long, + packed as libc::c_long, + id as libc::c_long, + 0 as libc::c_long, + MARKER_COOKIE as libc::c_long, + 0 as libc::c_long, + ) + } as i64; + if result != 0 { + bail!("QEMU marker rejected opcode {opcode}, phase {id}: return {result}"); + } + Ok(()) +} + +fn print_json(value: &impl Serialize) -> Result<()> { + println!("{GUEST_PREFIX}{}", serde_json::to_string(value)?); + Ok(()) +} + +fn quickjs_live_bytes_after_gc(guest: &Guest) -> u64 { + guest.with(|ctx| { + ctx.run_gc(); + let mut usage = MaybeUninit::::uninit(); + unsafe { + let runtime = pocket_mod::qjs::qjs::JS_GetRuntime(ctx.as_raw().as_ptr()); + pocket_mod::qjs::qjs::JS_ComputeMemoryUsage(runtime, usage.as_mut_ptr()); + usage.assume_init().memory_used_size.max(0) as u64 + } + }) +} + +fn finish_phase( + scenario: &Scenario, + phase_name: &str, + id: u32, + allocation: AllocationSnapshot, + guest: &Guest, + surface: &UiSurface, +) -> Result<()> { + print_json(&GuestPhaseRecord { + schema_version: 1, + event: "phase", + scenario_id: &scenario.id, + phase: phase_name, + phase_id: id, + iteration: 0, + alloc_calls: allocation.calls, + allocated_bytes: allocation.bytes, + current_bytes: allocation.current, + peak_bytes: allocation.peak.saturating_sub(allocation.baseline), + quickjs_live_bytes_after_gc: quickjs_live_bytes_after_gc(guest), + draw_list_hash: draw_hash(surface), + }) +} + +fn main() -> Result<()> { + let args = parse_args()?; + if (args.framebuffer_out.is_some() || args.framebuffer_dir.is_some()) && !args.correctness { + bail!("framebuffer output is only valid with --correctness"); + } + if args.correctness && args.markers { + bail!("--correctness and --markers are separate replays"); + } + let scenario = read_scenario(&args.scenario)?; + let render = render_config(&scenario)?; + if let Some(directory) = &args.framebuffer_dir { + fs::create_dir_all(directory).with_context(|| { + format!( + "creating correctness framebuffer directory {}", + directory.display() + ) + })?; + } + let (inputs, has_touch) = expand_tape(&scenario)?; + let bundle = fs::read_to_string(&args.bundle) + .with_context(|| format!("reading {}", args.bundle.display()))?; + let pak = match &args.pak { + Some(path) => fs::read(path).with_context(|| format!("reading {}", path.display()))?, + None => Vec::new(), + }; + + let measure_boot = scenario + .params + .get("measureBoot") + .and_then(Value::as_bool) + .unwrap_or(false); + + let runtime_phase = "runtime-init"; + let runtime_phase_id = phase_id(&scenario.id, runtime_phase); + if measure_boot { + reset_allocation_phase(); + marker(args.markers, MARKER_BEGIN, runtime_phase_id)?; + } + let guest = Guest::new().context("creating QuickJS guest")?; + let surface = UiSurface::new_with_density(render.viewport, render.density); + // Allocate once; the measured frame loop includes raster work but never + // pays a diagnostic framebuffer growth/allocation cost. + let framebuffer_bytes = render + .width + .checked_mul(render.scale as usize) + .and_then(|width| { + render + .height + .checked_mul(render.scale as usize) + .and_then(|height| width.checked_mul(height)) + }) + .and_then(|pixels| pixels.checked_mul(4)) + .ok_or_else(|| anyhow!("scaled framebuffer dimensions overflow"))?; + let mut framebuffer = vec![0; framebuffer_bytes]; + if !pak.is_empty() { + surface.feed_pak(&pak); + } + surface.mount(&guest).context("mounting ui surface")?; + if measure_boot { + let allocation = allocation_snapshot(); + marker(args.markers, MARKER_END, runtime_phase_id)?; + finish_phase( + &scenario, + runtime_phase, + runtime_phase_id, + allocation, + &guest, + &surface, + )?; + } + // Only the independent correctness replay installs observational JS + // hooks. Even a trace-array push would otherwise change the instruction + // and allocation counts of the measurement replay. + if args.correctness { + install_correctness_probes(&guest)?; + } + + let eval_phase = "bundle-eval"; + let eval_phase_id = phase_id(&scenario.id, eval_phase); + if measure_boot { + reset_allocation_phase(); + marker(args.markers, MARKER_BEGIN, eval_phase_id)?; + } + guest + .eval(&scenario.subject.entry, &bundle) + .context("evaluating app bundle")?; + if measure_boot { + let allocation = allocation_snapshot(); + marker(args.markers, MARKER_END, eval_phase_id)?; + finish_phase( + &scenario, + eval_phase, + eval_phase_id, + allocation, + &guest, + &surface, + )?; + } + + let mut active: Option<(&Phase, u32)> = None; + // Native correctness hashes every raw RGBA frame, writes each lowercase + // frame digest into a second SHA-256 stream, and finally digests that + // stream. Keep the identical operation entirely out of the measurement + // replay so it cannot affect marker counters or allocation samples. + let mut framebuffer_trace = args.correctness.then(Sha256::new); + for (frame_index, input) in inputs.iter().enumerate() { + let frame = frame_index as u32; + if let Some(phase) = scenario + .phases + .iter() + .find(|phase| phase.collect && phase.start_frame == frame) + { + if active.is_some() { + bail!("overlapping collected phases are not supported"); + } + let id = phase_id(&scenario.id, &phase.name); + reset_allocation_phase(); + marker(args.markers, MARKER_BEGIN, id)?; + active = Some((phase, id)); + } + + if has_touch { + frame_with_touch_hits( + &guest, + &surface, + input.buttons, + input.analog, + &input.touches, + )?; + } else { + guest.frame_with_analog(input.buttons, input.analog)?; + } + surface.tick(); + render_frame(&surface, &mut framebuffer, render.scale); + if let Some(trace) = framebuffer_trace.as_mut() { + let frame_hash = Sha256::digest(&framebuffer); + let frame_hash_hex = lowercase_hex(&frame_hash); + trace.update(frame_hash_hex.as_bytes()); + } + if args.correctness { + write_checkpoint_framebuffer( + &scenario, + frame, + args.framebuffer_dir.as_deref(), + &framebuffer, + )?; + } + + if let Some((phase, id)) = active { + if phase.end_frame == frame + 1 { + let allocation = allocation_snapshot(); + marker(args.markers, MARKER_END, id)?; + finish_phase(&scenario, &phase.name, id, allocation, &guest, &surface)?; + active = None; + } + } + } + if let Some((phase, _)) = active { + bail!("phase {} did not close", phase.name); + } + + if let Some(path) = &args.framebuffer_out { + fs::write(path, &framebuffer) + .with_context(|| format!("writing correctness framebuffer {}", path.display()))?; + } + let final_draw_list_hash = draw_hash(&surface); + let (final_state_hash, effect_hash) = if args.correctness { + let (effects_hash, effects_json) = effect_snapshot(&guest)?; + let (state_hash, state_json) = capture_state_snapshot(&guest, &surface)?; + if let Some(directory) = &args.framebuffer_dir { + fs::write(directory.join("effects.json"), effects_json) + .context("writing correctness effect trace")?; + fs::write(directory.join("state.json"), state_json) + .context("writing correctness state tree")?; + } + (state_hash, effects_hash) + } else { + // The bridge discards these two sentinels from the measurement replay + // and combines its phase/allocation records with the independent + // correctness replay's complete record. + ( + fnv1a64_bytes(b"measurement-replay-state".iter().copied()), + fnv1a64_bytes(b"measurement-replay-effects".iter().copied()), + ) + }; + let framebuffer_trace_hash = framebuffer_trace.map(|trace| lowercase_hex(&trace.finalize())); + print_json(&GuestComplete { + schema_version: 1, + event: "complete", + scenario_id: &scenario.id, + suite: &scenario.suite, + framework: &scenario.subject.framework, + final_draw_list_hash, + final_state_hash, + effect_hash, + framebuffer_trace_hash, + })?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{Sha256, fill_deterministic_random, lowercase_hex}; + + fn sha256(bytes: &[u8]) -> String { + lowercase_hex(&Sha256::digest(bytes)) + } + + #[test] + fn sha256_matches_standard_vectors() { + assert_eq!( + sha256(b""), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + assert_eq!( + sha256(b"abc"), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + assert_eq!( + sha256(b"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"), + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1" + ); + } + + #[test] + fn sha256_streaming_matches_one_shot_across_block_boundaries() { + let bytes = (0..257).map(|value| value as u8).collect::>(); + let expected = Sha256::digest(&bytes); + let mut streaming = Sha256::new(); + for chunk in bytes.chunks(13) { + streaming.update(chunk); + } + assert_eq!(streaming.finalize(), expected); + } + + #[test] + fn benchmark_random_stream_is_nonzero_and_call_indexed() { + let mut first = [0; 16]; + let mut repeated = [0; 16]; + let mut second = [0; 16]; + fill_deterministic_random(0, &mut first); + fill_deterministic_random(0, &mut repeated); + fill_deterministic_random(1, &mut second); + + assert_eq!(first, repeated); + assert_ne!(first, [0; 16]); + assert_ne!(first, second); + assert_eq!(first[..4], [0x53, 0x54, 0x55, 0x56]); + assert_eq!(second[..4], [0xf0, 0xf1, 0xf2, 0xf3]); + } +} diff --git a/tools/perf/qemu/.dockerignore b/tools/perf/qemu/.dockerignore new file mode 100644 index 00000000..443cee86 --- /dev/null +++ b/tools/perf/qemu/.dockerignore @@ -0,0 +1,3 @@ +build +__pycache__ +*.pyc diff --git a/tools/perf/qemu/.gitignore b/tools/perf/qemu/.gitignore new file mode 100644 index 00000000..255c2556 --- /dev/null +++ b/tools/perf/qemu/.gitignore @@ -0,0 +1,3 @@ +build/ +__pycache__/ +*.pyc diff --git a/tools/perf/qemu/Dockerfile b/tools/perf/qemu/Dockerfile new file mode 100644 index 00000000..946c5347 --- /dev/null +++ b/tools/perf/qemu/Dockerfile @@ -0,0 +1,66 @@ +FROM debian:bookworm-slim@sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241 + +ARG DEBIAN_SNAPSHOT=20260803T000000Z +ARG QEMU_VERSION=11.0.3 +ARG QEMU_SHA256=da5fcffc32762820568b828ed430a728864d34d50b6d2f30358597760cbb0523 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN rm -f /etc/apt/sources.list.d/debian.sources && \ + printf '%s\n' \ + "deb [check-valid-until=no] http://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT} bookworm main" \ + "deb [check-valid-until=no] http://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT} bookworm-updates main" \ + "deb [check-valid-until=no] http://snapshot.debian.org/archive/debian-security/${DEBIAN_SNAPSHOT} bookworm-security main" \ + > /etc/apt/sources.list && \ + apt-get -o Acquire::Check-Valid-Until=false update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + gcc-aarch64-linux-gnu \ + gcc-arm-linux-gnueabihf \ + libglib2.0-dev \ + ninja-build \ + pkg-config \ + python3 && \ + rm -rf /var/lib/apt/lists/* + +# QEMU creates an isolated build environment for its bundled Python tooling. +RUN apt-get -o Acquire::Check-Valid-Until=false update && \ + apt-get install -y --no-install-recommends python3-venv && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /tmp/qemu-source +RUN curl -fsSLo qemu.tar.xz \ + "https://download.qemu.org/qemu-${QEMU_VERSION}.tar.xz" && \ + echo "${QEMU_SHA256} qemu.tar.xz" | sha256sum --check - && \ + tar -xJf qemu.tar.xz --strip-components=1 && \ + rm qemu.tar.xz && \ + ./configure \ + --prefix=/opt/qemu \ + --target-list=arm-linux-user,aarch64-linux-user \ + --enable-linux-user \ + --disable-system \ + --enable-plugins \ + --disable-docs \ + --disable-tools \ + --disable-debug-info \ + --enable-strip && \ + ninja -C build -j2 && \ + ninja -C build install && \ + rm -rf /tmp/qemu-source + +# The cross-GCC metapackages only recommend their C library development +# sysroots; install both explicitly because recommended packages are disabled. +RUN apt-get -o Acquire::Check-Valid-Until=false update && \ + apt-get install -y --no-install-recommends \ + libc6-dev-arm64-cross \ + libc6-dev-armhf-cross && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/pocketjs-perf-qemu +COPY . . +RUN make all && make test-parser + +ENV QEMU_PREFIX=/opt/qemu +CMD ["./run-fixtures.sh"] diff --git a/tools/perf/qemu/Dockerfile.runner b/tools/perf/qemu/Dockerfile.runner new file mode 100644 index 00000000..8c2e8581 --- /dev/null +++ b/tools/perf/qemu/Dockerfile.runner @@ -0,0 +1,30 @@ +ARG QEMU_BASE_IMAGE=pocketjs-perf-qemu-base:11.0.3 +FROM ${QEMU_BASE_IMAGE} + +ARG RUST_VERSION=1.93.0 +ARG RUSTUP_INIT_SHA256_AMD64=4acc9acc76d5079515b46346a485974457b5a79893cfb01112423c89aeb5aa10 +ARG RUSTUP_INIT_SHA256_ARM64=9732d6c5e2a098d3521fca8145d826ae0aaa067ef2385ead08e6feac88fa5792 + +# Build the runtime guest in the same final image as QEMU. rustc and both +# target standard libraries are pinned; rustup-init is verified before use. +ENV RUSTUP_HOME=/opt/rust/rustup \ + CARGO_HOME=/opt/rust/cargo \ + PATH=/opt/rust/cargo/bin:${PATH} +RUN case "$(dpkg --print-architecture)" in \ + amd64) rustup_host=x86_64-unknown-linux-gnu; rustup_sha="${RUSTUP_INIT_SHA256_AMD64}" ;; \ + arm64) rustup_host=aarch64-unknown-linux-gnu; rustup_sha="${RUSTUP_INIT_SHA256_ARM64}" ;; \ + *) echo "unsupported container architecture: $(dpkg --print-architecture)" >&2; exit 1 ;; \ + esac && \ + curl -fsSLo /tmp/rustup-init \ + "https://static.rust-lang.org/rustup/dist/${rustup_host}/rustup-init" && \ + echo "${rustup_sha} /tmp/rustup-init" | sha256sum --check - && \ + chmod +x /tmp/rustup-init && \ + /tmp/rustup-init -y --no-modify-path --profile minimal \ + --default-toolchain "${RUST_VERSION}" \ + --target armv7-unknown-linux-gnueabihf \ + --target aarch64-unknown-linux-gnu && \ + rm /tmp/rustup-init && \ + rustc --version --verbose && \ + cargo --version + +WORKDIR /opt/pocketjs-perf-qemu diff --git a/tools/perf/qemu/Makefile b/tools/perf/qemu/Makefile new file mode 100644 index 00000000..09cbcfac --- /dev/null +++ b/tools/perf/qemu/Makefile @@ -0,0 +1,66 @@ +QEMU_PREFIX ?= /opt/qemu +BUILD_DIR ?= build +HOST_CC ?= cc +ARM_CC ?= arm-linux-gnueabihf-gcc +AARCH64_CC ?= aarch64-linux-gnu-gcc +PKG_CONFIG ?= pkg-config + +PLUGIN_CPPFLAGS = -I$(QEMU_PREFIX)/include \ + $(shell $(PKG_CONFIG) --cflags glib-2.0) +PLUGIN_CFLAGS := -std=gnu11 -O2 -g0 -fPIC -fvisibility=hidden \ + -Wall -Wextra -Werror -Wformat=2 -Wshadow -Wconversion +PLUGIN_LDLIBS = $(shell $(PKG_CONFIG) --libs glib-2.0) + +ARM_FLAGS := -O2 -g0 -static -march=armv7-a -mthumb \ + -mfpu=vfpv3-d16 -mfloat-abi=hard -fno-ident \ + -fno-asynchronous-unwind-tables -fno-unwind-tables \ + -Wl,--build-id=none +AARCH64_FLAGS := -O2 -g0 -static -march=armv8-a -fno-ident \ + -fno-asynchronous-unwind-tables -fno-unwind-tables \ + -Wl,--build-id=none + +.PHONY: all fixtures clean test test-parser + +all: $(BUILD_DIR)/pocketjs-perf-counter.so fixtures + +fixtures: \ + $(BUILD_DIR)/exact-armv7 \ + $(BUILD_DIR)/exact-aarch64 \ + $(BUILD_DIR)/marker-cases-armv7 \ + $(BUILD_DIR)/marker-cases-aarch64 \ + $(BUILD_DIR)/multivcpu-armv7 \ + $(BUILD_DIR)/multivcpu-aarch64 + +$(BUILD_DIR): + mkdir -p $@ + +$(BUILD_DIR)/pocketjs-perf-counter.so: perf_counter.c | $(BUILD_DIR) + $(HOST_CC) $(PLUGIN_CPPFLAGS) $(PLUGIN_CFLAGS) -shared -o $@ $< \ + $(PLUGIN_LDLIBS) + +$(BUILD_DIR)/exact-armv7: fixtures/exact-armv7.S | $(BUILD_DIR) + $(ARM_CC) $(ARM_FLAGS) -nostdlib -Wl,-e,_start -o $@ $< + +$(BUILD_DIR)/exact-aarch64: fixtures/exact-aarch64.S | $(BUILD_DIR) + $(AARCH64_CC) $(AARCH64_FLAGS) -nostdlib -Wl,-e,_start -o $@ $< + +$(BUILD_DIR)/marker-cases-armv7: fixtures/marker-cases.c guest_marker.h | $(BUILD_DIR) + $(ARM_CC) $(ARM_FLAGS) -Wall -Wextra -Werror -o $@ $< + +$(BUILD_DIR)/marker-cases-aarch64: fixtures/marker-cases.c guest_marker.h | $(BUILD_DIR) + $(AARCH64_CC) $(AARCH64_FLAGS) -Wall -Wextra -Werror -o $@ $< + +$(BUILD_DIR)/multivcpu-armv7: fixtures/multivcpu.c guest_marker.h | $(BUILD_DIR) + $(ARM_CC) $(ARM_FLAGS) -Wall -Wextra -Werror -pthread -o $@ $< + +$(BUILD_DIR)/multivcpu-aarch64: fixtures/multivcpu.c guest_marker.h | $(BUILD_DIR) + $(AARCH64_CC) $(AARCH64_FLAGS) -Wall -Wextra -Werror -pthread -o $@ $< + +test-parser: + python3 -m unittest discover -s tests -p 'test_*.py' + +test: all test-parser + ./run-fixtures.sh + +clean: + rm -rf -- $(BUILD_DIR) diff --git a/tools/perf/qemu/README.md b/tools/perf/qemu/README.md new file mode 100644 index 00000000..5651d2af --- /dev/null +++ b/tools/perf/qemu/README.md @@ -0,0 +1,110 @@ +# PocketJS QEMU guest counters + +This directory contains the deterministic QEMU side of the local performance +runner. **It targets QEMU 11.0.3 linux-user and plugin API 6.** It does not +estimate guest cycles, cache behavior, device frame rate, GPU cost, or power. + +## Marker ABI + +ARM32 and AArch64 guests issue syscall `4096` with these arguments: + +| syscall argument | value | +| --- | --- | +| `a1` | magic `0x504a424d` | +| `a2` | version in bits 15..8 and opcode in bits 7..0 | +| `a3` | unsigned 32-bit phase ID | +| `a4` | unsigned 32-bit iteration | +| `a5` | cookie `0xc001c0de` | + +Protocol version is `1`; opcode `1` is BEGIN and opcode `2` is END. Include +`guest_marker.h` and call `pocketjs_perf_begin(phase, iteration)` and +`pocketjs_perf_end(phase, iteration)`. Both calls return zero when the plugin +filters the marker. **A BEGIN/END pair must match phase, iteration, and vCPU.** +Nested markers, missing markers, a mismatched END, and more than one linux-user +vCPU make the result invalid. A guest `getrandom(2)` syscall while a marker is +active also makes the result invalid because entropy can change the dynamic +instruction path. + +The BEGIN syscall instruction is outside the snapshot. The END syscall +instruction is inside it because QEMU invokes the syscall filter after +dispatching that guest instruction. Compiler-generated argument setup before +END is therefore also measured. The assembly fixtures pin this boundary with +exact expected counts. + +## Output + +The plugin writes prefixed NDJSON through QEMU's plugin log. Consumers only +parse lines beginning with `POCKETJS_PERF_QEMU `. A successful run contains one +or more `measurement` records followed by one `complete` sentinel. An invalid +run ends with one `error` sentinel. Every sentinel contains the schema and +version, so absence or truncation cannot be accepted as a result. + +**Every direct QEMU invocation must include `-d plugin`.** Without that flag, +`qemu_plugin_outs()` records are disabled and a consumer must reject the run as +missing its terminal sentinel. + +Each measurement reports one vCPU snapshot with these metrics: + +- `guest_insn_dispatched` +- `guest_instruction_bytes` +- `guest_insn_size_2` +- `guest_insn_size_4` +- `guest_load_events` +- `guest_store_events` + +The instruction byte count is the sum of QEMU's translated guest instruction +sizes. Load and store values count guest memory-access events reported by the +plugin API, not bytes transferred. + +## Fixed local toolchain + +Run all build and behavioral checks with Docker: + +```sh +tools/perf/qemu/docker.sh test +``` + +The image pins the Debian base manifest and archive snapshot, QEMU source +version and SHA-256, QEMU plugin API, Rust 1.93.0, and both GNU cross +toolchains. QEMU is built only for `arm-linux-user` and +`aarch64-linux-user`. + +The ARM32 fixture flags are: + +```text +-march=armv7-a -mthumb -mfpu=vfpv3-d16 -mfloat-abi=hard +``` + +All ARM32 runs use `-cpu cortex-a9,neon=off,vfp-d32=off`; all AArch64 runs use +`-cpu cortex-a53`. Both use `-seed 1`. Fixing the CPU model keeps linux-user +hardware capability bits and dynamic library dispatch stable. The seed fixes +QEMU-provided `AT_RANDOM`; it does not intercept the guest `getrandom(2)` +syscall. The QuickJS performance guest supplies Rust `RandomState` through its +own fingerprinted deterministic shim, while this plugin rejects raw entropy +syscalls inside a measured phase. The ARM32 reference environment does not +expose NEON. + +The corresponding Rust lane uses `armv7-unknown-linux-gnueabihf` with +`-C target-feature=+thumb-mode`; QuickJS C sources use the flags above. This +directory deliberately performs no ELF instruction-mode inspection. + +For an already prepared Linux host, set `QEMU_PREFIX` to a QEMU 11.0.3 install +that includes `qemu-plugin.h`, then run: + +```sh +make -C tools/perf/qemu all +make -C tools/perf/qemu test +``` + +`run-fixtures.sh` first rejects any QEMU version other than 11.0.3. It then +checks the exact ARMv7 and AArch64 instruction/memory counts twenty times, +verifies an injected loop increases the instruction count by more than 10,000, +and exercises valid, nested, mismatched, missing, active-phase entropy, and +multithreaded marker cases. Parser-only tests need only Python 3: + +```sh +make -C tools/perf/qemu test-parser +``` + +The fixture checks are local only; this directory does not define a remote +workflow. diff --git a/tools/perf/qemu/docker.sh b/tools/perf/qemu/docker.sh new file mode 100755 index 00000000..f7f8881e --- /dev/null +++ b/tools/perf/qemu/docker.sh @@ -0,0 +1,31 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +BASE_IMAGE=${POCKETJS_QEMU_BASE_IMAGE:-pocketjs-perf-qemu-base:11.0.3} +IMAGE=${POCKETJS_QEMU_IMAGE:-pocketjs-perf-qemu:11.0.3} +COMMAND=${1:-test} + +build_image() { + docker build --file "$SCRIPT_DIR/Dockerfile" --tag "$BASE_IMAGE" "$SCRIPT_DIR" + docker build --file "$SCRIPT_DIR/Dockerfile.runner" \ + --build-arg "QEMU_BASE_IMAGE=$BASE_IMAGE" --tag "$IMAGE" "$SCRIPT_DIR" +} + +case "$COMMAND" in + build) + build_image + ;; + test) + build_image + docker run --rm "$IMAGE" + ;; + shell) + build_image + docker run --rm -it --entrypoint /bin/sh "$IMAGE" + ;; + *) + echo "usage: $0 [build|test|shell]" >&2 + exit 64 + ;; +esac diff --git a/tools/perf/qemu/fixtures/exact-aarch64.S b/tools/perf/qemu/fixtures/exact-aarch64.S new file mode 100644 index 00000000..641df675 --- /dev/null +++ b/tools/perf/qemu/fixtures/exact-aarch64.S @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: MIT */ + + .arch armv8-a + .text + .global _start + .type _start, %function +_start: + /* BEGIN(version=1, phase=7, iteration=0). Setup is not measured. */ + movz x10, #0x424d + movk x10, #0x504a, lsl #16 + mov x0, x10 + mov x1, #0x0101 + mov x2, #7 + mov x3, #0 + movz x4, #0xc0de + movk x4, #0xc001, lsl #16 + mov x8, #4096 + svc #0 + + /* Exactly 18 dispatched 4-byte instructions, one load and one store. */ + mov x5, #0 + mov x6, #3 +1: + add x5, x5, #1 + sub x6, x6, #1 + cbnz x6, 1b + sub sp, sp, #16 + str x5, [sp] + ldr x6, [sp] + add sp, sp, #16 + + /* Restore magic (BEGIN returned zero), then change opcode 1 to opcode 2. */ + mov x0, x10 + add x1, x1, #1 + svc #0 + + /* Linux AArch64 exit(0). This is outside the measurement. */ + mov x0, #0 + mov x8, #93 + svc #0 + .size _start, .-_start + + .section .note.GNU-stack,"",%progbits diff --git a/tools/perf/qemu/fixtures/exact-armv7.S b/tools/perf/qemu/fixtures/exact-armv7.S new file mode 100644 index 00000000..51c70d73 --- /dev/null +++ b/tools/perf/qemu/fixtures/exact-armv7.S @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: MIT */ + + .syntax unified + .arch armv7-a + .thumb + .text + .global _start + .type _start, %function + .thumb_func +_start: + /* BEGIN(version=1, phase=7, iteration=0). Setup is not measured. */ + movw r10, #0x424d + movt r10, #0x504a + mov r0, r10 + movw r1, #0x0101 + movs r2, #7 + movs r3, #0 + movw r4, #0xc0de + movt r4, #0xc001 + movw r7, #4096 + svc #0 + + /* Exactly 18 dispatched 16-bit instructions, one load and one store. */ + movs r5, #0 + movs r6, #3 +1: + adds r5, #1 + subs r6, #1 + bne 1b + sub sp, #4 + str r5, [sp, #0] + ldr r6, [sp, #0] + add sp, #4 + + /* Restore magic (BEGIN returned zero), then change opcode 1 to opcode 2. */ + mov r0, r10 + adds r1, #1 + svc #0 + + /* Linux ARM EABI exit(0). This is outside the measurement. */ + movs r0, #0 + movs r7, #1 + svc #0 + .size _start, .-_start + + .section .note.GNU-stack,"",%progbits diff --git a/tools/perf/qemu/fixtures/marker-cases.c b/tools/perf/qemu/fixtures/marker-cases.c new file mode 100644 index 00000000..00f537c1 --- /dev/null +++ b/tools/perf/qemu/fixtures/marker-cases.c @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: MIT */ +#include +#include +#include + +#include "../guest_marker.h" + +static volatile uint32_t value = 1; + +static int marker_ok(int64_t result) +{ + return result == 0 ? 0 : 2; +} + +static int request_random_byte(void) +{ + uint8_t byte; + return getrandom(&byte, sizeof(byte), 0) == (ssize_t)sizeof(byte) ? 0 : 3; +} + +int main(int argc, char **argv) +{ + const char *mode = argc > 1 ? argv[1] : "valid"; + + if (strcmp(mode, "valid") == 0) { + if (marker_ok(pocketjs_perf_begin(11, 3)) != 0) { + return 2; + } + value = value * 3 + 1; + return marker_ok(pocketjs_perf_end(11, 3)); + } + if (strcmp(mode, "valid-loop") == 0) { + uint32_t i; + if (marker_ok(pocketjs_perf_begin(11, 3)) != 0) { + return 2; + } + for (i = 0; i < 20000; i++) { + value = value * 3 + i; + } + return marker_ok(pocketjs_perf_end(11, 3)); + } + if (strcmp(mode, "getrandom-before") == 0) { + if (request_random_byte() != 0 || + marker_ok(pocketjs_perf_begin(11, 3)) != 0) { + return 3; + } + value = value * 3 + 1; + return marker_ok(pocketjs_perf_end(11, 3)); + } + if (strcmp(mode, "getrandom-active") == 0) { + if (marker_ok(pocketjs_perf_begin(11, 3)) != 0 || + request_random_byte() != 0) { + return 3; + } + return marker_ok(pocketjs_perf_end(11, 3)); + } + if (strcmp(mode, "nested") == 0) { + pocketjs_perf_begin(11, 3); + pocketjs_perf_begin(11, 3); + pocketjs_perf_end(11, 3); + return 0; + } + if (strcmp(mode, "mismatch") == 0) { + pocketjs_perf_begin(11, 3); + pocketjs_perf_end(12, 3); + return 0; + } + if (strcmp(mode, "unexpected-end") == 0) { + pocketjs_perf_end(11, 3); + return 0; + } + if (strcmp(mode, "missing-end") == 0) { + pocketjs_perf_begin(11, 3); + return 0; + } + if (strcmp(mode, "none") == 0) { + return 0; + } + return 64; +} diff --git a/tools/perf/qemu/fixtures/multivcpu.c b/tools/perf/qemu/fixtures/multivcpu.c new file mode 100644 index 00000000..0b25259e --- /dev/null +++ b/tools/perf/qemu/fixtures/multivcpu.c @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: MIT */ +#include +#include + +#include "../guest_marker.h" + +static void *worker(void *argument) +{ + volatile uintptr_t value = (uintptr_t)argument; + value++; + return (void *)(uintptr_t)value; +} + +int main(void) +{ + pthread_t thread; + void *result = 0; + + if (pthread_create(&thread, 0, worker, (void *)(uintptr_t)6) != 0) { + return 2; + } + if (pthread_join(thread, &result) != 0 || (uintptr_t)result != 7) { + return 3; + } + + pocketjs_perf_begin(19, 0); + pocketjs_perf_end(19, 0); + return 0; +} diff --git a/tools/perf/qemu/guest_marker.h b/tools/perf/qemu/guest_marker.h new file mode 100644 index 00000000..03af1025 --- /dev/null +++ b/tools/perf/qemu/guest_marker.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef POCKETJS_PERF_GUEST_MARKER_H +#define POCKETJS_PERF_GUEST_MARKER_H + +#include + +#define POCKETJS_PERF_MARKER_SYSCALL 4096U +#define POCKETJS_PERF_MARKER_MAGIC UINT32_C(0x504a424d) +#define POCKETJS_PERF_MARKER_COOKIE UINT32_C(0xc001c0de) +#define POCKETJS_PERF_MARKER_VERSION UINT32_C(1) +#define POCKETJS_PERF_MARKER_BEGIN UINT32_C(1) +#define POCKETJS_PERF_MARKER_END UINT32_C(2) + +/* bits 15..8: protocol version; bits 7..0: opcode; bits 31..16: zero */ +#define POCKETJS_PERF_MARKER_PACK(opcode) \ + ((POCKETJS_PERF_MARKER_VERSION << 8) | (opcode)) + +#if defined(__arm__) && !defined(__aarch64__) + +static __attribute__((always_inline)) inline int32_t +pocketjs_perf_marker(uint32_t opcode, uint32_t phase_id, uint32_t iteration) +{ + register uint32_t r0 __asm__("r0") = POCKETJS_PERF_MARKER_MAGIC; + register uint32_t r1 __asm__("r1") = POCKETJS_PERF_MARKER_PACK(opcode); + register uint32_t r2 __asm__("r2") = phase_id; + register uint32_t r3 __asm__("r3") = iteration; + register uint32_t r4 __asm__("r4") = POCKETJS_PERF_MARKER_COOKIE; + register uint32_t r7 __asm__("r7") = POCKETJS_PERF_MARKER_SYSCALL; + + __asm__ volatile("svc #0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r7) + : "memory", "cc"); + return (int32_t)r0; +} + +#elif defined(__aarch64__) + +static __attribute__((always_inline)) inline int64_t +pocketjs_perf_marker(uint32_t opcode, uint32_t phase_id, uint32_t iteration) +{ + register uint64_t x0 __asm__("x0") = POCKETJS_PERF_MARKER_MAGIC; + register uint64_t x1 __asm__("x1") = POCKETJS_PERF_MARKER_PACK(opcode); + register uint64_t x2 __asm__("x2") = phase_id; + register uint64_t x3 __asm__("x3") = iteration; + register uint64_t x4 __asm__("x4") = POCKETJS_PERF_MARKER_COOKIE; + register uint64_t x8 __asm__("x8") = POCKETJS_PERF_MARKER_SYSCALL; + + __asm__ volatile("svc #0" + : "+r"(x0) + : "r"(x1), "r"(x2), "r"(x3), "r"(x4), "r"(x8) + : "memory", "cc"); + return (int64_t)x0; +} + +#else +#error "PocketJS QEMU perf markers support only ARM32 and AArch64 guests" +#endif + +static __attribute__((always_inline)) inline int64_t +pocketjs_perf_begin(uint32_t phase_id, uint32_t iteration) +{ + return pocketjs_perf_marker(POCKETJS_PERF_MARKER_BEGIN, + phase_id, iteration); +} + +static __attribute__((always_inline)) inline int64_t +pocketjs_perf_end(uint32_t phase_id, uint32_t iteration) +{ + return pocketjs_perf_marker(POCKETJS_PERF_MARKER_END, + phase_id, iteration); +} + +#endif diff --git a/tools/perf/qemu/perf_counter.c b/tools/perf/qemu/perf_counter.c new file mode 100644 index 00000000..d5516c04 --- /dev/null +++ b/tools/perf/qemu/perf_counter.c @@ -0,0 +1,447 @@ +/* + * PocketJS deterministic guest-work counter for QEMU linux-user. + * + * This plugin intentionally targets the QEMU 11.0.3 plugin API (version 6). + * It counts guest events continuously in per-vCPU scoreboards and snapshots + * one vCPU at matching BEGIN/END magic syscalls. + * + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include +#include +#include + +#include + +#define OUTPUT_PREFIX "POCKETJS_PERF_QEMU " +#define OUTPUT_SCHEMA "pocketjs.perf.qemu" +#define OUTPUT_VERSION 1 +#define BUILT_FOR_QEMU "11.0.3" + +#define MARKER_SYSCALL 4096 +#define MARKER_MAGIC UINT64_C(0x504a424d) +#define MARKER_COOKIE UINT64_C(0xc001c0de) +#define MARKER_VERSION UINT64_C(1) +#define MARKER_OPCODE_BEGIN UINT64_C(1) +#define MARKER_OPCODE_END UINT64_C(2) +#define ARM_GETRANDOM_SYSCALL INT64_C(384) +#define AARCH64_GETRANDOM_SYSCALL INT64_C(278) + +QEMU_PLUGIN_EXPORT int qemu_plugin_version = QEMU_PLUGIN_VERSION; + +typedef struct { + uint64_t guest_insn_dispatched; + uint64_t guest_instruction_bytes; + uint64_t guest_insn_size_2; + uint64_t guest_insn_size_4; + uint64_t guest_load_events; + uint64_t guest_store_events; +} VcpuCounters; + +typedef struct { + bool active; + bool failed; + bool first_vcpu_seen; + unsigned int active_vcpu; + unsigned int first_vcpu; + unsigned int vcpu_init_calls; + uint32_t phase_id; + uint32_t iteration; + uint64_t measurement_count; + const char *error_code; + VcpuCounters begin; +} ProtocolState; + +static struct qemu_plugin_scoreboard *counter_scoreboard; +static qemu_plugin_u64 insn_count; +static qemu_plugin_u64 instruction_bytes; +static qemu_plugin_u64 insn_size_2; +static qemu_plugin_u64 insn_size_4; +static qemu_plugin_u64 load_events; +static qemu_plugin_u64 store_events; + +static GMutex state_lock; +static ProtocolState state; +static char *target_name; +static int64_t getrandom_syscall; + +static void emit_line(const char *json) +{ + char *line = g_strdup_printf(OUTPUT_PREFIX "%s\n", json); + qemu_plugin_outs(line); + g_free(line); +} + +static void emit_install_error(const char *code) +{ + char *json = g_strdup_printf( + "{\"schema\":\"%s\",\"version\":%d,\"event\":\"error\"," + "\"plugin_api\":%d,\"qemu_version\":\"%s\",\"code\":\"%s\"," + "\"measurements\":0}", + OUTPUT_SCHEMA, OUTPUT_VERSION, QEMU_PLUGIN_VERSION, BUILT_FOR_QEMU, + code); + emit_line(json); + g_free(json); +} + +static void fail_locked(const char *code) +{ + if (!state.failed) { + state.failed = true; + state.error_code = code; + } +} + +static VcpuCounters read_counters(unsigned int vcpu_index) +{ + VcpuCounters counters = { + .guest_insn_dispatched = qemu_plugin_u64_get(insn_count, vcpu_index), + .guest_instruction_bytes = + qemu_plugin_u64_get(instruction_bytes, vcpu_index), + .guest_insn_size_2 = qemu_plugin_u64_get(insn_size_2, vcpu_index), + .guest_insn_size_4 = qemu_plugin_u64_get(insn_size_4, vcpu_index), + .guest_load_events = qemu_plugin_u64_get(load_events, vcpu_index), + .guest_store_events = qemu_plugin_u64_get(store_events, vcpu_index), + }; + return counters; +} + +static bool counters_are_monotonic(const VcpuCounters *end, + const VcpuCounters *begin) +{ + return end->guest_insn_dispatched >= begin->guest_insn_dispatched && + end->guest_instruction_bytes >= begin->guest_instruction_bytes && + end->guest_insn_size_2 >= begin->guest_insn_size_2 && + end->guest_insn_size_4 >= begin->guest_insn_size_4 && + end->guest_load_events >= begin->guest_load_events && + end->guest_store_events >= begin->guest_store_events; +} + +static VcpuCounters subtract_counters(const VcpuCounters *end, + const VcpuCounters *begin) +{ + VcpuCounters delta = { + .guest_insn_dispatched = + end->guest_insn_dispatched - begin->guest_insn_dispatched, + .guest_instruction_bytes = + end->guest_instruction_bytes - begin->guest_instruction_bytes, + .guest_insn_size_2 = + end->guest_insn_size_2 - begin->guest_insn_size_2, + .guest_insn_size_4 = + end->guest_insn_size_4 - begin->guest_insn_size_4, + .guest_load_events = + end->guest_load_events - begin->guest_load_events, + .guest_store_events = + end->guest_store_events - begin->guest_store_events, + }; + return delta; +} + +static void emit_measurement(unsigned int vcpu_index, uint32_t phase_id, + uint32_t iteration, + const VcpuCounters *counters) +{ + char *json = g_strdup_printf( + "{\"schema\":\"%s\",\"version\":%d," + "\"event\":\"measurement\",\"plugin_api\":%d," + "\"qemu_version\":\"%s\",\"target\":\"%s\"," + "\"vcpu\":%u,\"phase_id\":%" PRIu32 "," + "\"iteration\":%" PRIu32 ",\"metrics\":{" + "\"guest_insn_dispatched\":%" PRIu64 "," + "\"guest_instruction_bytes\":%" PRIu64 "," + "\"guest_insn_size_2\":%" PRIu64 "," + "\"guest_insn_size_4\":%" PRIu64 "," + "\"guest_load_events\":%" PRIu64 "," + "\"guest_store_events\":%" PRIu64 "}}", + OUTPUT_SCHEMA, OUTPUT_VERSION, QEMU_PLUGIN_VERSION, BUILT_FOR_QEMU, + target_name, vcpu_index, phase_id, iteration, + counters->guest_insn_dispatched, + counters->guest_instruction_bytes, + counters->guest_insn_size_2, + counters->guest_insn_size_4, + counters->guest_load_events, + counters->guest_store_events); + emit_line(json); + g_free(json); +} + +static void vcpu_init(qemu_plugin_id_t id, unsigned int vcpu_index) +{ + (void)id; + + g_mutex_lock(&state_lock); + state.vcpu_init_calls++; + if (!state.first_vcpu_seen) { + state.first_vcpu_seen = true; + state.first_vcpu = vcpu_index; + } else { + /* linux-user guest threads are separate vCPUs. They are unsupported. */ + fail_locked("multiple_vcpus"); + } + g_mutex_unlock(&state_lock); +} + +static void vcpu_tb_trans(qemu_plugin_id_t id, struct qemu_plugin_tb *tb) +{ + size_t count = qemu_plugin_tb_n_insns(tb); + size_t i; + + (void)id; + + for (i = 0; i < count; i++) { + struct qemu_plugin_insn *insn = qemu_plugin_tb_get_insn(tb, i); + size_t size = qemu_plugin_insn_size(insn); + + qemu_plugin_register_vcpu_insn_exec_inline_per_vcpu( + insn, QEMU_PLUGIN_INLINE_ADD_U64, insn_count, 1); + qemu_plugin_register_vcpu_insn_exec_inline_per_vcpu( + insn, QEMU_PLUGIN_INLINE_ADD_U64, instruction_bytes, size); + + if (size == 2) { + qemu_plugin_register_vcpu_insn_exec_inline_per_vcpu( + insn, QEMU_PLUGIN_INLINE_ADD_U64, insn_size_2, 1); + } else if (size == 4) { + qemu_plugin_register_vcpu_insn_exec_inline_per_vcpu( + insn, QEMU_PLUGIN_INLINE_ADD_U64, insn_size_4, 1); + } + + qemu_plugin_register_vcpu_mem_inline_per_vcpu( + insn, QEMU_PLUGIN_MEM_R, QEMU_PLUGIN_INLINE_ADD_U64, + load_events, 1); + qemu_plugin_register_vcpu_mem_inline_per_vcpu( + insn, QEMU_PLUGIN_MEM_W, QEMU_PLUGIN_INLINE_ADD_U64, + store_events, 1); + } +} + +static bool marker_is_well_formed(uint64_t magic, uint64_t packed, + uint64_t phase_id, uint64_t iteration, + uint64_t cookie, uint64_t *opcode) +{ + uint64_t version; + + if (magic != MARKER_MAGIC) { + fail_locked("invalid_marker_magic"); + return false; + } + if (cookie != MARKER_COOKIE) { + fail_locked("invalid_marker_cookie"); + return false; + } + if ((packed & ~UINT64_C(0xffff)) != 0) { + fail_locked("invalid_marker_reserved_bits"); + return false; + } + + version = (packed >> 8) & UINT64_C(0xff); + *opcode = packed & UINT64_C(0xff); + if (version != MARKER_VERSION) { + fail_locked("unsupported_marker_version"); + return false; + } + if (*opcode != MARKER_OPCODE_BEGIN && *opcode != MARKER_OPCODE_END) { + fail_locked("invalid_marker_opcode"); + return false; + } + if (phase_id > UINT32_MAX || iteration > UINT32_MAX) { + fail_locked("invalid_marker_argument_range"); + return false; + } + return true; +} + +static bool vcpu_syscall_filter(qemu_plugin_id_t id, + unsigned int vcpu_index, int64_t number, + uint64_t a1, uint64_t a2, uint64_t a3, + uint64_t a4, uint64_t a5, uint64_t a6, + uint64_t a7, uint64_t a8, uint64_t *sysret) +{ + uint64_t opcode = 0; + VcpuCounters end; + VcpuCounters delta; + uint32_t phase_id; + uint32_t iteration; + bool should_emit = false; + + (void)id; + (void)a6; + (void)a7; + (void)a8; + + if (number == getrandom_syscall) { + g_mutex_lock(&state_lock); + if (state.active) { + /* Entropy inside a phase makes repeated counter runs diverge. */ + fail_locked("getrandom_during_measurement"); + } + g_mutex_unlock(&state_lock); + return false; + } + + if (number != MARKER_SYSCALL) { + return false; + } + + /* Never let the reserved benchmark syscall reach the host kernel. */ + *sysret = 0; + + g_mutex_lock(&state_lock); + + if (state.failed || + !marker_is_well_formed(a1, a2, a3, a4, a5, &opcode)) { + g_mutex_unlock(&state_lock); + return true; + } + + if (state.vcpu_init_calls != 1 || qemu_plugin_num_vcpus() != 1) { + fail_locked("multiple_vcpus"); + g_mutex_unlock(&state_lock); + return true; + } + + phase_id = (uint32_t)a3; + iteration = (uint32_t)a4; + + if (opcode == MARKER_OPCODE_BEGIN) { + if (state.active) { + fail_locked("nested_begin"); + } else if (!state.first_vcpu_seen || + state.first_vcpu != vcpu_index) { + fail_locked("begin_vcpu_mismatch"); + } else { + state.active = true; + state.active_vcpu = vcpu_index; + state.phase_id = phase_id; + state.iteration = iteration; + state.begin = read_counters(vcpu_index); + } + g_mutex_unlock(&state_lock); + return true; + } + + if (!state.active) { + fail_locked("unexpected_end"); + } else if (state.active_vcpu != vcpu_index) { + fail_locked("end_vcpu_mismatch"); + } else if (state.phase_id != phase_id || state.iteration != iteration) { + fail_locked("marker_mismatch"); + } else { + end = read_counters(vcpu_index); + if (!counters_are_monotonic(&end, &state.begin)) { + fail_locked("counter_overflow"); + } else { + delta = subtract_counters(&end, &state.begin); + state.active = false; + state.measurement_count++; + should_emit = true; + } + } + + g_mutex_unlock(&state_lock); + + if (should_emit) { + emit_measurement(vcpu_index, phase_id, iteration, &delta); + } + return true; +} + +static void plugin_exit(qemu_plugin_id_t id, void *userdata) +{ + const char *error_code = NULL; + uint64_t measurements; + char *json; + + (void)id; + (void)userdata; + + g_mutex_lock(&state_lock); + if (!state.failed && state.active) { + fail_locked("missing_end"); + } + if (!state.failed && state.measurement_count == 0) { + fail_locked("missing_begin"); + } + if (state.failed) { + error_code = state.error_code; + } + measurements = state.measurement_count; + g_mutex_unlock(&state_lock); + + if (error_code != NULL) { + json = g_strdup_printf( + "{\"schema\":\"%s\",\"version\":%d,\"event\":\"error\"," + "\"plugin_api\":%d,\"qemu_version\":\"%s\"," + "\"target\":\"%s\",\"code\":\"%s\"," + "\"measurements\":%" PRIu64 "}", + OUTPUT_SCHEMA, OUTPUT_VERSION, QEMU_PLUGIN_VERSION, + BUILT_FOR_QEMU, target_name, error_code, measurements); + } else { + json = g_strdup_printf( + "{\"schema\":\"%s\",\"version\":%d," + "\"event\":\"complete\",\"plugin_api\":%d," + "\"qemu_version\":\"%s\",\"target\":\"%s\"," + "\"measurements\":%" PRIu64 "}", + OUTPUT_SCHEMA, OUTPUT_VERSION, QEMU_PLUGIN_VERSION, + BUILT_FOR_QEMU, target_name, measurements); + } + emit_line(json); + g_free(json); + + qemu_plugin_scoreboard_free(counter_scoreboard); + g_free(target_name); + g_mutex_clear(&state_lock); +} + +QEMU_PLUGIN_EXPORT int qemu_plugin_install(qemu_plugin_id_t id, + const qemu_info_t *info, + int argc, char **argv) +{ + (void)argv; + + if (QEMU_PLUGIN_VERSION != 6 || info->version.cur != 6 || + info->version.min > 6) { + emit_install_error("plugin_api_mismatch"); + return -1; + } + if (info->system_emulation) { + emit_install_error("linux_user_required"); + return -1; + } + if (strcmp(info->target_name, "arm") != 0 && + strcmp(info->target_name, "aarch64") != 0) { + emit_install_error("unsupported_target"); + return -1; + } + if (argc != 0) { + emit_install_error("unexpected_plugin_argument"); + return -1; + } + + g_mutex_init(&state_lock); + target_name = g_strdup(info->target_name); + getrandom_syscall = strcmp(info->target_name, "arm") == 0 + ? ARM_GETRANDOM_SYSCALL + : AARCH64_GETRANDOM_SYSCALL; + counter_scoreboard = qemu_plugin_scoreboard_new(sizeof(VcpuCounters)); + insn_count = qemu_plugin_scoreboard_u64_in_struct( + counter_scoreboard, VcpuCounters, guest_insn_dispatched); + instruction_bytes = qemu_plugin_scoreboard_u64_in_struct( + counter_scoreboard, VcpuCounters, guest_instruction_bytes); + insn_size_2 = qemu_plugin_scoreboard_u64_in_struct( + counter_scoreboard, VcpuCounters, guest_insn_size_2); + insn_size_4 = qemu_plugin_scoreboard_u64_in_struct( + counter_scoreboard, VcpuCounters, guest_insn_size_4); + load_events = qemu_plugin_scoreboard_u64_in_struct( + counter_scoreboard, VcpuCounters, guest_load_events); + store_events = qemu_plugin_scoreboard_u64_in_struct( + counter_scoreboard, VcpuCounters, guest_store_events); + + qemu_plugin_register_vcpu_init_cb(id, vcpu_init); + qemu_plugin_register_vcpu_tb_trans_cb(id, vcpu_tb_trans); + qemu_plugin_register_vcpu_syscall_filter_cb(id, vcpu_syscall_filter); + qemu_plugin_register_atexit_cb(id, plugin_exit, NULL); + return 0; +} diff --git a/tools/perf/qemu/run-fixtures.sh b/tools/perf/qemu/run-fixtures.sh new file mode 100755 index 00000000..6ce6c7f8 --- /dev/null +++ b/tools/perf/qemu/run-fixtures.sh @@ -0,0 +1,114 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +BUILD_DIR=${BUILD_DIR:-"$SCRIPT_DIR/build"} +QEMU_PREFIX=${QEMU_PREFIX:-/opt/qemu} +PLUGIN=${PLUGIN:-"$BUILD_DIR/pocketjs-perf-counter.so"} +PYTHON=${PYTHON:-python3} +TMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/pocketjs-qemu-fixtures.XXXXXX") + +cleanup() { + rm -rf -- "$TMP_DIR" +} +trap cleanup EXIT HUP INT TERM + +require_version() { + emulator=$1 + version=$($emulator --version | sed -n '1p') + case "$version" in + *"version 11.0.3"*) ;; + *) + echo "expected QEMU 11.0.3, got: $version" >&2 + exit 2 + ;; + esac +} + +run_guest() { + name=$1 + emulator=$2 + binary=$3 + shift 3 + if [ "$emulator" = "$QEMU_ARM" ]; then + "$emulator" -cpu "cortex-a9,neon=off,vfp-d32=off" \ + -seed 1 \ + -d plugin -plugin "$PLUGIN" "$binary" "$@" \ + >"$TMP_DIR/$name.log" 2>&1 + else + "$emulator" -cpu cortex-a53 \ + -seed 1 \ + -d plugin -plugin "$PLUGIN" "$binary" "$@" \ + >"$TMP_DIR/$name.log" 2>&1 + fi +} + +assert_output() { + "$PYTHON" "$SCRIPT_DIR/tests/assert_output.py" "$@" +} + +QEMU_ARM="$QEMU_PREFIX/bin/qemu-arm" +QEMU_AARCH64="$QEMU_PREFIX/bin/qemu-aarch64" +require_version "$QEMU_ARM" +require_version "$QEMU_AARCH64" + +attempt=1 +while [ "$attempt" -le 20 ]; do + run_guest "exact-armv7-$attempt" "$QEMU_ARM" "$BUILD_DIR/exact-armv7" + assert_output "$TMP_DIR/exact-armv7-$attempt.log" --complete --target arm \ + --metrics "$SCRIPT_DIR/tests/expected/exact-armv7.json" + + run_guest "exact-aarch64-$attempt" "$QEMU_AARCH64" "$BUILD_DIR/exact-aarch64" + assert_output "$TMP_DIR/exact-aarch64-$attempt.log" --complete --target aarch64 \ + --metrics "$SCRIPT_DIR/tests/expected/exact-aarch64.json" + attempt=$((attempt + 1)) +done + +for architecture in armv7 aarch64; do + if [ "$architecture" = armv7 ]; then + emulator=$QEMU_ARM + else + emulator=$QEMU_AARCH64 + fi + binary="$BUILD_DIR/marker-cases-$architecture" + + run_guest "valid-$architecture" "$emulator" "$binary" valid + assert_output "$TMP_DIR/valid-$architecture.log" --complete + + run_guest "valid-loop-$architecture" "$emulator" "$binary" valid-loop + assert_output "$TMP_DIR/valid-loop-$architecture.log" --complete + "$PYTHON" "$SCRIPT_DIR/tests/assert_metric_delta.py" \ + "$TMP_DIR/valid-$architecture.log" \ + "$TMP_DIR/valid-loop-$architecture.log" \ + --metric guest_insn_dispatched --minimum-delta 10000 + + run_guest "getrandom-before-$architecture" "$emulator" "$binary" \ + getrandom-before + assert_output "$TMP_DIR/getrandom-before-$architecture.log" --complete + + run_guest "getrandom-active-$architecture" "$emulator" "$binary" \ + getrandom-active + assert_output "$TMP_DIR/getrandom-active-$architecture.log" \ + --error getrandom_during_measurement + + run_guest "nested-$architecture" "$emulator" "$binary" nested + assert_output "$TMP_DIR/nested-$architecture.log" --error nested_begin + + run_guest "mismatch-$architecture" "$emulator" "$binary" mismatch + assert_output "$TMP_DIR/mismatch-$architecture.log" --error marker_mismatch + + run_guest "unexpected-end-$architecture" "$emulator" "$binary" unexpected-end + assert_output "$TMP_DIR/unexpected-end-$architecture.log" --error unexpected_end + + run_guest "missing-end-$architecture" "$emulator" "$binary" missing-end + assert_output "$TMP_DIR/missing-end-$architecture.log" --error missing_end + + run_guest "missing-begin-$architecture" "$emulator" "$binary" none + assert_output "$TMP_DIR/missing-begin-$architecture.log" --error missing_begin + + run_guest "multivcpu-$architecture" "$emulator" \ + "$BUILD_DIR/multivcpu-$architecture" + assert_output "$TMP_DIR/multivcpu-$architecture.log" --error multiple_vcpus +done + +echo "QEMU perf plugin fixtures passed" diff --git a/tools/perf/qemu/tests/assert_metric_delta.py b/tools/perf/qemu/tests/assert_metric_delta.py new file mode 100644 index 00000000..06d3ae39 --- /dev/null +++ b/tools/perf/qemu/tests/assert_metric_delta.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Require a deterministic injected workload to increase one guest counter.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from protocol_parser import parse_output + + +def metric(path: Path, name: str) -> int: + parsed = parse_output(path.read_text(encoding="utf-8")) + if parsed.terminal["event"] != "complete" or len(parsed.measurements) != 1: + raise ValueError(f"{path} is not one complete measurement") + return int(parsed.measurements[0]["metrics"][name]) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("base", type=Path) + parser.add_argument("candidate", type=Path) + parser.add_argument("--metric", required=True) + parser.add_argument("--minimum-delta", required=True, type=int) + args = parser.parse_args() + delta = metric(args.candidate, args.metric) - metric(args.base, args.metric) + if delta <= args.minimum_delta: + parser.error( + f"{args.metric} delta {delta} did not exceed {args.minimum_delta}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/perf/qemu/tests/assert_output.py b/tools/perf/qemu/tests/assert_output.py new file mode 100755 index 00000000..fa82230f --- /dev/null +++ b/tools/perf/qemu/tests/assert_output.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Validate a captured fixture run against its expected terminal and metrics.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from protocol_parser import ProtocolError, parse_output + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("log", type=Path) + parser.add_argument("--complete", action="store_true") + parser.add_argument("--error") + parser.add_argument("--metrics", type=Path) + parser.add_argument("--target") + args = parser.parse_args() + + try: + parsed = parse_output(args.log.read_text(encoding="utf-8")) + except ProtocolError as error: + parser.error(str(error)) + + if args.complete and parsed.terminal["event"] != "complete": + parser.error(f"expected complete, got {parsed.terminal!r}") + if args.error is not None: + actual = parsed.terminal.get("code") + if parsed.terminal["event"] != "error" or actual != args.error: + parser.error(f"expected error {args.error!r}, got {parsed.terminal!r}") + if args.target is not None and parsed.terminal.get("target") != args.target: + parser.error(f"expected target {args.target!r}, got {parsed.terminal!r}") + if args.metrics is not None: + expected = json.loads(args.metrics.read_text(encoding="utf-8")) + if len(parsed.measurements) != 1: + parser.error(f"expected one measurement, got {len(parsed.measurements)}") + actual = parsed.measurements[0]["metrics"] + if actual != expected: + parser.error(f"metric mismatch: expected {expected!r}, got {actual!r}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/perf/qemu/tests/data/complete.log b/tools/perf/qemu/tests/data/complete.log new file mode 100644 index 00000000..45266c34 --- /dev/null +++ b/tools/perf/qemu/tests/data/complete.log @@ -0,0 +1,3 @@ +unrelated guest output is ignored +POCKETJS_PERF_QEMU {"schema":"pocketjs.perf.qemu","version":1,"event":"measurement","plugin_api":6,"qemu_version":"11.0.3","target":"arm","vcpu":0,"phase_id":7,"iteration":0,"metrics":{"guest_insn_dispatched":18,"guest_instruction_bytes":36,"guest_insn_size_2":18,"guest_insn_size_4":0,"guest_load_events":1,"guest_store_events":1}} +POCKETJS_PERF_QEMU {"schema":"pocketjs.perf.qemu","version":1,"event":"complete","plugin_api":6,"qemu_version":"11.0.3","target":"arm","measurements":1} diff --git a/tools/perf/qemu/tests/data/error.log b/tools/perf/qemu/tests/data/error.log new file mode 100644 index 00000000..8a3de6c1 --- /dev/null +++ b/tools/perf/qemu/tests/data/error.log @@ -0,0 +1 @@ +POCKETJS_PERF_QEMU {"schema":"pocketjs.perf.qemu","version":1,"event":"error","plugin_api":6,"qemu_version":"11.0.3","target":"arm","code":"missing_end","measurements":0} diff --git a/tools/perf/qemu/tests/data/malformed.log b/tools/perf/qemu/tests/data/malformed.log new file mode 100644 index 00000000..679866c5 --- /dev/null +++ b/tools/perf/qemu/tests/data/malformed.log @@ -0,0 +1 @@ +POCKETJS_PERF_QEMU {not-json} diff --git a/tools/perf/qemu/tests/expected/exact-aarch64.json b/tools/perf/qemu/tests/expected/exact-aarch64.json new file mode 100644 index 00000000..2ddabef1 --- /dev/null +++ b/tools/perf/qemu/tests/expected/exact-aarch64.json @@ -0,0 +1,8 @@ +{ + "guest_insn_dispatched": 18, + "guest_instruction_bytes": 72, + "guest_insn_size_2": 0, + "guest_insn_size_4": 18, + "guest_load_events": 1, + "guest_store_events": 1 +} diff --git a/tools/perf/qemu/tests/expected/exact-armv7.json b/tools/perf/qemu/tests/expected/exact-armv7.json new file mode 100644 index 00000000..3c6f4afc --- /dev/null +++ b/tools/perf/qemu/tests/expected/exact-armv7.json @@ -0,0 +1,8 @@ +{ + "guest_insn_dispatched": 18, + "guest_instruction_bytes": 36, + "guest_insn_size_2": 18, + "guest_insn_size_4": 0, + "guest_load_events": 1, + "guest_store_events": 1 +} diff --git a/tools/perf/qemu/tests/protocol_parser.py b/tools/perf/qemu/tests/protocol_parser.py new file mode 100644 index 00000000..459bfbb2 --- /dev/null +++ b/tools/perf/qemu/tests/protocol_parser.py @@ -0,0 +1,84 @@ +"""Strict parser for the QEMU fixture runner's prefixed NDJSON.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any + +PREFIX = "POCKETJS_PERF_QEMU " +SCHEMA = "pocketjs.perf.qemu" +VERSION = 1 +METRICS = { + "guest_insn_dispatched", + "guest_instruction_bytes", + "guest_insn_size_2", + "guest_insn_size_4", + "guest_load_events", + "guest_store_events", +} + + +class ProtocolError(ValueError): + pass + + +@dataclass(frozen=True) +class ParsedOutput: + measurements: tuple[dict[str, Any], ...] + terminal: dict[str, Any] + + +def parse_output(output: str) -> ParsedOutput: + records: list[dict[str, Any]] = [] + + for line_number, line in enumerate(output.splitlines(), 1): + if not line.startswith(PREFIX): + continue + payload = line[len(PREFIX) :] + try: + record = json.loads(payload) + except json.JSONDecodeError as error: + raise ProtocolError( + f"invalid JSON on protocol line {line_number}: {error.msg}" + ) from error + if not isinstance(record, dict): + raise ProtocolError(f"protocol line {line_number} is not an object") + if record.get("schema") != SCHEMA or record.get("version") != VERSION: + raise ProtocolError(f"schema mismatch on protocol line {line_number}") + records.append(record) + + if not records: + raise ProtocolError("no QEMU perf protocol records") + + terminals = [ + record for record in records if record.get("event") in {"complete", "error"} + ] + if len(terminals) != 1: + raise ProtocolError("expected exactly one complete/error sentinel") + if records[-1] is not terminals[0]: + raise ProtocolError("complete/error sentinel is not the final protocol record") + + measurements = tuple( + record for record in records if record.get("event") == "measurement" + ) + known_count = len(measurements) + 1 + if len(records) != known_count: + raise ProtocolError("unknown protocol event") + + for measurement in measurements: + metrics = measurement.get("metrics") + if not isinstance(metrics, dict) or set(metrics) != METRICS: + raise ProtocolError("measurement metric set mismatch") + if any(type(value) is not int or value < 0 for value in metrics.values()): + raise ProtocolError("measurement metrics must be non-negative integers") + + terminal = terminals[0] + if terminal.get("measurements") != len(measurements): + raise ProtocolError("terminal measurement count mismatch") + if terminal["event"] == "complete" and not measurements: + raise ProtocolError("complete sentinel has no measurements") + if terminal["event"] == "error" and not isinstance(terminal.get("code"), str): + raise ProtocolError("error sentinel has no code") + + return ParsedOutput(measurements=measurements, terminal=terminal) diff --git a/tools/perf/qemu/tests/test_protocol_parser.py b/tools/perf/qemu/tests/test_protocol_parser.py new file mode 100644 index 00000000..77d5cc07 --- /dev/null +++ b/tools/perf/qemu/tests/test_protocol_parser.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import unittest +from pathlib import Path + +from protocol_parser import ProtocolError, parse_output + + +FIXTURES = Path(__file__).with_name("data") + + +class ProtocolParserTests(unittest.TestCase): + def test_complete_fixture(self) -> None: + parsed = parse_output((FIXTURES / "complete.log").read_text()) + self.assertEqual(parsed.terminal["event"], "complete") + self.assertEqual(len(parsed.measurements), 1) + self.assertEqual( + parsed.measurements[0]["metrics"]["guest_insn_dispatched"], 18 + ) + + def test_error_fixture(self) -> None: + parsed = parse_output((FIXTURES / "error.log").read_text()) + self.assertEqual(parsed.terminal["event"], "error") + self.assertEqual(parsed.terminal["code"], "missing_end") + + def test_rejects_malformed_prefixed_json(self) -> None: + with self.assertRaises(ProtocolError): + parse_output((FIXTURES / "malformed.log").read_text()) + + def test_requires_terminal_to_be_last(self) -> None: + complete = (FIXTURES / "complete.log").read_text() + measurement = complete.splitlines()[1] + with self.assertRaises(ProtocolError): + parse_output(complete + measurement + "\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/perf/receipts/factory.ts b/tools/perf/receipts/factory.ts new file mode 100644 index 00000000..cf509a5d --- /dev/null +++ b/tools/perf/receipts/factory.ts @@ -0,0 +1,494 @@ +import { gateMetricIds } from "../core/catalog.ts"; +import { parseReceiptV1 } from "../core/schema.ts"; +import type { + CorrectnessReceiptV1, + MetricSampleV1, + ReceiptProvenanceV1, + ReceiptV1, + ScenarioV1, +} from "../core/types.ts"; +import type { NativeRunResult } from "../runner/native.ts"; +import { + guestDigestToSha256, + scenarioPhaseId, + sha256Json, +} from "./hash.ts"; +import { parseNativeResult } from "./native-protocol.ts"; +import { + parseGuestOutput, + parseQemuOutput, + type GuestPhaseRecordV1, + type QemuMeasurementRecordV1, + type QemuTarget, +} from "./protocol.ts"; + +const SHA256 = /^[a-f0-9]{64}$/; + +export type ReceiptEnvironmentV1 = Omit; +export type ArtifactMetricId = + | "artifact.bundle_bytes" + | "artifact.pak_bytes" + | "artifact.elf_text_rodata_bytes"; +export type ArtifactMetrics = Readonly>>; + +export interface ReceiptFactoryOptions { + readonly provenance: ReceiptEnvironmentV1; + readonly artifactMetrics?: ArtifactMetrics; + readonly createdAt?: string; +} + +export interface NativeReceiptOptions extends ReceiptFactoryOptions { + /** Optional assertion for callers that already wrapped the observed FNV digest. */ + readonly observedDrawListHash?: string; +} + +export interface QemuReceiptOptions extends ReceiptFactoryOptions { + readonly target: QemuTarget; + /** Optional assertion from an independent correctness oracle. */ + readonly framebufferHash?: string; + /** Guest protocol emitted by the separate observational correctness replay. */ + readonly correctnessGuestOutput?: string; +} + +export const NATIVE_CORRECTNESS_MAPPING = Object.freeze({ + framebufferHash: "correctness.framebufferTraceHash", + drawListHash: "SHA-256 envelope of correctness.drawListHash", + stateHash: "correctness.stateHash", + effectHash: "correctness.effectHash", + replayInvariant: + "correctness final framebuffer/drawList hashes equal their measurement replay counterparts", +} as const); + +function exact(value: number, unit: "count" | "bytes" | "ns"): MetricSampleV1 { + return { kind: "exact", value, unit }; +} + +function date(options: ReceiptFactoryOptions): string { + return options.createdAt ?? new Date().toISOString(); +} + +function provenance( + scenario: ScenarioV1, + scenarioKey: string, + environment: ReceiptEnvironmentV1, +): ReceiptProvenanceV1 { + return { + ...environment, + scenario: { + id: scenarioKey, + suite: scenario.suite, + framework: scenario.subject.framework, + manifestHash: sha256Json(scenario), + inputTapeHash: sha256Json(scenario.tape), + }, + }; +} + +function receipt( + scenario: ScenarioV1, + scenarioKey: string, + options: ReceiptFactoryOptions, + metrics: Readonly>, + correctness: CorrectnessReceiptV1 | null, + reasons: readonly string[], + unsupportedMetrics: readonly string[] = [], +): ReceiptV1 { + const uniqueReasons = [...new Set(reasons.filter((reason) => reason.length > 0))]; + const gateMetrics = gateMetricIds(scenario.params); + const value = uniqueReasons.length === 0 + ? { + schemaVersion: 1, + kind: "pocketjs.perf.receipt", + createdAt: date(options), + status: "valid", + invalidReasons: [], + provenance: provenance(scenario, scenarioKey, options.provenance), + gateMetrics, + unsupportedMetrics, + correctness, + metrics, + } + : { + schemaVersion: 1, + kind: "pocketjs.perf.receipt", + createdAt: date(options), + status: "invalid", + invalidReasons: uniqueReasons, + provenance: provenance(scenario, scenarioKey, options.provenance), + gateMetrics, + unsupportedMetrics, + correctness, + metrics, + }; + return parseReceiptV1(value); +} + +function collectArtifactMetrics( + input: unknown, + reasons: string[], +): Record { + const metrics: Record = {}; + if (input === undefined) return metrics; + if (typeof input !== "object" || input === null || Array.isArray(input)) { + reasons.push("artifact metrics must be an object"); + return metrics; + } + const allowed = new Set([ + "artifact.bundle_bytes", + "artifact.pak_bytes", + "artifact.elf_text_rodata_bytes", + ]); + for (const [id, value] of Object.entries(input)) { + if (!allowed.has(id as ArtifactMetricId)) { + reasons.push(`unknown artifact metric ${id}`); + } else if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + reasons.push(`${id} must be a non-negative safe integer`); + } else { + metrics[id] = exact(value, "bytes"); + } + } + return metrics; +} + +function nativeArtifactMetrics( + result: Extract, + external: ArtifactMetrics | undefined, + reasons: string[], +): Record { + const metrics = collectArtifactMetrics(external, reasons); + for (const id of ["artifact.bundle_bytes", "artifact.pak_bytes"] as const) { + const sample = result.exactMetrics[id]; + if (!sample) continue; + if (sample.unit !== "bytes" || !Number.isSafeInteger(sample.value) || sample.value < 0) { + reasons.push(`native exact metric ${id} is invalid`); + continue; + } + const externalSample = metrics[id]; + if (externalSample?.kind === "exact" && externalSample.value !== sample.value) { + reasons.push(`native and external ${id} values disagree`); + } else { + metrics[id] = exact(sample.value, "bytes"); + } + } + return metrics; +} + +/** Convert one native scenario result into one receipt-v1 document. */ +export function createNativeReceipt( + scenario: ScenarioV1, + value: unknown, + options: NativeReceiptOptions, +): ReceiptV1 { + const parsed = parseNativeResult(value); + const protocolReasons: string[] = parsed.success === false ? [...parsed.reasons] : []; + if (!parsed.success) { + return receipt( + scenario, + scenario.id, + options, + collectArtifactMetrics(options.artifactMetrics, protocolReasons), + null, + protocolReasons, + ); + } + const result = parsed.data; + if (result.scenarioId !== scenario.id) { + protocolReasons.push( + `native scenarioId ${JSON.stringify(result.scenarioId)} does not match ${JSON.stringify(scenario.id)}`, + ); + } + if (result.status === "unsupported") { + protocolReasons.push(...result.reasons.map((reason) => `native executor unsupported: ${reason}`)); + return receipt( + scenario, + scenario.id, + options, + collectArtifactMetrics(options.artifactMetrics, protocolReasons), + null, + protocolReasons, + ); + } + + const metrics = nativeArtifactMetrics(result, options.artifactMetrics, protocolReasons); + const measuredWallTime = result.measurement.phases.reduce((sum, phase) => sum + phase.wallTimeNs, 0); + if (!Number.isSafeInteger(measuredWallTime)) { + protocolReasons.push("native measured wall time exceeds the safe integer range"); + } else { + metrics["native.wall_time_ns"] = exact(measuredWallTime, "ns"); + } + const reportedWallTime = result.diagnosticMetrics["native.wall_time_ns"]; + if (!reportedWallTime || reportedWallTime.unit !== "ns" || reportedWallTime.value !== measuredWallTime) { + protocolReasons.push("native.wall_time_ns does not equal the sum of measured phases"); + } + + const expectedPhases = scenario.phases.filter((phase) => phase.collect); + if (result.measurement.phases.length !== expectedPhases.length) { + protocolReasons.push( + `native measured ${result.measurement.phases.length} phases; expected ${expectedPhases.length}`, + ); + } + expectedPhases.forEach((expected, index) => { + const actual = result.measurement.phases[index]; + if (!actual || actual.name !== expected.name || actual.startFrame !== expected.startFrame || + actual.endFrame !== expected.endFrame) { + protocolReasons.push(`native measured phase ${index} does not match ${expected.name}`); + } + }); + if (result.correctness.finalFramebufferHash !== result.measurement.finalFramebufferHash) { + protocolReasons.push("native correctness and measurement final framebuffers differ"); + } + if (result.correctness.drawListHash !== result.measurement.finalDrawListHash) { + protocolReasons.push("native correctness and measurement final draw lists differ"); + } + + let correctness: CorrectnessReceiptV1 | null = null; + let observedDrawListHash: string | null = null; + try { + observedDrawListHash = guestDigestToSha256("draw-list", result.correctness.drawListHash); + } catch { + // parseNativeResult normally reports this first; retain a defensive check + // for callers that bypass TypeScript types at runtime. + protocolReasons.push("native correctness replay did not capture a valid drawListHash"); + } + if (options.observedDrawListHash !== undefined) { + if (!SHA256.test(options.observedDrawListHash)) { + protocolReasons.push("observedDrawListHash must be a lowercase SHA-256 digest"); + } else if (observedDrawListHash !== options.observedDrawListHash) { + protocolReasons.push("observedDrawListHash does not match the native correctness replay"); + } + } + if (observedDrawListHash) { + // framebufferHash intentionally represents the complete trace. The final + // frame is separately checked above as a replay invariant. + correctness = { + framebufferHash: result.correctness.framebufferTraceHash, + drawListHash: observedDrawListHash, + stateHash: result.correctness.stateHash, + effectHash: result.correctness.effectHash, + }; + } + + const requestedGateMetrics = gateMetricIds(scenario.params); + const requestedGateMetricSet = new Set(requestedGateMetrics); + const unsupportedMetrics = new Set(result.unsupportedMetrics); + for (const metricId of unsupportedMetrics) { + if (!requestedGateMetricSet.has(metricId)) { + protocolReasons.push(`native marked non-gate metric ${metricId} as unsupported`); + } + if (Object.hasOwn(metrics, metricId)) { + protocolReasons.push(`native gate metric ${metricId} is both observed and unsupported`); + } + } + for (const metricId of requestedGateMetrics) { + if (!Object.hasOwn(metrics, metricId) && !unsupportedMetrics.has(metricId)) { + protocolReasons.push(`required gate metric ${metricId} is missing without an explicit native unsupported declaration`); + } + } + + return receipt( + scenario, + scenario.id, + options, + metrics, + correctness, + protocolReasons, + result.unsupportedMetrics, + ); +} + +interface ExpectedPhase { + readonly name: string; + readonly id: number; + readonly endFrame: number | null; +} + +function expectedQemuPhases(scenario: ScenarioV1): ExpectedPhase[] { + const names: { name: string; endFrame: number | null }[] = []; + if (scenario.params.measureBoot === true) { + names.push({ name: "runtime-init", endFrame: null }, { name: "bundle-eval", endFrame: null }); + } + names.push(...scenario.phases + .filter((phase) => phase.collect) + .map((phase) => ({ name: phase.name, endFrame: phase.endFrame }))); + return names.map((phase) => ({ + ...phase, + id: scenarioPhaseId(scenario.id, phase.name), + })); +} + +function metricsForQemuPhase( + guest: GuestPhaseRecordV1 | undefined, + qemu: QemuMeasurementRecordV1 | undefined, + artifacts: Readonly>, + reasons: string[], +): Record { + const metrics: Record = { ...artifacts }; + if (qemu) { + const counters = qemu.metrics; + const loadStores = counters.guest_load_events + counters.guest_store_events; + if (!Number.isSafeInteger(loadStores)) reasons.push("guest load/store sum exceeds the safe integer range"); + else metrics["guest.load_store_events"] = exact(loadStores, "count"); + metrics["guest.instructions"] = exact(counters.guest_insn_dispatched, "count"); + metrics["guest.instruction_bytes"] = exact(counters.guest_instruction_bytes, "bytes"); + metrics["guest.thumb16_instructions"] = exact(counters.guest_insn_size_2, "count"); + metrics["guest.thumb32_instructions"] = exact(counters.guest_insn_size_4, "count"); + metrics["guest.loads"] = exact(counters.guest_load_events, "count"); + metrics["guest.stores"] = exact(counters.guest_store_events, "count"); + } + if (guest) { + metrics["memory.allocations"] = exact(guest.allocCalls, "count"); + metrics["memory.allocated_bytes"] = exact(guest.allocatedBytes, "bytes"); + metrics["memory.current_bytes"] = exact(guest.currentBytes, "bytes"); + metrics["memory.peak_bytes"] = exact(guest.peakBytes, "bytes"); + metrics["quickjs.live_bytes_after_gc"] = exact(guest.quickjsLiveBytesAfterGc, "bytes"); + } + return metrics; +} + +/** + * Parse an interleaved QEMU/guest log and create one receipt per phase. In + * receipt schema v1 the phase is encoded in scenario.id as `scenario#phase`; + * changing that key would make old and new receipts spuriously comparable. + */ +export function createQemuReceipts( + scenario: ScenarioV1, + output: string, + options: QemuReceiptOptions, +): readonly ReceiptV1[] { + const generatedCTrace = scenario.subject.family === "vapor"; + const guest = parseGuestOutput(output, { + framebufferTraceHash: generatedCTrace ? "required" : "forbidden", + }); + const correctnessGuest = options.correctnessGuestOutput === undefined + ? null + : parseGuestOutput(options.correctnessGuestOutput, { framebufferTraceHash: "required" }); + const qemu = parseQemuOutput(output); + const phases = expectedQemuPhases(scenario); + const globalReasons = [ + ...guest.status === "invalid" ? guest.reasons : [], + ...qemu.status === "invalid" ? qemu.reasons : [], + ...(correctnessGuest?.status === "invalid" + ? correctnessGuest.reasons.map((reason) => `correctness guest: ${reason}`) + : []), + ]; + const artifacts = collectArtifactMetrics(options.artifactMetrics, globalReasons); + + if (guest.complete) { + if (guest.complete.scenarioId !== scenario.id) globalReasons.push("guest complete scenarioId mismatch"); + if (guest.complete.suite !== scenario.suite) globalReasons.push("guest complete suite mismatch"); + if (guest.complete.framework !== scenario.subject.framework) { + globalReasons.push("guest complete framework mismatch"); + } + } + if (options.framebufferHash !== undefined && !SHA256.test(options.framebufferHash)) { + globalReasons.push("framebufferHash must be a lowercase SHA-256 digest"); + } + const requiresSeparateGuestTrace = scenario.subject.family === "guest-app"; + if (requiresSeparateGuestTrace && correctnessGuest === null) { + globalReasons.push("QEMU guest-app receipt has no correctness guest output"); + } + const separateGuestTraceHash = correctnessGuest?.complete?.framebufferTraceHash; + if (separateGuestTraceHash && options.framebufferHash && + separateGuestTraceHash !== options.framebufferHash) { + globalReasons.push("QEMU correctness framebuffer trace differs from the independent correctness replay"); + } + const generatedCTraceHash = generatedCTrace ? guest.complete?.framebufferTraceHash : undefined; + if (generatedCTraceHash && options.framebufferHash && generatedCTraceHash !== options.framebufferHash) { + globalReasons.push("generated-C framebuffer trace differs from the independent correctness replay"); + } + const receiptFramebufferHash = generatedCTrace + ? generatedCTraceHash + : correctnessGuest === null + ? options.framebufferHash + : separateGuestTraceHash; + if (!receiptFramebufferHash) { + globalReasons.push("QEMU receipt has no framebuffer hash from a correctness replay"); + } + + if (guest.phases.length !== phases.length) { + globalReasons.push(`guest emitted ${guest.phases.length} phases; expected ${phases.length}`); + } + if (qemu.measurements.length !== phases.length) { + globalReasons.push(`QEMU emitted ${qemu.measurements.length} phases; expected ${phases.length}`); + } + const expectedIds = new Set(phases.map((phase) => phase.id)); + for (const phase of guest.phases) { + if (!expectedIds.has(phase.phaseId)) globalReasons.push(`unknown guest phaseId ${phase.phaseId}`); + } + for (const measurement of qemu.measurements) { + if (!expectedIds.has(measurement.phase_id)) { + globalReasons.push(`unknown QEMU phaseId ${measurement.phase_id}`); + } + if (measurement.target !== options.target) globalReasons.push("QEMU measurement target mismatch"); + if (measurement.vcpu !== 0) globalReasons.push(`QEMU measurement used unexpected vCPU ${measurement.vcpu}`); + } + if (qemu.terminal && Object.hasOwn(qemu.terminal, "target") && qemu.terminal.target !== options.target) { + globalReasons.push("QEMU terminal target mismatch"); + } + + if (phases.length === 0) { + return [receipt( + scenario, + `${scenario.id}#protocol`, + options, + artifacts, + null, + [...globalReasons, "scenario defines no measurable QEMU phases"], + )]; + } + + return phases.map((expected, index) => { + const reasons = [...globalReasons]; + const guestPhase = guest.phases[index]; + const qemuPhase = qemu.measurements[index]; + if (!guestPhase) reasons.push(`missing guest phase ${expected.name}`); + else { + if (guestPhase.scenarioId !== scenario.id) reasons.push(`guest phase ${expected.name} scenarioId mismatch`); + if (guestPhase.phase !== expected.name) { + reasons.push(`guest phase order mismatch: expected ${expected.name}, got ${guestPhase.phase}`); + } + if (guestPhase.phaseId !== expected.id) { + reasons.push( + `guest phaseId mismatch for ${expected.name}: expected ${expected.id}, got ${guestPhase.phaseId}`, + ); + } + if (guestPhase.iteration !== 0) reasons.push(`guest phase ${expected.name} iteration must be 0`); + } + if (!qemuPhase) reasons.push(`missing QEMU measurement ${expected.name}`); + else { + if (qemuPhase.phase_id !== expected.id) { + reasons.push( + `QEMU phaseId mismatch for ${expected.name}: expected ${expected.id}, got ${qemuPhase.phase_id}`, + ); + } + if (qemuPhase.iteration !== 0) reasons.push(`QEMU phase ${expected.name} iteration must be 0`); + } + if (guestPhase && qemuPhase && + (guestPhase.phaseId !== qemuPhase.phase_id || guestPhase.iteration !== qemuPhase.iteration)) { + reasons.push(`guest/QEMU phase marker mismatch for ${expected.name}`); + } + + if (expected.endFrame === scenario.frames && guest.complete && guestPhase && + guestPhase.drawListHash !== guest.complete.finalDrawListHash) { + reasons.push(`final guest draw-list hash differs from phase ${expected.name}`); + } + + let correctness: CorrectnessReceiptV1 | null = null; + if (receiptFramebufferHash && SHA256.test(receiptFramebufferHash) && guest.complete && guestPhase) { + correctness = { + framebufferHash: receiptFramebufferHash, + drawListHash: guestDigestToSha256("draw-list", guestPhase.drawListHash), + stateHash: guestDigestToSha256("state", guest.complete.finalStateHash), + effectHash: guestDigestToSha256("effects", guest.complete.effectHash), + }; + } + const metrics = metricsForQemuPhase(guestPhase, qemuPhase, artifacts, reasons); + const requiredMetrics = gateMetricIds(scenario.params); + for (const metricId of requiredMetrics) { + if (!Object.hasOwn(metrics, metricId)) { + reasons.push(`required gate metric ${metricId} is missing`); + } + } + return receipt(scenario, `${scenario.id}#${expected.name}`, options, metrics, correctness, reasons); + }); +} diff --git a/tools/perf/receipts/hash.ts b/tools/perf/receipts/hash.ts new file mode 100644 index 00000000..29083afd --- /dev/null +++ b/tools/perf/receipts/hash.ts @@ -0,0 +1,79 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; + +/** Serialize JSON with recursively sorted object keys and no implicit coercions. */ +export function canonicalJson(value: unknown): string { + return canonicalize(value, new Set()); +} + +function canonicalize(value: unknown, ancestors: Set): string { + if (value === null || typeof value === "boolean" || typeof value === "string") { + return JSON.stringify(value); + } + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new TypeError("canonical JSON cannot contain a non-finite number"); + return JSON.stringify(value); + } + if (typeof value !== "object") { + throw new TypeError(`canonical JSON cannot contain ${typeof value}`); + } + if (ancestors.has(value)) throw new TypeError("canonical JSON cannot contain a cycle"); + ancestors.add(value); + try { + if (Array.isArray(value)) { + const items: string[] = []; + for (let index = 0; index < value.length; index += 1) { + if (!Object.hasOwn(value, index)) throw new TypeError("canonical JSON cannot contain array holes"); + items.push(canonicalize(value[index], ancestors)); + } + return `[${items.join(",")}]`; + } + if (![Object.prototype, null].includes(Object.getPrototypeOf(value))) { + throw new TypeError("canonical JSON requires plain objects"); + } + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalize(record[key], ancestors)}`) + .join(",")}}`; + } finally { + ancestors.delete(value); + } +} + +export function sha256Bytes(value: string | Uint8Array): string { + return createHash("sha256").update(value).digest("hex"); +} + +export function sha256Json(value: unknown): string { + return sha256Bytes(canonicalJson(value)); +} + +export function sha256File(path: string): string { + return sha256Bytes(readFileSync(path)); +} + +/** Must remain byte-for-byte equivalent to tools/perf/guest/src/main.rs. */ +export function scenarioPhaseId(scenarioId: string, phase: string): number { + let hash = 0x811c9dc5; + for (const byte of new TextEncoder().encode(`${scenarioId}\0${phase}`)) { + hash ^= byte; + hash = Math.imul(hash, 0x01000193) >>> 0; + } + return hash; +} + +/** + * Receipt v1 stores SHA-256 values. The guest intentionally uses a cheap FNV + * digest in the measured process, so the host hashes that tagged digest into + * the receipt instead of mislabelling the 64-bit value as SHA-256. + */ +export function guestDigestToSha256( + domain: "draw-list" | "state" | "effects", + digest: string, +): string { + if (!/^fnv1a64:[a-f0-9]{16}$/.test(digest)) { + throw new TypeError(`invalid guest ${domain} digest: ${JSON.stringify(digest)}`); + } + return sha256Bytes(`pocketjs.perf.${domain}.fnv1a64\0${digest}`); +} diff --git a/tools/perf/receipts/index.ts b/tools/perf/receipts/index.ts new file mode 100644 index 00000000..be78e3a9 --- /dev/null +++ b/tools/perf/receipts/index.ts @@ -0,0 +1,4 @@ +export * from "./hash.ts"; +export * from "./protocol.ts"; +export * from "./native-protocol.ts"; +export * from "./factory.ts"; diff --git a/tools/perf/receipts/native-protocol.ts b/tools/perf/receipts/native-protocol.ts new file mode 100644 index 00000000..a5e26bde --- /dev/null +++ b/tools/perf/receipts/native-protocol.ts @@ -0,0 +1,202 @@ +import type { NativeRunResult } from "../runner/native.ts"; +import { isMetricId } from "../core/catalog.ts"; + +export const NATIVE_RUN_OUTPUT_PREFIX = "POCKETJS_PERF_NATIVE "; + +type PlainRecord = Record; + +export type NativeResultParseResult = + | { readonly success: true; readonly data: NativeRunResult } + | { readonly success: false; readonly reasons: readonly string[] }; + +const SHA256 = /^[a-f0-9]{64}$/; +const FNV1A64 = /^fnv1a64:[a-f0-9]{16}$/; + +function isRecord(value: unknown): value is PlainRecord { + return typeof value === "object" && value !== null && !Array.isArray(value) && + [Object.prototype, null].includes(Object.getPrototypeOf(value)); +} + +function keys( + value: unknown, + path: string, + required: readonly string[], + reasons: string[], +): PlainRecord | null { + if (!isRecord(value)) { + reasons.push(`${path} must be an object`); + return null; + } + const allowed = new Set(required); + for (const key of Object.keys(value)) { + if (!allowed.has(key)) reasons.push(`${path}.${key} is unknown`); + } + for (const key of required) { + if (!Object.hasOwn(value, key)) reasons.push(`${path}.${key} is missing`); + } + return value; +} + +function string(value: unknown, path: string, reasons: string[]): value is string { + if (typeof value === "string" && value.trim().length > 0) return true; + reasons.push(`${path} must be a non-empty string`); + return false; +} + +function uint(value: unknown, path: string, reasons: string[]): value is number { + if (typeof value === "number" && Number.isSafeInteger(value) && value >= 0) return true; + reasons.push(`${path} must be a non-negative safe integer`); + return false; +} + +function hash(value: unknown, path: string, reasons: string[]): value is string { + if (typeof value === "string" && SHA256.test(value)) return true; + reasons.push(`${path} must be a lowercase SHA-256 digest`); + return false; +} + +function drawHash(value: unknown, path: string, reasons: string[]): value is string { + if (typeof value === "string" && FNV1A64.test(value)) return true; + reasons.push(`${path} must be a lowercase FNV-1a-64 digest`); + return false; +} + +function validateMetricMap( + value: unknown, + path: string, + units: ReadonlySet, + reasons: string[], +): void { + if (!isRecord(value)) { + reasons.push(`${path} must be an object`); + return; + } + for (const [id, sampleValue] of Object.entries(value)) { + const sample = keys(sampleValue, `${path}.${id}`, ["value", "unit"], reasons); + if (!sample) continue; + uint(sample.value, `${path}.${id}.value`, reasons); + if (typeof sample.unit !== "string" || !units.has(sample.unit)) { + reasons.push(`${path}.${id}.unit is invalid`); + } + } +} + +/** Validate the native runner's JSON result before trusting any metric or hash. */ +export function parseNativeResult(value: unknown): NativeResultParseResult { + const reasons: string[] = []; + if (!isRecord(value)) return { success: false, reasons: ["native result must be an object"] }; + + if (value.status === "unsupported") { + const result = keys(value, "native", [ + "schemaVersion", "kind", "status", "scenarioId", "executor", "reasons", + ], reasons); + if (!result) return { success: false, reasons }; + if (result.schemaVersion !== 1) reasons.push("native.schemaVersion must be 1"); + if (result.kind !== "pocketjs.perf.native-result") reasons.push("native.kind is invalid"); + if (result.executor !== "native") reasons.push("native.executor must be native"); + string(result.scenarioId, "native.scenarioId", reasons); + if (!Array.isArray(result.reasons) || result.reasons.length === 0) { + reasons.push("native.reasons must be a non-empty array"); + } else { + result.reasons.forEach((reason, index) => string(reason, `native.reasons[${index}]`, reasons)); + } + } else if (value.status === "ok") { + const result = keys(value, "native", [ + "schemaVersion", "kind", "status", "scenarioId", "executor", "sourceRoot", + "correctness", "measurement", "diagnosticMetrics", "exactMetrics", + "unsupportedMetrics", + ], reasons); + if (!result) return { success: false, reasons }; + if (result.schemaVersion !== 1) reasons.push("native.schemaVersion must be 1"); + if (result.kind !== "pocketjs.perf.native-result") reasons.push("native.kind is invalid"); + if (result.executor !== "native") reasons.push("native.executor must be native"); + string(result.scenarioId, "native.scenarioId", reasons); + string(result.sourceRoot, "native.sourceRoot", reasons); + + const correctness = keys(result.correctness, "native.correctness", [ + "framebufferTraceHash", "finalFramebufferHash", "drawListHash", "stateHash", "effectHash", + "checkpoints", + ], reasons); + if (correctness) { + hash(correctness.framebufferTraceHash, "native.correctness.framebufferTraceHash", reasons); + hash(correctness.finalFramebufferHash, "native.correctness.finalFramebufferHash", reasons); + drawHash(correctness.drawListHash, "native.correctness.drawListHash", reasons); + hash(correctness.stateHash, "native.correctness.stateHash", reasons); + hash(correctness.effectHash, "native.correctness.effectHash", reasons); + if (!isRecord(correctness.checkpoints)) { + reasons.push("native.correctness.checkpoints must be an object"); + } else { + for (const [frame, captureValue] of Object.entries(correctness.checkpoints)) { + if (!/^(0|[1-9][0-9]*)$/.test(frame)) { + reasons.push(`native.correctness.checkpoints.${frame} has an invalid frame key`); + } + if (!isRecord(captureValue)) { + reasons.push(`native.correctness.checkpoints.${frame} must be an object`); + continue; + } + for (const [capture, digest] of Object.entries(captureValue)) { + if (!["framebuffer", "drawList", "state", "effects"].includes(capture)) { + reasons.push(`native.correctness.checkpoints.${frame}.${capture} is unknown`); + } + if (capture === "drawList") { + drawHash(digest, `native.correctness.checkpoints.${frame}.${capture}`, reasons); + } else { + hash(digest, `native.correctness.checkpoints.${frame}.${capture}`, reasons); + } + } + } + } + } + + const measurement = keys(result.measurement, "native.measurement", [ + "bootWallTimeNs", "phases", "finalFramebufferHash", "finalDrawListHash", + ], reasons); + if (measurement) { + uint(measurement.bootWallTimeNs, "native.measurement.bootWallTimeNs", reasons); + hash(measurement.finalFramebufferHash, "native.measurement.finalFramebufferHash", reasons); + drawHash(measurement.finalDrawListHash, "native.measurement.finalDrawListHash", reasons); + if (!Array.isArray(measurement.phases)) { + reasons.push("native.measurement.phases must be an array"); + } else { + const names = new Set(); + for (const [index, phaseValue] of measurement.phases.entries()) { + const path = `native.measurement.phases[${index}]`; + const phase = keys(phaseValue, path, ["name", "startFrame", "endFrame", "wallTimeNs"], reasons); + if (!phase) continue; + if (string(phase.name, `${path}.name`, reasons)) { + if (names.has(phase.name)) reasons.push(`${path}.name is duplicated`); + names.add(phase.name); + } + const start = phase.startFrame; + const end = phase.endFrame; + const startOk = uint(start, `${path}.startFrame`, reasons); + const endOk = uint(end, `${path}.endFrame`, reasons); + uint(phase.wallTimeNs, `${path}.wallTimeNs`, reasons); + if (startOk && endOk && end <= start) { + reasons.push(`${path}.endFrame must be greater than startFrame`); + } + } + } + } + + validateMetricMap(result.diagnosticMetrics, "native.diagnosticMetrics", new Set(["ns", "count"]), reasons); + validateMetricMap(result.exactMetrics, "native.exactMetrics", new Set(["bytes"]), reasons); + if (!Array.isArray(result.unsupportedMetrics)) { + reasons.push("native.unsupportedMetrics must be an array"); + } else { + const seen = new Set(); + for (const [index, metric] of result.unsupportedMetrics.entries()) { + if (string(metric, `native.unsupportedMetrics[${index}]`, reasons)) { + if (!isMetricId(metric)) reasons.push(`native.unsupportedMetrics[${index}] is not a catalog metric`); + if (seen.has(metric)) reasons.push(`native.unsupportedMetrics[${index}] is duplicated`); + seen.add(metric); + } + } + } + } else { + reasons.push("native.status must be ok or unsupported"); + } + + if (reasons.length > 0) return { success: false, reasons: [...new Set(reasons)] }; + return { success: true, data: value as unknown as NativeRunResult }; +} diff --git a/tools/perf/receipts/protocol.ts b/tools/perf/receipts/protocol.ts new file mode 100644 index 00000000..8fd27f9b --- /dev/null +++ b/tools/perf/receipts/protocol.ts @@ -0,0 +1,403 @@ +import type { FrameworkId } from "../core/types.ts"; + +export const GUEST_OUTPUT_PREFIX = "POCKETJS_PERF_GUEST "; +export const QEMU_OUTPUT_PREFIX = "POCKETJS_PERF_QEMU "; + +const FNV1A64 = /^fnv1a64:[a-f0-9]{16}$/; +const SHA256 = /^[a-f0-9]{64}$/; +const FRAMEWORKS = new Set(["solid", "vue-vapor", "octane", "core"]); +const QEMU_TARGETS = new Set(["arm", "aarch64"]); +const QEMU_METRIC_KEYS = [ + "guest_insn_dispatched", + "guest_instruction_bytes", + "guest_insn_size_2", + "guest_insn_size_4", + "guest_load_events", + "guest_store_events", +] as const; + +type PlainRecord = Record; + +export interface GuestPhaseRecordV1 { + readonly schemaVersion: 1; + readonly event: "phase"; + readonly scenarioId: string; + readonly phase: string; + readonly phaseId: number; + readonly iteration: number; + readonly allocCalls: number; + readonly allocatedBytes: number; + readonly currentBytes: number; + readonly peakBytes: number; + readonly quickjsLiveBytesAfterGc: number; + readonly drawListHash: string; +} + +export interface GuestCompleteRecordV1 { + readonly schemaVersion: 1; + readonly event: "complete"; + readonly scenarioId: string; + readonly suite: string; + readonly framework: FrameworkId; + readonly finalDrawListHash: string; + readonly finalStateHash: string; + readonly effectHash: string; + /** Present only in an observational correctness replay. */ + readonly framebufferTraceHash?: string; +} + +export type QemuTarget = "arm" | "aarch64"; + +export interface QemuCountersV1 { + readonly guest_insn_dispatched: number; + readonly guest_instruction_bytes: number; + readonly guest_insn_size_2: number; + readonly guest_insn_size_4: number; + readonly guest_load_events: number; + readonly guest_store_events: number; +} + +export interface QemuMeasurementRecordV1 { + readonly schema: "pocketjs.perf.qemu"; + readonly version: 1; + readonly event: "measurement"; + readonly plugin_api: 6; + readonly qemu_version: "11.0.3"; + readonly target: QemuTarget; + readonly vcpu: number; + readonly phase_id: number; + readonly iteration: number; + readonly metrics: QemuCountersV1; +} + +export interface QemuCompleteRecordV1 { + readonly schema: "pocketjs.perf.qemu"; + readonly version: 1; + readonly event: "complete"; + readonly plugin_api: 6; + readonly qemu_version: "11.0.3"; + readonly target: QemuTarget; + readonly measurements: number; +} + +export interface QemuErrorRecordV1 { + readonly schema: "pocketjs.perf.qemu"; + readonly version: 1; + readonly event: "error"; + readonly plugin_api: 6; + readonly qemu_version: "11.0.3"; + readonly target?: QemuTarget; + readonly code: string; + readonly measurements: number; +} + +export type GuestProtocolResult = + | { + readonly status: "valid"; + readonly reasons: readonly []; + readonly phases: readonly GuestPhaseRecordV1[]; + readonly complete: GuestCompleteRecordV1; + } + | { + readonly status: "invalid"; + readonly reasons: readonly string[]; + readonly phases: readonly GuestPhaseRecordV1[]; + readonly complete: GuestCompleteRecordV1 | null; + }; + +export interface GuestProtocolParseOptions { + /** Correctness replays require the trace; measurement replays forbid it. */ + readonly framebufferTraceHash?: "optional" | "required" | "forbidden"; +} + +export type QemuProtocolResult = + | { + readonly status: "valid"; + readonly reasons: readonly []; + readonly measurements: readonly QemuMeasurementRecordV1[]; + readonly terminal: QemuCompleteRecordV1; + } + | { + readonly status: "invalid"; + readonly reasons: readonly string[]; + readonly measurements: readonly QemuMeasurementRecordV1[]; + readonly terminal: QemuCompleteRecordV1 | QemuErrorRecordV1 | null; + }; + +interface PrefixedRecord { + readonly line: number; + readonly value: PlainRecord; +} + +function isPlainRecord(value: unknown): value is PlainRecord { + return typeof value === "object" && value !== null && !Array.isArray(value) && + [Object.prototype, null].includes(Object.getPrototypeOf(value)); +} + +function exactKeys( + record: PlainRecord, + required: readonly string[], + optional: readonly string[] = [], +): string | null { + const allowed = new Set([...required, ...optional]); + const unknown = Object.keys(record).filter((key) => !allowed.has(key)); + if (unknown.length > 0) return `unknown properties: ${unknown.join(", ")}`; + const missing = required.filter((key) => !Object.hasOwn(record, key)); + return missing.length > 0 ? `missing properties: ${missing.join(", ")}` : null; +} + +function nonEmptyString(value: unknown): value is string { + return typeof value === "string" && value.trim().length > 0; +} + +function nonNegativeSafeInteger(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + +function readPrefixed(output: string, prefix: string, label: string): { + records: PrefixedRecord[]; + reasons: string[]; +} { + const records: PrefixedRecord[] = []; + const reasons: string[] = []; + for (const [index, line] of output.split(/\r?\n/u).entries()) { + if (!line.startsWith(prefix)) continue; + const lineNumber = index + 1; + let parsed: unknown; + try { + parsed = JSON.parse(line.slice(prefix.length)); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + reasons.push(`${label} line ${lineNumber}: invalid JSON (${detail})`); + continue; + } + if (!isPlainRecord(parsed)) { + reasons.push(`${label} line ${lineNumber}: protocol value is not an object`); + continue; + } + records.push({ line: lineNumber, value: parsed }); + } + if (records.length === 0) reasons.push(`no ${label} protocol records`); + return { records, reasons }; +} + +function parseGuestPhase(record: PlainRecord): string | GuestPhaseRecordV1 { + const keys = exactKeys(record, [ + "schemaVersion", "event", "scenarioId", "phase", "phaseId", "iteration", + "allocCalls", "allocatedBytes", "currentBytes", "peakBytes", + "quickjsLiveBytesAfterGc", "drawListHash", + ]); + if (keys) return keys; + if (record.schemaVersion !== 1) return "schemaVersion must be 1"; + if (record.event !== "phase") return "event must be phase"; + if (!nonEmptyString(record.scenarioId)) return "scenarioId must be a non-empty string"; + if (!nonEmptyString(record.phase)) return "phase must be a non-empty string"; + for (const key of [ + "phaseId", "iteration", "allocCalls", "allocatedBytes", "currentBytes", + "peakBytes", "quickjsLiveBytesAfterGc", + ] as const) { + if (!nonNegativeSafeInteger(record[key])) return `${key} must be a non-negative safe integer`; + } + if (typeof record.drawListHash !== "string" || !FNV1A64.test(record.drawListHash)) { + return "drawListHash must be a lowercase FNV-1a-64 digest"; + } + return record as unknown as GuestPhaseRecordV1; +} + +function parseGuestComplete(record: PlainRecord): string | GuestCompleteRecordV1 { + const keys = exactKeys(record, [ + "schemaVersion", "event", "scenarioId", "suite", "framework", + "finalDrawListHash", "finalStateHash", "effectHash", + ], ["framebufferTraceHash"]); + if (keys) return keys; + if (record.schemaVersion !== 1) return "schemaVersion must be 1"; + if (record.event !== "complete") return "event must be complete"; + if (!nonEmptyString(record.scenarioId)) return "scenarioId must be a non-empty string"; + if (!nonEmptyString(record.suite)) return "suite must be a non-empty string"; + if (typeof record.framework !== "string" || !FRAMEWORKS.has(record.framework as FrameworkId)) { + return "framework is unknown"; + } + for (const key of ["finalDrawListHash", "finalStateHash", "effectHash"] as const) { + if (typeof record[key] !== "string" || !FNV1A64.test(record[key])) { + return `${key} must be a lowercase FNV-1a-64 digest`; + } + } + if (record.framebufferTraceHash !== undefined && + (typeof record.framebufferTraceHash !== "string" || !SHA256.test(record.framebufferTraceHash))) { + return "framebufferTraceHash must be a lowercase SHA-256 digest"; + } + return record as unknown as GuestCompleteRecordV1; +} + +export function parseGuestOutput( + output: string, + options: GuestProtocolParseOptions = {}, +): GuestProtocolResult { + const prefixed = readPrefixed(output, GUEST_OUTPUT_PREFIX, "guest"); + const reasons = [...prefixed.reasons]; + const phases: GuestPhaseRecordV1[] = []; + const completes: GuestCompleteRecordV1[] = []; + const terminalIndexes: number[] = []; + + prefixed.records.forEach(({ line, value }, index) => { + if (value.event === "phase") { + const parsed = parseGuestPhase(value); + if (typeof parsed === "string") reasons.push(`guest line ${line}: ${parsed}`); + else phases.push(parsed); + } else if (value.event === "complete") { + const parsed = parseGuestComplete(value); + if (typeof parsed === "string") reasons.push(`guest line ${line}: ${parsed}`); + else { + completes.push(parsed); + terminalIndexes.push(index); + } + } else { + reasons.push(`guest line ${line}: unknown protocol event ${JSON.stringify(value.event)}`); + } + }); + + if (completes.length !== 1) reasons.push("expected exactly one guest complete sentinel"); + if (completes.length === 1 && terminalIndexes[0] !== prefixed.records.length - 1) { + reasons.push("guest complete sentinel is not the final guest protocol record"); + } + if (completes.length === 1) { + const traceMode = options.framebufferTraceHash ?? "optional"; + const hasTrace = completes[0]!.framebufferTraceHash !== undefined; + if (traceMode === "required" && !hasTrace) { + reasons.push("guest complete has no required framebufferTraceHash"); + } else if (traceMode === "forbidden" && hasTrace) { + reasons.push("guest complete emitted correctness-only framebufferTraceHash"); + } + } + if (phases.length === 0) reasons.push("guest complete run has no phase records"); + + const keys = new Set(); + const ids = new Set(); + for (const phase of phases) { + const nameKey = `${phase.phase}\0${phase.iteration}`; + const idKey = `${phase.phaseId}\0${phase.iteration}`; + if (keys.has(nameKey)) reasons.push(`duplicate guest phase ${phase.phase} iteration ${phase.iteration}`); + if (ids.has(idKey)) reasons.push(`duplicate guest phaseId ${phase.phaseId} iteration ${phase.iteration}`); + keys.add(nameKey); + ids.add(idKey); + } + + const uniqueReasons = [...new Set(reasons)]; + if (uniqueReasons.length > 0 || completes.length !== 1) { + return { status: "invalid", reasons: uniqueReasons, phases, complete: completes[0] ?? null }; + } + return { status: "valid", reasons: [], phases, complete: completes[0]! }; +} + +function validateQemuEnvelope(record: PlainRecord): string | null { + if (record.schema !== "pocketjs.perf.qemu") return "schema must be pocketjs.perf.qemu"; + if (record.version !== 1) return "version must be 1"; + if (record.plugin_api !== 6) return "plugin_api must be 6"; + if (record.qemu_version !== "11.0.3") return "qemu_version must be 11.0.3"; + return null; +} + +function parseQemuMeasurement(record: PlainRecord): string | QemuMeasurementRecordV1 { + const keys = exactKeys(record, [ + "schema", "version", "event", "plugin_api", "qemu_version", "target", + "vcpu", "phase_id", "iteration", "metrics", + ]); + if (keys) return keys; + const envelope = validateQemuEnvelope(record); + if (envelope) return envelope; + if (record.event !== "measurement") return "event must be measurement"; + if (typeof record.target !== "string" || !QEMU_TARGETS.has(record.target as QemuTarget)) { + return "target must be arm or aarch64"; + } + for (const key of ["vcpu", "phase_id", "iteration"] as const) { + if (!nonNegativeSafeInteger(record[key])) return `${key} must be a non-negative safe integer`; + } + if (!isPlainRecord(record.metrics)) return "metrics must be an object"; + const metricKeys = exactKeys(record.metrics, QEMU_METRIC_KEYS); + if (metricKeys) return `metric set mismatch (${metricKeys})`; + for (const key of QEMU_METRIC_KEYS) { + if (!nonNegativeSafeInteger(record.metrics[key])) { + return `metric ${key} must be a non-negative safe integer`; + } + } + return record as unknown as QemuMeasurementRecordV1; +} + +function parseQemuTerminal(record: PlainRecord): string | QemuCompleteRecordV1 | QemuErrorRecordV1 { + const isError = record.event === "error"; + const required = [ + "schema", "version", "event", "plugin_api", "qemu_version", "measurements", + ...(isError ? ["code"] : ["target"]), + ]; + const keys = exactKeys(record, required, isError ? ["target"] : []); + if (keys) return keys; + const envelope = validateQemuEnvelope(record); + if (envelope) return envelope; + if (record.event !== "complete" && record.event !== "error") return "unknown terminal event"; + if (Object.hasOwn(record, "target") && + (typeof record.target !== "string" || !QEMU_TARGETS.has(record.target as QemuTarget))) { + return "target must be arm or aarch64"; + } + if (!nonNegativeSafeInteger(record.measurements)) { + return "measurements must be a non-negative safe integer"; + } + if (isError && !nonEmptyString(record.code)) return "error code must be a non-empty string"; + return record as unknown as QemuCompleteRecordV1 | QemuErrorRecordV1; +} + +export function parseQemuOutput(output: string): QemuProtocolResult { + const prefixed = readPrefixed(output, QEMU_OUTPUT_PREFIX, "QEMU"); + const reasons = [...prefixed.reasons]; + const measurements: QemuMeasurementRecordV1[] = []; + const terminals: (QemuCompleteRecordV1 | QemuErrorRecordV1)[] = []; + const terminalIndexes: number[] = []; + + prefixed.records.forEach(({ line, value }, index) => { + if (value.event === "measurement") { + const parsed = parseQemuMeasurement(value); + if (typeof parsed === "string") reasons.push(`QEMU line ${line}: ${parsed}`); + else measurements.push(parsed); + } else if (value.event === "complete" || value.event === "error") { + const parsed = parseQemuTerminal(value); + if (typeof parsed === "string") reasons.push(`QEMU line ${line}: ${parsed}`); + else { + terminals.push(parsed); + terminalIndexes.push(index); + } + } else { + reasons.push(`QEMU line ${line}: unknown protocol event ${JSON.stringify(value.event)}`); + } + }); + + if (terminals.length !== 1) reasons.push("expected exactly one QEMU complete/error sentinel"); + if (terminals.length === 1 && terminalIndexes[0] !== prefixed.records.length - 1) { + reasons.push("QEMU complete/error sentinel is not the final QEMU protocol record"); + } + const terminal = terminals[0] ?? null; + if (terminal && terminal.measurements !== measurements.length) { + reasons.push( + `QEMU terminal measurement count ${terminal.measurements} does not match ${measurements.length} records`, + ); + } + if (terminal?.event === "complete" && measurements.length === 0) { + reasons.push("QEMU complete run has no measurements"); + } + if (terminal?.event === "error") reasons.push(`QEMU plugin reported ${terminal.code}`); + + const keys = new Set(); + for (const measurement of measurements) { + const key = `${measurement.phase_id}\0${measurement.iteration}`; + if (keys.has(key)) { + reasons.push( + `duplicate QEMU phaseId ${measurement.phase_id} iteration ${measurement.iteration}`, + ); + } + keys.add(key); + } + + const uniqueReasons = [...new Set(reasons)]; + if (uniqueReasons.length > 0 || !terminal || terminal.event !== "complete") { + return { status: "invalid", reasons: uniqueReasons, measurements, terminal }; + } + return { status: "valid", reasons: [], measurements, terminal }; +} diff --git a/tools/perf/runner/input.ts b/tools/perf/runner/input.ts new file mode 100644 index 00000000..c69fe0e2 --- /dev/null +++ b/tools/perf/runner/input.ts @@ -0,0 +1,188 @@ +import type { InputTapeV1, InputTrackV1, JsonValue } from "../core/types.ts"; + +/** + * Hardware-neutral controls used by the benchmark tapes. The adapter owns + * the mapping to the legacy PSP-shaped guest frame ABI; tapes never contain + * PSP SDK masks or device-specific crank/button names. + */ +const GUEST_BUTTON_MASK: Readonly> = Object.freeze({ + primary: 0x2000, + secondary: 0x1000, + tertiary: 0x4000, + quaternary: 0x8000, + select: 0x0001, + start: 0x0008, + up: 0x0010, + right: 0x0020, + down: 0x0040, + left: 0x0080, + "shoulder-left": 0x0100, + "shoulder-right": 0x0200, +}); + +export interface RelativeAxisEvent { + readonly control: string; + readonly delta: number; +} + +export interface EffectEvent { + readonly effect: string; + readonly value: JsonValue; +} + +export interface ExpandedInputFrame { + readonly buttons: number; + /** PSP-compatible packed analog value at the final guest ABI boundary. */ + readonly analog: number; + /** Packed legacy touch contacts consumed by hosts/sim. */ + readonly touches: readonly number[] | undefined; + readonly relativeAxes: readonly RelativeAxisEvent[]; + readonly effects: readonly EffectEvent[]; +} + +interface ActiveTouch { + readonly id: number; + x: number; + y: number; +} + +function eventTable(track: T): Map { + const table = new Map(); + for (const sample of track.samples) { + const values = table.get(sample.frame); + if (values) values.push(sample); + else table.set(sample.frame, [sample]); + } + return table; +} + +function analogByte(value: number): number { + // Benchmark tapes use a target-neutral -1..1 range. The guest ABI mapping + // happens here, immediately before frame(), and nowhere in scenario data. + const clamped = Math.max(-1, Math.min(1, value)); + if (clamped === 0) return 128; + return clamped < 0 + ? Math.round(128 + clamped * 128) + : Math.round(128 + clamped * 127); +} + +function packTouch(id: number, x: number, y: number): number { + // hosts/sim currently consumes the legacy 9-bit logical-coordinate form. + // All v1 scenarios use the stock 480x272 viewport, so no information is + // lost at this adapter boundary. + return (((id & 0xff) << 18) | ((y & 0x1ff) << 9) | (x & 0x1ff)) >>> 0; +} + +/** Expand a strict sparse tape into the exact input delivered each frame. */ +export function expandInputTape(tape: InputTapeV1): ExpandedInputFrame[] { + const buttons = new Map(); + const analog = new Map(); + const touches = new Map(); + const tables = tape.tracks.map((track) => ({ track, samples: eventTable(track) })); + const out: ExpandedInputFrame[] = []; + + for (let frame = 0; frame < tape.frames; frame++) { + const relativeAxes: RelativeAxisEvent[] = []; + const effects: EffectEvent[] = []; + + for (const { track, samples } of tables) { + const at = samples.get(frame) ?? []; + switch (track.kind) { + case "button": + for (const sample of at as typeof track.samples) { + buttons.set(track.control, sample.pressed); + } + break; + case "analog": + for (const sample of at as typeof track.samples) { + analog.set(track.control, sample.value); + } + break; + case "touch": { + const id = touchId(track.control); + for (const sample of at as typeof track.samples) { + if (sample.phase === "end" || sample.phase === "cancel") { + touches.delete(track.control); + } else { + touches.set(track.control, { id, x: sample.x, y: sample.y }); + } + } + break; + } + case "relative-axis": + for (const sample of at as typeof track.samples) { + relativeAxes.push({ control: track.control, delta: sample.delta }); + } + break; + case "effect": + for (const sample of at as typeof track.samples) { + effects.push({ effect: track.effect, value: sample.value }); + } + break; + } + } + + let buttonMask = 0; + for (const [control, pressed] of buttons) { + if (!pressed) continue; + const mask = GUEST_BUTTON_MASK[control]; + if (mask === undefined) { + throw new Error(`native perf runner does not map button control ${JSON.stringify(control)}`); + } + buttonMask |= mask; + } + + const x = analogByte(analog.get("x") ?? 0); + const y = analogByte(analog.get("y") ?? 0); + const packedTouches = [...touches.values()] + .sort((a, b) => a.id - b.id) + .map((touch) => packTouch(touch.id, touch.x, touch.y)); + out.push({ + buttons: buttonMask, + analog: ((x << 8) | y) >>> 0, + touches: packedTouches.length > 0 ? packedTouches : undefined, + relativeAxes, + effects, + }); + } + return out; +} + +function touchId(control: string): number { + const match = /^contact-(\d+)$/.exec(control); + if (!match) { + throw new Error( + `native perf runner touch controls must be contact-N, got ${JSON.stringify(control)}`, + ); + } + const id = Number(match[1]); + if (!Number.isInteger(id) || id < 0 || id > 7) { + throw new Error(`native perf runner touch id must be 0..7, got ${id}`); + } + return id; +} + +export const NATIVE_INPUT_CAPABILITIES = Object.freeze([ + "input.buttons", + "input.analog", + "input.touch", +] as const); + +/** Report adapter gaps before execution; never substitute neutral/default input. */ +export function nativeInputUnsupportedReasons(tape: InputTapeV1): string[] { + const reasons: string[] = []; + for (const track of tape.tracks) { + if (track.kind === "button" && GUEST_BUTTON_MASK[track.control] === undefined) { + reasons.push(`native guest ABI has no button mapping for ${JSON.stringify(track.control)}`); + } else if (track.kind === "analog" && track.control !== "x" && track.control !== "y") { + reasons.push(`native guest ABI has no analog mapping for ${JSON.stringify(track.control)}`); + } else if (track.kind === "touch") { + try { + touchId(track.control); + } catch (error) { + reasons.push(error instanceof Error ? error.message : String(error)); + } + } + } + return reasons; +} diff --git a/tools/perf/runner/legacy-input.ts b/tools/perf/runner/legacy-input.ts new file mode 100644 index 00000000..3a3d2cb8 --- /dev/null +++ b/tools/perf/runner/legacy-input.ts @@ -0,0 +1,255 @@ +import { parseInputTapeV1 } from "../core/index.ts"; +import type { InputTapeV1, InputTrackV1 } from "../core/types.ts"; + +const RAW_BUTTONS = [ + [0x2000, "primary"], + [0x1000, "secondary"], + [0x4000, "tertiary"], + [0x8000, "quaternary"], + [0x0001, "select"], + [0x0008, "start"], + [0x0020, "right"], + [0x0080, "left"], + [0x0010, "up"], + [0x0040, "down"], + [0x0200, "shoulder-right"], + [0x0100, "shoulder-left"], +] as const; + +const VAPOR_BUTTONS = [ + "primary", + "secondary", + "select", + "start", + "right", + "left", + "up", + "down", + "shoulder-right", + "shoulder-left", +] as const; + +interface LegacyTouchPoint { + readonly id: number; + readonly x: number; + readonly y: number; +} + +interface GoldenSpecLike { + readonly frames: number; + readonly input?: (frame: number) => number; + readonly touch?: (frame: number) => readonly LegacyTouchPoint[]; +} + +interface DevtoolsTapeLike { + readonly frames: number; + readonly masks: readonly (readonly [value: number, count: number])[]; + readonly analog?: readonly (readonly [value: number, count: number])[]; + readonly touch?: readonly (readonly [frame: number, contacts: readonly number[]])[]; + readonly startFrame?: number; +} + +/** Freeze a GoldenSpec closure into serializable benchmark input. */ +export function goldenSpecToInputTape( + id: string, + spec: GoldenSpecLike, +): InputTapeV1 { + const masks = Array.from({ length: spec.frames }, (_, frame) => spec.input?.(frame) ?? 0); + const touches = Array.from( + { length: spec.frames }, + (_, frame) => spec.touch?.(frame) ?? [], + ); + return buildTape(id, masks, undefined, touches); +} + +/** Convert the always-from-boot subset of a DevTools tape. */ +export function devtoolsTapeToInputTape( + id: string, + tape: DevtoolsTapeLike, +): InputTapeV1 { + if ((tape.startFrame ?? 0) !== 0) { + throw new Error("wrapped DevTools tapes cannot be benchmark inputs: startFrame must be 0"); + } + const masks = expandPairs(tape.masks, 0, tape.frames); + const analog = tape.analog ? expandPairs(tape.analog, 0x8080, tape.frames) : undefined; + const contacts: LegacyTouchPoint[][] = Array.from({ length: tape.frames }, () => []); + for (const [frame, packed] of tape.touch ?? []) { + if (!Number.isInteger(frame) || frame < 0 || frame >= tape.frames) { + throw new Error(`DevTools touch frame ${frame} is outside the tape`); + } + contacts[frame] = packed.map(unpackTouch); + } + return buildTape(id, masks, analog, contacts); +} + +/** Convert tools/bench-ppsspp.ts's threshold-state input string. */ +export function ppssppScriptToInputTape( + id: string, + frames: number, + script: string, +): InputTapeV1 { + const changes = script + .split(",") + .filter(Boolean) + .map((entry) => { + const [frame, value] = entry.split(":"); + return [Number(frame), Number(value)] as const; + }); + let current = 0; + let at = 0; + const masks = new Array(frames).fill(0); + for (const [frame, value] of changes) { + if (!Number.isInteger(frame) || frame < at || frame >= frames || !Number.isInteger(value)) { + throw new Error(`invalid PPSSPP input change ${frame}:${value}`); + } + masks.fill(current, at, frame); + current = value; + at = frame; + } + masks.fill(current, at); + return buildTape(id, masks); +} + +/** Convert Vapor's ordered Button IDs into one-frame logical press pulses. */ +export function vaporTodoToInputTape( + id: string, + buttons: readonly number[], + options: { readonly bootFrames?: number; readonly spacing?: number } = {}, +): InputTapeV1 { + const bootFrames = options.bootFrames ?? 0; + const spacing = options.spacing ?? 2; + if (!Number.isInteger(bootFrames) || bootFrames < 0 || !Number.isInteger(spacing) || spacing < 2) { + throw new Error("Vapor tape bootFrames must be >= 0 and spacing must be >= 2"); + } + const frames = Math.max(1, bootFrames + buttons.length * spacing); + const tracks = new Map(); + buttons.forEach((button, index) => { + const control = VAPOR_BUTTONS[button]; + if (!control) throw new Error(`unknown Vapor Button ID ${button}`); + const frame = bootFrames + index * spacing; + const samples = tracks.get(control) ?? []; + samples.push({ frame, pressed: true }, { frame: frame + 1, pressed: false }); + tracks.set(control, samples); + }); + return parseInputTapeV1({ + schemaVersion: 1, + kind: "pocketjs.perf.input-tape", + id, + frames, + tracks: [...tracks].map(([control, samples]) => ({ kind: "button", control, samples })), + }); +} + +function buildTape( + id: string, + masks: readonly number[], + analog?: readonly number[], + touches?: readonly (readonly LegacyTouchPoint[])[], +): InputTapeV1 { + const frames = masks.length; + if (frames === 0) throw new Error("benchmark tapes must contain at least one frame"); + if (analog && analog.length !== frames) throw new Error("analog frame count differs from buttons"); + if (touches && touches.length !== frames) throw new Error("touch frame count differs from buttons"); + const tracks: InputTrackV1[] = [...buttonTracks(masks)]; + if (analog) tracks.push(...analogTracks(analog)); + if (touches) tracks.push(...touchTracks(touches)); + return parseInputTapeV1({ + schemaVersion: 1, + kind: "pocketjs.perf.input-tape", + id, + frames, + tracks, + }); +} + +function buttonTracks(masks: readonly number[]): InputTrackV1[] { + return RAW_BUTTONS.flatMap(([mask, control]) => { + const samples: { frame: number; pressed: boolean }[] = []; + let previous = false; + for (let frame = 0; frame < masks.length; frame++) { + const pressed = (masks[frame] & mask) !== 0; + if (pressed === previous) continue; + samples.push({ frame, pressed }); + previous = pressed; + } + return samples.length > 0 ? [{ kind: "button", control, samples }] : []; + }); +} + +function analogTracks(values: readonly number[]): InputTrackV1[] { + const tracks: InputTrackV1[] = []; + for (const [control, shift] of [["x", 8], ["y", 0]] as const) { + const samples: { frame: number; value: number }[] = []; + let previous = 128; + for (let frame = 0; frame < values.length; frame++) { + const raw = (values[frame] >>> shift) & 0xff; + if (raw === previous) continue; + const value = raw < 128 ? (raw - 128) / 128 : (raw - 128) / 127; + samples.push({ frame, value }); + previous = raw; + } + if (samples.length > 0) tracks.push({ kind: "analog", control, samples }); + } + return tracks; +} + +function touchTracks(frames: readonly (readonly LegacyTouchPoint[])[]): InputTrackV1[] { + const events = new Map(); + let previous = new Map(); + for (let frame = 0; frame < frames.length; frame++) { + const current = new Map(frames[frame].map((point) => [point.id, point])); + for (const [id, point] of current) { + const before = previous.get(id); + if (!before) appendTouch(events, id, { frame, phase: "start", x: point.x, y: point.y }); + else if (before.x !== point.x || before.y !== point.y) { + appendTouch(events, id, { frame, phase: "move", x: point.x, y: point.y }); + } + } + for (const [id, point] of previous) { + if (!current.has(id)) appendTouch(events, id, { frame, phase: "end", x: point.x, y: point.y }); + } + previous = current; + } + return [...events] + .sort(([a], [b]) => a - b) + .map(([id, samples]) => ({ kind: "touch", control: `contact-${id}`, samples })); +} + +function appendTouch( + events: Map, + id: number, + event: { frame: number; phase: "start" | "move" | "end"; x: number; y: number }, +): void { + const list = events.get(id); + if (list) list.push(event); + else events.set(id, [event]); +} + +function expandPairs( + pairs: readonly (readonly [number, number])[], + fill: number, + frames: number, +): number[] { + const out = new Array(frames).fill(fill); + let at = 0; + for (const [value, count] of pairs) { + if (!Number.isInteger(value) || !Number.isInteger(count) || count <= 0 || at + count > frames) { + throw new Error(`invalid RLE pair [${value},${count}]`); + } + out.fill(value, at, at + count); + at += count; + } + if (at !== frames) throw new Error(`RLE expands to ${at} frames, expected ${frames}`); + return out; +} + +function unpackTouch(value: number): LegacyTouchPoint { + const wide = (value & 0x80000000) !== 0; + const bits = wide ? 10 : 9; + const mask = (1 << bits) - 1; + return { + id: (value >>> (bits * 2)) & 0xff, + x: value & mask, + y: (value >>> bits) & mask, + }; +} diff --git a/tools/perf/runner/native-cli.ts b/tools/perf/runner/native-cli.ts new file mode 100644 index 00000000..7af25730 --- /dev/null +++ b/tools/perf/runner/native-cli.ts @@ -0,0 +1,51 @@ +import { resolve } from "node:path"; +import { loadScenario, runNativeQuick } from "./native.ts"; +import { runNativeSuite } from "./suite.ts"; +import { isDamageScenario, runNativeDamageScenario } from "../executors/damage.ts"; +import { runNativeVaporScenario } from "../executors/vapor.ts"; +import { NATIVE_RUN_OUTPUT_PREFIX } from "../receipts/native-protocol.ts"; + +function value(flag: string): string | undefined { + const exact = process.argv.indexOf(flag); + if (exact >= 0) return process.argv[exact + 1]; + const prefix = `${flag}=`; + return process.argv.find((arg) => arg.startsWith(prefix))?.slice(prefix.length); +} + +const scenarioPath = process.argv[2]?.startsWith("--") ? undefined : process.argv[2]; +const suite = value("--suite"); +if ((!scenarioPath && !suite) || (scenarioPath && suite)) { + console.error( + "usage:\n" + + " bun tools/perf/runner/native-cli.ts [--source-root PATH] " + + "[--harness-root PATH] [--out-dir PATH]\n" + + " bun tools/perf/runner/native-cli.ts --suite quick [--max-estimated-seconds N] " + + "[--source-root PATH] [--out-dir PATH]", + ); + process.exit(2); +} + +const sourceRoot = resolve(value("--source-root") ?? new URL("../../..", import.meta.url).pathname); +const harnessRoot = resolve(value("--harness-root") ?? new URL("../../..", import.meta.url).pathname); +const outDir = value("--out-dir") ? resolve(value("--out-dir")!) : undefined; +if (suite) { + const result = await runNativeSuite(suite, { + sourceRoot, + harnessRoot, + outDir, + maxEstimatedSeconds: value("--max-estimated-seconds") + ? Number(value("--max-estimated-seconds")) + : undefined, + }); + console.log(JSON.stringify(result, null, 2)); + if (result.results.some((item) => item.status === "unsupported")) process.exitCode = 2; +} else { + const scenario = loadScenario(resolve(scenarioPath!)); + const result = isDamageScenario(scenario) + ? await runNativeDamageScenario(scenario, { sourceRoot, harnessRoot, outDir }) + : scenario.subject.family === "vapor" + ? await runNativeVaporScenario(scenario, { sourceRoot, harnessRoot, outDir }) + : await runNativeQuick(scenario, { sourceRoot, outDir }); + console.log(`${NATIVE_RUN_OUTPUT_PREFIX}${JSON.stringify(result)}`); + if (result.status === "unsupported") process.exitCode = 2; +} diff --git a/tools/perf/runner/native-world.ts b/tools/perf/runner/native-world.ts new file mode 100644 index 00000000..6f751107 --- /dev/null +++ b/tools/perf/runner/native-world.ts @@ -0,0 +1,104 @@ +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { createWasmUi } from "../../../hosts/web/wasm-ops.js"; +import { createTouchHitFacts } from "../../../framework/src/touch.ts"; +import { buildRenderConfig, type ScenarioV1 } from "../core/index.ts"; +import type { NativeSimWorld } from "./native.ts"; + +interface EffectEvent { + readonly t: "command" | "delivery"; + readonly frame: number; + readonly id: number; + readonly kind: string; +} + +/** + * Boot a source checkout with the current, versioned benchmark host. + * + * The host lives with the harness instead of the source checkout on purpose: + * `perf local` must be able to measure a revision that predates benchmark-only + * observations such as DrawList hashing. The source checkout still supplies + * every measured product artifact: WASM core, app bundle and PAK. + */ +export async function bootNativePerfWorld( + sourceRoot: string, + scenario: ScenarioV1, +): Promise { + const app = scenario.subject.entry; + const wasmPath = join(sourceRoot, "hosts/web/pocketjs.wasm"); + const bundlePath = join(sourceRoot, "dist", `${app}.js`); + const pakPath = join(sourceRoot, "dist", `${app}.pak`); + const viewport = buildRenderConfig(scenario.params); + const wasmBytes = await Bun.file(wasmPath).arrayBuffer(); + const wasm = await createWasmUi(wasmBytes, viewport); + if (!wasm.drawHash) { + throw new Error(`${scenario.id}: source WASM has no ui_draw_hash export`); + } + + const effects: EffectEvent[] = []; + const inbox: string[] = []; + const outbox: string[] = []; + const global = globalThis as Record; + global.ui = wasm.ops; + global.__pak = existsSync(pakPath) ? await Bun.file(pakPath).arrayBuffer() : undefined; + global.frame = undefined; + global.audio = undefined; + global.__pocketApp = app; + global.__simHz = 60; + global.__pocketEffectTrace = (event: EffectEvent) => effects.push(event); + global.__pocketEffectDriver = undefined; + global.__pocketDevtoolsTransport = { + send: (line: string) => outbox.push(line), + recv: () => (inbox.length > 0 ? inbox.shift() : null), + }; + + const source = await Bun.file(bundlePath).text(); + (0, eval)(source); + const appFrame = global.frame as + | ((buttons: number, analog?: number, touches?: readonly number[], hits?: readonly number[]) => void) + | undefined; + if (typeof appFrame !== "function") { + throw new Error(`${scenario.id}: bundle did not install globalThis.frame`); + } + // `Guest::eval` drains QuickJS promise jobs before the QEMU adapter starts + // its first frame. Give Bun's framework runtime the same microtask boundary. + await Promise.resolve(); + + const hitTestBounds = (wasm.ops as { hitTestBounds?: (x: number, y: number) => number }) + .hitTestBounds; + const hitFacts = hitTestBounds ? createTouchHitFacts(hitTestBounds) : undefined; + const frame = (buttons: number, analog?: number, touches?: readonly number[]): void => { + appFrame(buttons, analog, touches, hitFacts?.(touches)); + }; + const renderScale = viewport.renderScale; + + return { + frame, + // Vue Vapor batches ref effects in a promise job; Solid and Octane finish + // synchronously. The boundary is nevertheless an executor-wide host + // contract, not a framework special case. It mirrors Guest::frame_*() + // draining QuickJS jobs before Core tick/render. + drainJobs: async () => { + await Promise.resolve(); + }, + tick: wasm.tick, + render: () => wasm.renderScaled(renderScale), + drawHash: () => { + const unsigned = BigInt.asUintN(64, wasm.drawHash!()); + return `fnv1a64:${unsigned.toString(16).padStart(16, "0")}`; + }, + ticksPerFrame: 1, + effects, + getTree: () => { + outbox.length = 0; + inbox.push(JSON.stringify({ t: "getTree" })); + frame(0); + wasm.tick(); + for (const line of outbox) { + const message = JSON.parse(line) as { t: string; root?: unknown }; + if (message.t === "tree") return message.root; + } + return null; + }, + }; +} diff --git a/tools/perf/runner/native.ts b/tools/perf/runner/native.ts new file mode 100644 index 00000000..023d64a2 --- /dev/null +++ b/tools/perf/runner/native.ts @@ -0,0 +1,389 @@ +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { gateMetricIds, parseScenarioV1 } from "../core/index.ts"; +import type { CorrectnessCapture, ScenarioV1 } from "../core/types.ts"; +import { + expandInputTape, + nativeInputUnsupportedReasons, + NATIVE_INPUT_CAPABILITIES, +} from "./input.ts"; +import { bootNativePerfWorld } from "./native-world.ts"; + +export interface NativeSimWorld { + frame(buttons: number, analog?: number, touches?: readonly number[]): void; + /** Drain framework promise jobs scheduled by `frame`, matching QuickJS hosts. */ + drainJobs(): Promise; + tick(): void; + render(): Uint8Array; + drawHash(): string; + readonly ticksPerFrame: number; + readonly effects: readonly unknown[]; + getTree(): unknown; +} + +export interface NativeBootAdapter { + boot(sourceRoot: string, scenario: ScenarioV1): Promise; +} + +export interface NativeRunOptions { + /** Revision/worktree to execute. It need not contain tools/perf itself. */ + readonly sourceRoot: string; + /** Optional directory for the JSON runner result. */ + readonly outDir?: string; + /** Tests can replace the expensive real WASM boot while exercising the runner. */ + readonly bootAdapter?: NativeBootAdapter; +} + +export interface NativeUnsupportedResult { + readonly schemaVersion: 1; + readonly kind: "pocketjs.perf.native-result"; + readonly status: "unsupported"; + readonly scenarioId: string; + readonly executor: "native"; + readonly reasons: readonly string[]; +} + +export interface NativePhaseTiming { + readonly name: string; + readonly startFrame: number; + readonly endFrame: number; + /** Diagnostic host time. It is never presented as target/device time. */ + readonly wallTimeNs: number; +} + +export interface NativeOkResult { + readonly schemaVersion: 1; + readonly kind: "pocketjs.perf.native-result"; + readonly status: "ok"; + readonly scenarioId: string; + readonly executor: "native"; + readonly sourceRoot: string; + readonly correctness: { + readonly framebufferTraceHash: string; + readonly finalFramebufferHash: string; + readonly drawListHash: string; + readonly stateHash: string; + readonly effectHash: string; + readonly checkpoints: Readonly>>>; + }; + readonly measurement: { + readonly bootWallTimeNs: number; + readonly phases: readonly NativePhaseTiming[]; + readonly finalFramebufferHash: string; + readonly finalDrawListHash: string; + }; + readonly diagnosticMetrics: Readonly>; + readonly exactMetrics: Readonly>; + /** Metrics the native sim cannot truthfully observe. */ + readonly unsupportedMetrics: readonly string[]; +} + +export type NativeRunResult = NativeOkResult | NativeUnsupportedResult; + +const NATIVE_CAPABILITIES = new Set([ + "guest.frame", + "core.ui", + "renderer.framebuffer", + "assets.pak", + "correctness.framebuffer", + "correctness.draw-list", + "correctness.effects", + "correctness.state-final", + ...NATIVE_INPUT_CAPABILITIES, +]); + +const OBSERVABLE_CAPTURES = new Set([ + "framebuffer", + "drawList", + "effects", +]); + +/** Load and strictly validate one scenario manifest. */ +export function loadScenario(path: string): ScenarioV1 { + return parseScenarioV1(JSON.parse(readFileSync(path, "utf8"))); +} + +/** + * Execute the full PocketJS framework -> HostOps -> WASM core -> software + * raster path twice: an observed correctness replay, then a minimally + * observed measurement replay. The two final frame hashes must agree. + */ +export async function runNativeQuick( + scenario: ScenarioV1, + options: NativeRunOptions, +): Promise { + const sourceRoot = resolve(options.sourceRoot); + const reasons = unsupportedReasons(scenario); + if (reasons.length > 0) { + return writeResult( + { + schemaVersion: 1, + kind: "pocketjs.perf.native-result", + status: "unsupported", + scenarioId: scenario.id, + executor: "native", + reasons, + }, + options.outDir, + ); + } + + const adapter = options.bootAdapter ?? DEFAULT_BOOT_ADAPTER; + const inputs = expandInputTape(scenario.tape); + const correctness = await correctnessReplay(adapter, sourceRoot, scenario, inputs); + const measurement = await measurementReplay(adapter, sourceRoot, scenario, inputs); + if (correctness.finalFramebufferHash !== measurement.finalFramebufferHash) { + throw new Error( + `${scenario.id}: correctness/measurement replay diverged: ` + + `${correctness.finalFramebufferHash} != ${measurement.finalFramebufferHash}`, + ); + } + if (correctness.drawListHash !== measurement.finalDrawListHash) { + throw new Error( + `${scenario.id}: correctness/measurement DrawList replay diverged: ` + + `${correctness.drawListHash} != ${measurement.finalDrawListHash}`, + ); + } + + const diagnosticMetrics: Record = { + "native.boot_wall_time_ns": { value: measurement.bootWallTimeNs, unit: "ns" }, + "native.measured_frames": { + value: scenario.phases + .filter((phase) => phase.collect) + .reduce((sum, phase) => sum + phase.endFrame - phase.startFrame, 0), + unit: "count", + }, + }; + for (const phase of measurement.phases) { + diagnosticMetrics[`native.phase.${phase.name}.wall_time_ns`] = { + value: phase.wallTimeNs, + unit: "ns", + }; + } + diagnosticMetrics["native.wall_time_ns"] = { + value: measurement.phases.reduce((sum, phase) => sum + phase.wallTimeNs, 0), + unit: "ns", + }; + const exactMetrics: Record = {}; + const bundlePath = join(sourceRoot, "dist", `${scenario.subject.entry}.js`); + const pakPath = join(sourceRoot, "dist", `${scenario.subject.entry}.pak`); + if (existsSync(bundlePath)) { + exactMetrics["artifact.bundle_bytes"] = { value: statSync(bundlePath).size, unit: "bytes" }; + } + if (existsSync(pakPath)) { + exactMetrics["artifact.pak_bytes"] = { value: statSync(pakPath).size, unit: "bytes" }; + } + const requestedGateMetrics = gateMetricIds(scenario.params); + + return writeResult( + { + schemaVersion: 1, + kind: "pocketjs.perf.native-result", + status: "ok", + scenarioId: scenario.id, + executor: "native", + sourceRoot, + correctness, + measurement, + diagnosticMetrics, + exactMetrics, + unsupportedMetrics: [ + ...requestedGateMetrics.filter((metric) => !Object.hasOwn(exactMetrics, metric)), + ], + }, + options.outDir, + ); +} + +function unsupportedReasons(scenario: ScenarioV1): string[] { + const reasons: string[] = []; + if (scenario.subject.family !== "guest-app") { + reasons.push(`native runner has no adapter for subject family ${JSON.stringify(scenario.subject.family)}`); + } + for (const requirement of scenario.executorRequirements) { + if (!NATIVE_CAPABILITIES.has(requirement)) reasons.push(`missing executor capability ${requirement}`); + } + for (const track of scenario.tape.tracks) { + if (track.kind === "relative-axis") { + reasons.push("hosts/sim guest frame has no RelativeAxis delivery adapter"); + } else if (track.kind === "effect") { + reasons.push("native runner has no generic recorded-effect delivery adapter"); + } + } + reasons.push(...nativeInputUnsupportedReasons(scenario.tape)); + for (const checkpoint of scenario.checkpoints) { + for (const capture of checkpoint.capture) { + if (capture === "state" && checkpoint.frame === scenario.frames - 1) continue; + if (!OBSERVABLE_CAPTURES.has(capture)) { + reasons.push(`cannot capture ${capture} at frame ${checkpoint.frame}`); + } + } + } + return [...new Set(reasons)]; +} + +async function correctnessReplay( + adapter: NativeBootAdapter, + sourceRoot: string, + scenario: ScenarioV1, + inputs: ReturnType, +): Promise { + const world = await adapter.boot(sourceRoot, scenario); + const trace = createHash("sha256"); + const checkpoints: Record> = {}; + let finalFramebufferHash = ""; + let drawListHash = ""; + + for (let frame = 0; frame < scenario.frames; frame++) { + const input = inputs[frame]; + world.frame(input.buttons, input.analog, input.touches); + await world.drainJobs(); + for (let tick = 0; tick < world.ticksPerFrame; tick++) world.tick(); + const framebuffer = world.render(); + drawListHash = world.drawHash(); + const frameHash = sha256(framebuffer); + trace.update(frameHash); + finalFramebufferHash = frameHash; + + for (const checkpoint of scenario.checkpoints) { + if (checkpoint.frame !== frame) continue; + const captured: Record = {}; + for (const capture of checkpoint.capture) { + if (capture === "framebuffer") captured.framebuffer = frameHash; + else if (capture === "drawList") captured.drawList = drawListHash; + else if (capture === "effects") captured.effects = hashJson(world.effects); + } + checkpoints[String(frame)] = captured; + } + } + + // getTree is intentionally correctness-only. The DevTools probe advances + // one extra frame, so it must never run inside a measured phase. + const effectHash = hashJson(world.effects); + const stateHash = hashJson(world.getTree()); + const finalCheckpoint = checkpoints[String(scenario.frames - 1)]; + if (finalCheckpoint && scenario.checkpoints + .find((checkpoint) => checkpoint.frame === scenario.frames - 1) + ?.capture.includes("state")) { + finalCheckpoint.state = stateHash; + } + return { + framebufferTraceHash: trace.digest("hex"), + finalFramebufferHash, + drawListHash, + stateHash, + effectHash, + checkpoints, + }; +} + +async function measurementReplay( + adapter: NativeBootAdapter, + sourceRoot: string, + scenario: ScenarioV1, + inputs: ReturnType, +): Promise { + const bootStarted = process.hrtime.bigint(); + const world = await adapter.boot(sourceRoot, scenario); + const bootWallTimeNs = safeNs(process.hrtime.bigint() - bootStarted); + const started = new Map(); + const timings: NativePhaseTiming[] = []; + let finalFramebuffer: Uint8Array | null = null; + + for (let frame = 0; frame < scenario.frames; frame++) { + for (const phase of scenario.phases) { + if (phase.collect && phase.startFrame === frame) started.set(phase.name, process.hrtime.bigint()); + } + const input = inputs[frame]; + world.frame(input.buttons, input.analog, input.touches); + await world.drainJobs(); + for (let tick = 0; tick < world.ticksPerFrame; tick++) world.tick(); + finalFramebuffer = world.render(); + for (const phase of scenario.phases) { + if (!phase.collect || phase.endFrame !== frame + 1) continue; + const phaseStarted = started.get(phase.name); + if (phaseStarted === undefined) throw new Error(`${scenario.id}: phase ${phase.name} never started`); + timings.push({ + name: phase.name, + startFrame: phase.startFrame, + endFrame: phase.endFrame, + wallTimeNs: safeNs(process.hrtime.bigint() - phaseStarted), + }); + } + } + if (!finalFramebuffer) throw new Error(`${scenario.id}: no framebuffer was rendered`); + // Correctness fingerprints stay outside every measured interval. + const finalDrawListHash = world.drawHash(); + return { + bootWallTimeNs, + phases: timings, + finalFramebufferHash: sha256(finalFramebuffer), + finalDrawListHash, + }; +} + +const DEFAULT_BOOT_ADAPTER: NativeBootAdapter = { + async boot(sourceRoot, scenario) { + assertPrebuilt(sourceRoot, scenario); + return await bootNativePerfWorld(sourceRoot, scenario); + }, +}; + +function assertPrebuilt(sourceRoot: string, scenario: ScenarioV1): void { + const missing = [ + join(sourceRoot, "hosts/web/pocketjs.wasm"), + join(sourceRoot, "dist", `${scenario.subject.entry}.js`), + ].filter((path) => !existsSync(path)); + if (scenario.executorRequirements.includes("assets.pak")) { + const pak = join(sourceRoot, "dist", `${scenario.subject.entry}.pak`); + if (!existsSync(pak)) missing.push(pak); + } + if (missing.length > 0) { + throw new Error( + `${scenario.id}: native runner only consumes prebuilt artifacts; missing ${missing.join(", ")}`, + ); + } +} + +function safeNs(value: bigint): number { + const number = Number(value); + if (!Number.isSafeInteger(number)) throw new Error(`native timing exceeds Number safe range: ${value}`); + return number; +} + +function sha256(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +function hashJson(value: unknown): string { + return createHash("sha256").update(canonicalJson(value)).digest("hex"); +} + +function canonicalJson(value: unknown): string { + if (value === null || typeof value === "boolean" || typeof value === "string") { + return JSON.stringify(value); + } + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new Error("cannot hash non-finite JSON number"); + return JSON.stringify(value); + } + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object") { + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`) + .join(",")}}`; + } + if (value === undefined) return "null"; + throw new Error(`cannot hash ${typeof value} as JSON`); +} + +function writeResult(result: T, outDir?: string): T { + if (!outDir) return result; + mkdirSync(outDir, { recursive: true }); + const safeId = result.scenarioId.replace(/[^a-zA-Z0-9._-]+/g, "-"); + writeFileSync(join(outDir, `${safeId}.native.json`), `${JSON.stringify(result, null, 2)}\n`); + return result; +} diff --git a/tools/perf/runner/suite.ts b/tools/perf/runner/suite.ts new file mode 100644 index 00000000..474111d9 --- /dev/null +++ b/tools/perf/runner/suite.ts @@ -0,0 +1,181 @@ +import { readdirSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { parseScenarioV1 } from "../core/index.ts"; +import type { FrameworkId, ScenarioV1 } from "../core/types.ts"; +import { isDamageScenario, runNativeDamageScenario } from "../executors/damage.ts"; +import { runNativeVaporScenario } from "../executors/vapor.ts"; +import { + loadScenario, + runNativeQuick, + type NativeRunOptions, + type NativeRunResult, +} from "./native.ts"; + +export interface NativeSuiteResult { + readonly schemaVersion: 1; + readonly kind: "pocketjs.perf.native-suite-result"; + readonly suite: string; + readonly estimatedSeconds: number; + readonly results: readonly NativeRunResult[]; +} + +export interface NativeSuiteAdapters { + readonly damage: ( + scenario: ScenarioV1, + options: { readonly sourceRoot: string; readonly harnessRoot: string; readonly outDir?: string }, + ) => Promise; + readonly vapor: ( + scenario: ScenarioV1, + options: { readonly sourceRoot: string; readonly harnessRoot: string; readonly outDir?: string }, + ) => Promise; +} + +const DEFAULT_SUITE_ADAPTERS: NativeSuiteAdapters = { + damage: runNativeDamageScenario, + vapor: runNativeVaporScenario, +}; + +export function loadScenarioSuite( + suite: string, + scenarioDir = new URL("../scenarios", import.meta.url).pathname, +): ScenarioV1[] { + return readdirSync(scenarioDir) + .filter((file) => file.endsWith(".json")) + .sort() + .map((file) => loadScenario(join(scenarioDir, file))) + .filter((scenario) => scenario.suite === suite); +} + +export function estimatedSuiteSeconds(scenarios: readonly ScenarioV1[]): number { + return scenarios.reduce((total, scenario) => { + const estimate = scenario.params.estimatedSeconds; + if (typeof estimate !== "number" || !Number.isFinite(estimate) || estimate < 0) { + throw new Error(`${scenario.id}: params.estimatedSeconds must be a non-negative number`); + } + return total + estimate; + }, 0); +} + +/** Expand only manifests that explicitly request a framework matrix. */ +export function expandScenarioFrameworks(scenario: ScenarioV1): ScenarioV1[] { + const configured = scenario.params.frameworks; + if (configured === undefined) return [scenario]; + if (scenario.subject.family !== "guest-app" || !Array.isArray(configured) || configured.length === 0) { + throw new Error(`${scenario.id}: params.frameworks requires a non-empty guest-app framework list`); + } + const allowed = new Set(["solid", "vue-vapor", "octane"]); + const artifactSuffix: Readonly, string>> = { + solid: "", + "vue-vapor": ".vue-vapor", + octane: ".octane", + }; + const configuredSubjects = scenario.params.frameworkSubjects; + if (configuredSubjects !== undefined && + (typeof configuredSubjects !== "object" || configuredSubjects === null || + Array.isArray(configuredSubjects))) { + throw new Error(`${scenario.id}: params.frameworkSubjects must be an object`); + } + const subjectOverrides = configuredSubjects as Record | undefined; + for (const framework of Object.keys(subjectOverrides ?? {})) { + if (!allowed.has(framework as FrameworkId) || !configured.includes(framework)) { + throw new Error(`${scenario.id}: frameworkSubjects has unconfigured framework ${framework}`); + } + } + const seen = new Set(); + return configured.map((framework) => { + if (typeof framework !== "string" || !allowed.has(framework as FrameworkId)) { + throw new Error(`${scenario.id}: unknown params.frameworks entry ${JSON.stringify(framework)}`); + } + if (seen.has(framework)) throw new Error(`${scenario.id}: duplicate framework ${framework}`); + seen.add(framework); + const override = subjectOverrides?.[framework]; + if (override !== undefined && + (typeof override !== "object" || override === null || Array.isArray(override))) { + throw new Error(`${scenario.id}: frameworkSubjects.${framework} must be an object`); + } + const subject = override as Record | undefined; + const unknownSubjectKeys = Object.keys(subject ?? {}).filter((key) => key !== "id" && key !== "entry"); + if (unknownSubjectKeys.length > 0) { + throw new Error( + `${scenario.id}: frameworkSubjects.${framework} has unknown fields ${unknownSubjectKeys.join(", ")}`, + ); + } + const id = subject?.id ?? scenario.subject.id; + const entry = subject?.entry ?? + `${scenario.subject.entry}${artifactSuffix[framework as Exclude]}`; + if (typeof id !== "string" || id.length === 0 || typeof entry !== "string" || entry.length === 0) { + throw new Error(`${scenario.id}: frameworkSubjects.${framework} requires non-empty id and entry`); + } + return parseScenarioV1({ + ...scenario, + subject: { + ...scenario.subject, + framework, + id, + entry, + }, + }); + }); +} + +export function expandSuiteFrameworks(scenarios: readonly ScenarioV1[]): ScenarioV1[] { + return scenarios.flatMap(expandScenarioFrameworks); +} + +/** Run serially: every sim boot temporarily owns process-wide guest globals. */ +export async function runNativeSuite( + suite: string, + options: NativeRunOptions & { + readonly scenarioDir?: string; + readonly maxEstimatedSeconds?: number; + readonly harnessRoot?: string; + /** Tests may replace the two specialized, expensive adapters. */ + readonly suiteAdapters?: NativeSuiteAdapters; + }, +): Promise { + const scenarios = expandSuiteFrameworks(loadScenarioSuite(suite, options.scenarioDir)); + if (scenarios.length === 0) throw new Error(`no performance scenarios in suite ${JSON.stringify(suite)}`); + const estimatedSeconds = estimatedSuiteSeconds(scenarios); + if ( + options.maxEstimatedSeconds !== undefined && + estimatedSeconds > options.maxEstimatedSeconds + ) { + throw new Error( + `${suite} suite estimate ${estimatedSeconds}s exceeds the ${options.maxEstimatedSeconds}s limit`, + ); + } + const results: NativeRunResult[] = []; + const sourceRoot = resolve(options.sourceRoot); + const harnessRoot = resolve( + options.harnessRoot ?? new URL("../../..", import.meta.url).pathname, + ); + const adapters = options.suiteAdapters ?? DEFAULT_SUITE_ADAPTERS; + for (const scenario of scenarios) { + if (isDamageScenario(scenario)) { + results.push(await adapters.damage(scenario, { + sourceRoot, + harnessRoot, + outDir: options.outDir, + })); + } else if (scenario.subject.family === "vapor") { + results.push(await adapters.vapor(scenario, { + sourceRoot, + harnessRoot, + outDir: options.outDir, + })); + } else { + results.push(await runNativeQuick(scenario, { + sourceRoot, + outDir: options.outDir, + bootAdapter: options.bootAdapter, + })); + } + } + return { + schemaVersion: 1, + kind: "pocketjs.perf.native-suite-result", + suite, + estimatedSeconds, + results, + }; +} diff --git a/tools/perf/scenarios/boot.json b/tools/perf/scenarios/boot.json new file mode 100644 index 00000000..20d80a2e --- /dev/null +++ b/tools/perf/scenarios/boot.json @@ -0,0 +1,60 @@ +{ + "schemaVersion": 1, + "kind": "pocketjs.perf.scenario", + "id": "guest.hero.boot.v1", + "suite": "quick", + "subject": { + "id": "hero-main", + "family": "guest-app", + "framework": "solid", + "entry": "hero-main" + }, + "executorRequirements": [ + "guest.frame", + "core.ui", + "renderer.framebuffer", + "assets.pak", + "correctness.framebuffer", + "correctness.draw-list", + "correctness.effects", + "correctness.state-final" + ], + "frames": 120, + "tape": { + "schemaVersion": 1, + "kind": "pocketjs.perf.input-tape", + "id": "guest.hero.boot.idle-120.v1", + "frames": 120, + "tracks": [] + }, + "phases": [ + { "name": "first-frame", "startFrame": 0, "endFrame": 1, "collect": true }, + { "name": "settle", "startFrame": 1, "endFrame": 120, "collect": false } + ], + "checkpoints": [ + { "frame": 0, "capture": ["framebuffer"] }, + { "frame": 119, "capture": ["framebuffer", "effects", "state"] } + ], + "params": { + "estimatedSeconds": 4, + "measureBoot": true, + "frameworks": ["solid", "vue-vapor", "octane"], + "frameworkSubjects": { + "vue-vapor": { + "id": "hero-vue-vapor-main", + "entry": "hero-vue-vapor-main.vue-vapor" + } + }, + "gateMetrics": [ + "artifact.bundle_bytes", + "guest.instructions", + "quickjs.live_bytes_after_gc" + ], + "diagnosticMetrics": [ + "memory.current_bytes", + "memory.peak_bytes", + "native.boot_wall_time_ns", + "native.phase.first-frame.wall_time_ns" + ] + } +} diff --git a/tools/perf/scenarios/damage.json b/tools/perf/scenarios/damage.json new file mode 100644 index 00000000..a6053b30 --- /dev/null +++ b/tools/perf/scenarios/damage.json @@ -0,0 +1,42 @@ +{ + "schemaVersion": 1, + "kind": "pocketjs.perf.scenario", + "id": "core.damage-cases.v1", + "suite": "quick", + "subject": { + "id": "core-damage-lab", + "family": "core-lab", + "framework": "core", + "entry": "tools/perf/fixtures/core-damage-lab" + }, + "executorRequirements": ["fixture.core.damage", "correctness.framebuffer", "correctness.draw-list"], + "frames": 960, + "tape": { + "schemaVersion": 1, + "kind": "pocketjs.perf.input-tape", + "id": "core.damage-cases.idle.v1", + "frames": 960, + "tracks": [] + }, + "phases": [ + { "name": "single-small", "startFrame": 0, "endFrame": 120, "collect": true }, + { "name": "corner-touch", "startFrame": 120, "endFrame": 240, "collect": true }, + { "name": "overlap", "startFrame": 240, "endFrame": 360, "collect": true }, + { "name": "eight-sparse", "startFrame": 360, "endFrame": 480, "collect": true }, + { "name": "structural", "startFrame": 480, "endFrame": 600, "collect": true }, + { "name": "clip-transform", "startFrame": 600, "endFrame": 720, "collect": true }, + { "name": "texture-in-place", "startFrame": 720, "endFrame": 840, "collect": true }, + { "name": "settle", "startFrame": 840, "endFrame": 960, "collect": true } + ], + "checkpoints": [ + { "frame": 119, "capture": ["framebuffer", "drawList"] }, + { "frame": 479, "capture": ["framebuffer", "drawList"] }, + { "frame": 839, "capture": ["framebuffer", "drawList"] }, + { "frame": 959, "capture": ["framebuffer", "drawList"] } + ], + "params": { + "estimatedSeconds": 8, + "gateMetrics": ["guest.instructions", "guest.load_store_events", "memory.allocated_bytes"], + "diagnosticMetrics": [] + } +} diff --git a/tools/perf/scenarios/deepzoom.json b/tools/perf/scenarios/deepzoom.json new file mode 100644 index 00000000..53412e27 --- /dev/null +++ b/tools/perf/scenarios/deepzoom.json @@ -0,0 +1,76 @@ +{ + "schemaVersion": 1, + "kind": "pocketjs.perf.scenario", + "id": "guest.zoomlab.deepzoom.v1", + "suite": "quick", + "subject": { + "id": "zoomlab-main", + "family": "guest-app", + "framework": "solid", + "entry": "zoomlab-main" + }, + "executorRequirements": [ + "guest.frame", + "core.ui", + "renderer.framebuffer", + "assets.pak", + "input.buttons", + "input.analog", + "correctness.framebuffer", + "correctness.draw-list", + "correctness.effects", + "correctness.state-final" + ], + "frames": 480, + "tape": { + "schemaVersion": 1, + "kind": "pocketjs.perf.input-tape", + "id": "guest.zoomlab.zoom-pan.v1", + "frames": 480, + "tracks": [ + { + "kind": "button", + "control": "shoulder-right", + "samples": [ + { "frame": 60, "pressed": true }, + { "frame": 180, "pressed": false } + ] + }, + { + "kind": "analog", + "control": "x", + "samples": [ + { "frame": 180, "value": -1 }, + { "frame": 360, "value": 0 } + ] + } + ] + }, + "phases": [ + { "name": "warmup", "startFrame": 0, "endFrame": 60, "collect": false }, + { "name": "zoom", "startFrame": 60, "endFrame": 180, "collect": true }, + { "name": "pan", "startFrame": 180, "endFrame": 360, "collect": true }, + { "name": "glide", "startFrame": 360, "endFrame": 480, "collect": true } + ], + "checkpoints": [ + { "frame": 59, "capture": ["framebuffer"] }, + { "frame": 179, "capture": ["framebuffer"] }, + { "frame": 359, "capture": ["framebuffer"] }, + { "frame": 479, "capture": ["framebuffer", "effects", "state"] } + ], + "params": { + "estimatedSeconds": 12, + "gateMetrics": [ + "guest.instructions", + "guest.load_store_events", + "memory.allocated_bytes" + ], + "diagnosticMetrics": [ + "memory.current_bytes", + "memory.peak_bytes", + "native.phase.zoom.wall_time_ns", + "native.phase.pan.wall_time_ns", + "native.phase.glide.wall_time_ns" + ] + } +} diff --git a/tools/perf/scenarios/fixed-text.json b/tools/perf/scenarios/fixed-text.json new file mode 100644 index 00000000..cb7f0eb7 --- /dev/null +++ b/tools/perf/scenarios/fixed-text.json @@ -0,0 +1,50 @@ +{ + "schemaVersion": 1, + "kind": "pocketjs.perf.scenario", + "id": "guest.stats.text-update.v1", + "suite": "quick", + "subject": { + "id": "stats-main", + "family": "guest-app", + "framework": "solid", + "entry": "stats-main" + }, + "executorRequirements": [ + "guest.frame", + "core.ui", + "renderer.framebuffer", + "assets.pak", + "correctness.framebuffer", + "correctness.draw-list", + "correctness.effects", + "correctness.state-final" + ], + "frames": 180, + "tape": { + "schemaVersion": 1, + "kind": "pocketjs.perf.input-tape", + "id": "guest.stats.idle-180.v1", + "frames": 180, + "tracks": [] + }, + "phases": [ + { "name": "text-active", "startFrame": 0, "endFrame": 80, "collect": true }, + { "name": "settled", "startFrame": 80, "endFrame": 180, "collect": true } + ], + "checkpoints": [ + { "frame": 79, "capture": ["framebuffer"] }, + { "frame": 179, "capture": ["framebuffer", "effects", "state"] } + ], + "params": { + "estimatedSeconds": 4, + "gateMetrics": [ + "guest.instructions", + "guest.load_store_events", + "memory.allocated_bytes" + ], + "diagnosticMetrics": [ + "native.phase.text-active.wall_time_ns", + "native.phase.settled.wall_time_ns" + ] + } +} diff --git a/tools/perf/scenarios/idle.json b/tools/perf/scenarios/idle.json new file mode 100644 index 00000000..66897e7c --- /dev/null +++ b/tools/perf/scenarios/idle.json @@ -0,0 +1,52 @@ +{ + "schemaVersion": 1, + "kind": "pocketjs.perf.scenario", + "id": "guest.fixture.idle-600.v1", + "suite": "quick", + "subject": { + "id": "tools/perf/apps/idle-fixture-main.tsx", + "family": "guest-app", + "framework": "solid", + "entry": "idle-fixture-main" + }, + "executorRequirements": [ + "guest.frame", + "core.ui", + "renderer.framebuffer", + "assets.pak", + "correctness.framebuffer", + "correctness.draw-list", + "correctness.effects", + "correctness.state-final" + ], + "frames": 720, + "tape": { + "schemaVersion": 1, + "kind": "pocketjs.perf.input-tape", + "id": "guest.fixture.idle-720.v1", + "frames": 720, + "tracks": [] + }, + "phases": [ + { "name": "warmup", "startFrame": 0, "endFrame": 120, "collect": false }, + { "name": "idle", "startFrame": 120, "endFrame": 720, "collect": true } + ], + "checkpoints": [ + { "frame": 119, "capture": ["framebuffer", "drawList"] }, + { "frame": 719, "capture": ["framebuffer", "drawList", "effects", "state"] } + ], + "params": { + "estimatedSeconds": 6, + "gateMetrics": [ + "guest.instructions", + "guest.load_store_events", + "memory.allocated_bytes", + "quickjs.live_bytes_after_gc" + ], + "diagnosticMetrics": [ + "memory.current_bytes", + "memory.peak_bytes", + "native.phase.idle.wall_time_ns" + ] + } +} diff --git a/tools/perf/scenarios/list.json b/tools/perf/scenarios/list.json new file mode 100644 index 00000000..c9151b77 --- /dev/null +++ b/tools/perf/scenarios/list.json @@ -0,0 +1,85 @@ +{ + "schemaVersion": 1, + "kind": "pocketjs.perf.scenario", + "id": "guest.fixture.keyed-list.v1", + "suite": "quick", + "subject": { + "id": "tools/perf/apps/list-fixture-main.tsx", + "family": "guest-app", + "framework": "solid", + "entry": "list-fixture-main" + }, + "executorRequirements": [ + "guest.frame", + "core.ui", + "renderer.framebuffer", + "assets.pak", + "input.buttons", + "correctness.framebuffer", + "correctness.draw-list", + "correctness.effects", + "correctness.state-final" + ], + "frames": 180, + "tape": { + "schemaVersion": 1, + "kind": "pocketjs.perf.input-tape", + "id": "guest.fixture.keyed-list.v1", + "frames": 180, + "tracks": [ + { + "kind": "button", + "control": "quaternary", + "samples": [ + { "frame": 32, "pressed": true }, + { "frame": 33, "pressed": false } + ] + }, + { + "kind": "button", + "control": "secondary", + "samples": [ + { "frame": 62, "pressed": true }, + { "frame": 63, "pressed": false } + ] + }, + { + "kind": "button", + "control": "primary", + "samples": [ + { "frame": 92, "pressed": true }, + { "frame": 93, "pressed": false } + ] + } + ] + }, + "phases": [ + { "name": "warmup", "startFrame": 0, "endFrame": 30, "collect": false }, + { "name": "keyed-insert", "startFrame": 30, "endFrame": 60, "collect": true }, + { "name": "keyed-reorder", "startFrame": 60, "endFrame": 90, "collect": true }, + { "name": "keyed-delete", "startFrame": 90, "endFrame": 120, "collect": true }, + { "name": "steady", "startFrame": 120, "endFrame": 180, "collect": true } + ], + "checkpoints": [ + { "frame": 59, "capture": ["framebuffer", "drawList"] }, + { "frame": 89, "capture": ["framebuffer", "drawList"] }, + { "frame": 119, "capture": ["framebuffer", "drawList"] }, + { "frame": 179, "capture": ["framebuffer", "drawList", "effects", "state"] } + ], + "params": { + "estimatedSeconds": 8, + "gateMetrics": [ + "guest.instructions", + "guest.load_store_events", + "memory.allocated_bytes" + ], + "diagnosticMetrics": [ + "memory.current_bytes", + "memory.peak_bytes", + "native.phase.keyed-insert.wall_time_ns", + "native.phase.keyed-reorder.wall_time_ns", + "native.phase.keyed-delete.wall_time_ns", + "native.phase.steady.wall_time_ns" + ] + } +} diff --git a/tools/perf/scenarios/style.json b/tools/perf/scenarios/style.json new file mode 100644 index 00000000..cbd91b44 --- /dev/null +++ b/tools/perf/scenarios/style.json @@ -0,0 +1,67 @@ +{ + "schemaVersion": 1, + "kind": "pocketjs.perf.scenario", + "id": "guest.settings.style-journey.v1", + "suite": "quick", + "subject": { + "id": "settings-main", + "family": "guest-app", + "framework": "solid", + "entry": "settings-main" + }, + "executorRequirements": [ + "guest.frame", + "core.ui", + "renderer.framebuffer", + "assets.pak", + "input.buttons", + "correctness.framebuffer", + "correctness.draw-list", + "correctness.effects", + "correctness.state-final" + ], + "frames": 120, + "tape": { + "schemaVersion": 1, + "kind": "pocketjs.perf.input-tape", + "id": "guest.settings.style-journey.v1", + "frames": 120, + "tracks": [ + { + "kind": "button", + "control": "down", + "samples": [ + { "frame": 4, "pressed": true }, { "frame": 5, "pressed": false }, + { "frame": 16, "pressed": true }, { "frame": 17, "pressed": false }, + { "frame": 28, "pressed": true }, { "frame": 29, "pressed": false }, + { "frame": 40, "pressed": true }, { "frame": 41, "pressed": false }, + { "frame": 44, "pressed": true }, { "frame": 45, "pressed": false }, + { "frame": 48, "pressed": true }, { "frame": 49, "pressed": false } + ] + }, + { + "kind": "button", + "control": "primary", + "samples": [ + { "frame": 10, "pressed": true }, { "frame": 11, "pressed": false }, + { "frame": 22, "pressed": true }, { "frame": 23, "pressed": false }, + { "frame": 34, "pressed": true }, { "frame": 35, "pressed": false }, + { "frame": 54, "pressed": true }, { "frame": 55, "pressed": false } + ] + } + ] + }, + "phases": [ + { "name": "interaction", "startFrame": 0, "endFrame": 70, "collect": true }, + { "name": "settle", "startFrame": 70, "endFrame": 120, "collect": true } + ], + "checkpoints": [ + { "frame": 69, "capture": ["framebuffer"] }, + { "frame": 119, "capture": ["framebuffer", "effects", "state"] } + ], + "params": { + "estimatedSeconds": 6, + "gateMetrics": ["guest.instructions", "guest.load_store_events", "memory.allocated_bytes"], + "diagnosticMetrics": ["native.phase.interaction.wall_time_ns", "native.phase.settle.wall_time_ns"] + } +} diff --git a/tools/perf/scenarios/timer.json b/tools/perf/scenarios/timer.json new file mode 100644 index 00000000..c7fa7bdd --- /dev/null +++ b/tools/perf/scenarios/timer.json @@ -0,0 +1,68 @@ +{ + "schemaVersion": 1, + "kind": "pocketjs.perf.scenario", + "id": "guest.cafe.timer-effects.v1", + "suite": "quick", + "subject": { + "id": "cafe-main", + "family": "guest-app", + "framework": "solid", + "entry": "cafe-main" + }, + "executorRequirements": [ + "guest.frame", + "core.ui", + "renderer.framebuffer", + "assets.pak", + "input.buttons", + "correctness.framebuffer", + "correctness.draw-list", + "correctness.effects", + "correctness.state-final" + ], + "frames": 390, + "tape": { + "schemaVersion": 1, + "kind": "pocketjs.perf.input-tape", + "id": "guest.cafe.timer-effects.v1", + "frames": 390, + "tracks": [ + { + "kind": "button", + "control": "primary", + "samples": [ + { "frame": 60, "pressed": true }, { "frame": 61, "pressed": false }, + { "frame": 120, "pressed": true }, { "frame": 121, "pressed": false }, + { "frame": 180, "pressed": true }, { "frame": 181, "pressed": false } + ] + }, + { + "kind": "button", + "control": "down", + "samples": [ + { "frame": 90, "pressed": true }, { "frame": 91, "pressed": false } + ] + }, + { + "kind": "button", + "control": "start", + "samples": [ + { "frame": 210, "pressed": true }, { "frame": 211, "pressed": false } + ] + } + ] + }, + "phases": [ + { "name": "timer-effects", "startFrame": 0, "endFrame": 390, "collect": true } + ], + "checkpoints": [ + { "frame": 29, "capture": ["framebuffer", "effects"] }, + { "frame": 269, "capture": ["framebuffer", "effects"] }, + { "frame": 389, "capture": ["framebuffer", "effects", "state"] } + ], + "params": { + "estimatedSeconds": 7, + "gateMetrics": ["guest.instructions", "memory.allocated_bytes"], + "diagnosticMetrics": ["native.phase.timer-effects.wall_time_ns"] + } +} diff --git a/tools/perf/scenarios/touch.json b/tools/perf/scenarios/touch.json new file mode 100644 index 00000000..b4845def --- /dev/null +++ b/tools/perf/scenarios/touch.json @@ -0,0 +1,55 @@ +{ + "schemaVersion": 1, + "kind": "pocketjs.perf.scenario", + "id": "guest.im.touch-row.v1", + "suite": "quick", + "subject": { + "id": "im-main", + "family": "guest-app", + "framework": "solid", + "entry": "im-main" + }, + "executorRequirements": [ + "guest.frame", + "core.ui", + "renderer.framebuffer", + "assets.pak", + "input.touch", + "correctness.framebuffer", + "correctness.draw-list", + "correctness.effects", + "correctness.state-final" + ], + "frames": 110, + "tape": { + "schemaVersion": 1, + "kind": "pocketjs.perf.input-tape", + "id": "guest.im.row-touch.v1", + "frames": 110, + "tracks": [ + { + "kind": "touch", + "control": "contact-0", + "samples": [ + { "frame": 60, "phase": "start", "x": 240, "y": 90 }, + { "frame": 63, "phase": "end", "x": 240, "y": 90 } + ] + } + ] + }, + "phases": [ + { "name": "bootstrap", "startFrame": 0, "endFrame": 50, "collect": false }, + { "name": "touch", "startFrame": 50, "endFrame": 80, "collect": true }, + { "name": "settle", "startFrame": 80, "endFrame": 110, "collect": false } + ], + "checkpoints": [ + { "frame": 59, "capture": ["framebuffer"] }, + { "frame": 79, "capture": ["framebuffer"] }, + { "frame": 109, "capture": ["framebuffer", "effects", "state"] } + ], + "params": { + "estimatedSeconds": 5, + "gateMetrics": ["guest.instructions", "memory.allocated_bytes"], + "diagnosticMetrics": ["memory.current_bytes", "native.phase.touch.wall_time_ns"] + } +} diff --git a/tools/perf/scenarios/vapor.json b/tools/perf/scenarios/vapor.json new file mode 100644 index 00000000..3f9f46bd --- /dev/null +++ b/tools/perf/scenarios/vapor.json @@ -0,0 +1,75 @@ +{ + "schemaVersion": 1, + "kind": "pocketjs.perf.scenario", + "id": "vapor.todo.reactive-grid.v1", + "suite": "quick", + "subject": { + "id": "vapor-todo", + "family": "vapor", + "framework": "core", + "entry": "vapor/examples/todo/todo.tsx" + }, + "executorRequirements": [ + "fixture.vapor.generated-c", + "input.buttons", + "input.relative-axis", + "correctness.draw-list", + "correctness.effects", + "correctness.framebuffer", + "correctness.state-final" + ], + "frames": 720, + "tape": { + "schemaVersion": 1, + "kind": "pocketjs.perf.input-tape", + "id": "vapor.todo.quick-actions.v1", + "frames": 720, + "tracks": [ + { + "kind": "button", + "control": "down", + "samples": [ + { "frame": 120, "pressed": true }, + { "frame": 121, "pressed": false } + ] + }, + { + "kind": "button", + "control": "primary", + "samples": [ + { "frame": 180, "pressed": true }, + { "frame": 181, "pressed": false } + ] + }, + { + "kind": "relative-axis", + "control": "primary", + "samples": [ + { "frame": 240, "delta": 44999 }, + { "frame": 241, "delta": 1 }, + { "frame": 300, "delta": -45000 } + ] + } + ] + }, + "phases": [ + { "name": "idle", "startFrame": 0, "endFrame": 120, "collect": true }, + { "name": "reactive", "startFrame": 120, "endFrame": 360, "collect": true }, + { "name": "settle", "startFrame": 360, "endFrame": 720, "collect": true } + ], + "checkpoints": [ + { "frame": 119, "capture": ["state"] }, + { "frame": 359, "capture": ["state"] }, + { "frame": 719, "capture": ["state"] } + ], + "params": { + "estimatedSeconds": 20, + "gateMetrics": [ + "guest.instructions", + "memory.allocations", + "memory.allocated_bytes", + "artifact.elf_text_rodata_bytes" + ], + "diagnosticMetrics": [] + } +} diff --git a/tools/test.ts b/tools/test.ts index 679b3f47..2062caa2 100644 --- a/tools/test.ts +++ b/tools/test.ts @@ -74,6 +74,13 @@ const SUITE: readonly Stage[] = [ "tests/video-outro.test.ts", "tests/osk-layout.test.ts", "tests/test-suite.test.ts", + "tests/perf-comparator.test.ts", + "tests/perf-receipts.test.ts", + "tests/perf-runner.test.ts", + "tests/perf-cli.test.ts", + "tests/perf-qemu-executor.test.ts", + "tests/perf-damage-executor.test.ts", + "tests/perf-vapor-executor.test.ts", ], }, {