From 9c7f42f8db8307615c3f65abd1f1c38be0a04287 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 22 Aug 2026 00:52:08 +0900 Subject: [PATCH] =?UTF-8?q?test(scripts):=20smol-worker=20A/B=20gate=20har?= =?UTF-8?q?ness=20=E2=80=94=20verdict=20FAIL,=20smol=20flags=20not=20lande?= =?UTF-8?q?d?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fresh-child-process isolation per run (in-process sequential runs contaminate baselines via allocator page retention — the first version measured a phantom win). Peak from Subprocess.resourceUsage().maxRSS over the audited workload shape (100MB row materialization + aggregate JSON). Result on Bun 1.4.0 darwin/arm64: elapsed within bound but median peak RSS NOT reduced (447.76MB vs 447.81MB) — the burst-allocation batch shape is dominated by live data, not heap growth policy. Per the audited pre-landing gate, the production Worker call sites keep full-size heaps; harness + devlog record are the deliverable. Unit: devlog/_plan/260822_260822-bun14-followup-memory/030 --- .../030_smol_workers.md | 27 ++++ scripts/smol-worker-ab.ts | 120 ++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 scripts/smol-worker-ab.ts diff --git a/devlog/_plan/260822_260822-bun14-followup-memory/030_smol_workers.md b/devlog/_plan/260822_260822-bun14-followup-memory/030_smol_workers.md index b1d3919b87..72b127b3da 100644 --- a/devlog/_plan/260822_260822-bun14-followup-memory/030_smol_workers.md +++ b/devlog/_plan/260822_260822-bun14-followup-memory/030_smol_workers.md @@ -43,3 +43,30 @@ unit. ## Measurement claim Local A/B is the primary evidence (host-independent fixtures); macmini optional. + +## A/B gate result (2026-08-22, local darwin/arm64, Bun 1.4.0) + +Harness: scripts/smol-worker-ab.ts — fresh child process per run, +peak = Subprocess.resourceUsage().maxRSS, audited workload shape +(100 MB row materialization + aggregate JSON serialization), 3 runs per arm. + +| arm | median elapsed | median maxRSS | +|---|---|---| +| smol off | 37.84 ms | 447,758,336 B | +| smol on | 38.72 ms | 447,807,488 B | + +completionSuccess: true; elapsedWithin25Pct: true; peakRssReduced: FALSE → +**gate verdict: FAIL — smol flags are NOT landed.** + +Reading: for this allocation shape (large short-lived arrays + one aggregate +string) the Small heap growth policy changes peak RSS by ~0.01% — the peak is +dominated by the live data itself, which no GC policy can shrink. smol's +documented benefit targets long-lived idle workers, not burst-allocation batch +jobs. Honest outcome per the audited plan: production Worker call sites keep +the full-size heap; the harness and this record are the deliverable. + +Measurement-integrity note: an earlier in-process sequential version of the +harness produced a phantom smol win (contaminated baselines — allocator page +retention across runs). The fresh-child-process isolation is the valid design; +report.json carries the isolation note. + diff --git a/scripts/smol-worker-ab.ts b/scripts/smol-worker-ab.ts new file mode 100644 index 0000000000..218466b3bd --- /dev/null +++ b/scripts/smol-worker-ab.ts @@ -0,0 +1,120 @@ +/** + * smol-worker A/B gate (devlog/_plan/260822_260822-bun14-followup-memory/030). + * + * Measures whether Worker({ smol: true }) reduces peak RSS without breaking the + * audited large-payload shapes of the three production workers: + * - policy/restore (cleanup.ts): materialize all rows into arrays, then + * serialize the aggregate backup JSON; + * - history (history-provider.ts): full SQLite result sets + rollout buffers. + * + * Isolation: every run is a FRESH child process (in-process sequential runs + * contaminate baselines — the allocator retains pages across runs, which the + * first version of this script measured as a phantom smol win). The child runs + * the workload in a real Worker thread and exits; the parent reads the child's + * peak RSS from Subprocess.resourceUsage().maxRSS. + * + * Acceptance (audited gate): completion success, elapsed within +25% of + * baseline, peak RSS reduced. + * + * Usage: bun scripts/smol-worker-ab.ts [payloadMb] [runs] + */ +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +const outDir = process.argv[2]; +const PAYLOAD_MB = Number.parseInt(process.argv[3] ?? "100", 10); +const RUNS = Number.parseInt(process.argv[4] ?? "3", 10); +if (!outDir) throw new Error("usage: bun scripts/smol-worker-ab.ts [payloadMb] [runs]"); +mkdirSync(outDir, { recursive: true }); + +/** Child body: runs the audited workload shape inside a real Worker, prints elapsed. */ +const childSource = "const smol = process.argv[2] === \"smol\";\nconst payloadMb = Number.parseInt(process.argv[3] ?? \"100\", 10);\nconst workerSource = \"self.onmessage = (event) => {\\n const { payloadMb } = event.data;\\n const t0 = performance.now();\\n try {\\n const ROW_BYTES = 4096;\\n const rowCount = Math.floor((payloadMb * 1024 * 1024) / ROW_BYTES);\\n const rows = [];\\n for (let i = 0; i < rowCount; i++) {\\n rows.push({\\n id: \\\"row-\\\" + i,\\n kind: i % 3 === 0 ? \\\"thread\\\" : i % 3 === 1 ? \\\"log\\\" : \\\"memory\\\",\\n payload: \\\"x\\\".repeat(ROW_BYTES - 96),\\n at: Date.now(),\\n });\\n }\\n const backup = JSON.stringify({ rows });\\n self.postMessage({ type: \\\"done\\\", rowCount, bytes: backup.length, elapsedMs: performance.now() - t0 });\\n } catch (error) {\\n self.postMessage({ type: \\\"error\\\", message: String(error), elapsedMs: performance.now() - t0 });\\n }\\n};\";\nconst url = URL.createObjectURL(new Blob([workerSource], { type: \"application/javascript\" }));\nconst worker = new Worker(url, smol ? { smol: true } : {});\nconst outcome = await new Promise((resolve, reject) => {\n const timeout = setTimeout(() => reject(new Error(\"worker timeout (120s)\")), 120_000);\n worker.onmessage = (event) => { clearTimeout(timeout); resolve(event.data); };\n worker.onerror = (event) => { clearTimeout(timeout); reject(new Error(event.message || \"worker error\")); };\n worker.postMessage({ payloadMb });\n});\nworker.terminate();\nconsole.log(JSON.stringify(outcome));"; + +const childPath = join(outDir, "ab-child.ts"); +writeFileSync(childPath, childSource); + +type RunResult = { + smol: boolean; + runIndex: number; + ok: boolean; + elapsedMs: number; + maxRssBytes: number; + error?: string; +}; + +async function oneRun(smol: boolean, runIndex: number): Promise { + const child = Bun.spawn([process.execPath, childPath, smol ? "smol" : "full", String(PAYLOAD_MB)], { + stdout: "pipe", + stderr: "pipe", + }); + const exitCode = await child.exited; + const stdout = await new Response(child.stdout).text(); + const stderr = await new Response(child.stderr).text(); + const usage = child.resourceUsage(); + // Bun reports maxRSS in bytes on darwin (ru_maxrss is bytes on macOS, KiB on Linux; + // Bun.resourceUsage normalizes to bytes). + const maxRssBytes = usage?.maxRSS ?? -1; + let elapsedMs = -1; + let ok = false; + let error: string | undefined; + try { + const line = stdout.trim().split(/\r?\n/).pop() ?? ""; + const parsed = JSON.parse(line) as { type: string; elapsedMs: number; message?: string }; + ok = exitCode === 0 && parsed.type === "done"; + elapsedMs = parsed.elapsedMs; + error = parsed.message; + } catch { + error = "unparseable child output; exit " + exitCode + "; stderr: " + stderr.slice(0, 200); + } + return { smol, runIndex, ok, elapsedMs, maxRssBytes, ...(error ? { error } : {}) }; +} + +const results: RunResult[] = []; +for (let run = 0; run < RUNS; run++) { + for (const smol of [false, true]) { + const r = await oneRun(smol, run); + results.push(r); + console.log(JSON.stringify(r)); + } +} + +function median(values: number[]): number { + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)]!; +} + +const off = results.filter(r => !r.smol && r.ok); +const on = results.filter(r => r.smol && r.ok); +const gate = { + completionSuccess: on.length === RUNS && off.length === RUNS, + medianElapsedOffMs: median(off.map(r => r.elapsedMs)), + medianElapsedOnMs: median(on.map(r => r.elapsedMs)), + medianMaxRssOffBytes: median(off.map(r => r.maxRssBytes)), + medianMaxRssOnBytes: median(on.map(r => r.maxRssBytes)), + elapsedWithin25Pct: false, + peakRssReduced: false, + verdict: "fail" as "pass" | "fail", +}; +if (gate.completionSuccess) { + gate.elapsedWithin25Pct = gate.medianElapsedOnMs <= gate.medianElapsedOffMs * 1.25; + gate.peakRssReduced = gate.medianMaxRssOnBytes < gate.medianMaxRssOffBytes; + gate.verdict = gate.elapsedWithin25Pct && gate.peakRssReduced ? "pass" : "fail"; +} + +const report = { + bunVersion: Bun.version, + bunRevision: Bun.revision, + platform: process.platform, + arch: process.arch, + at: new Date().toISOString(), + payloadMb: PAYLOAD_MB, + runs: RUNS, + isolation: "fresh child process per run; peak = Subprocess.resourceUsage().maxRSS", + results, + gate, +}; +writeFileSync(join(outDir, "report.json"), JSON.stringify(report, null, 2)); +console.log(JSON.stringify(gate)); +console.log("report: " + join(outDir, "report.json")); + +