diff --git a/.github/openclaw/artifacts.py b/.github/openclaw/artifacts.py index d142126df018c..0e745538e8882 100644 --- a/.github/openclaw/artifacts.py +++ b/.github/openclaw/artifacts.py @@ -11,7 +11,7 @@ import sys import time -BASE = '1600131e46b5af48bbda3559af8d8a3327230b6e' +BASE = '5718a6ec579b98362ea7276a426deedcc6281ef5' ROOT = Path(__file__).resolve().parents[2] CONFIG = json.loads((ROOT / '.github/openclaw/variants.json').read_text()) RECIPE = ['.github/scripts/lanes.mjs', 'Dockerfile', 'Dockerfile.musl', 'Dockerfile.macos', 'Dockerfile.windows', 'icu', 'macos-cross'] @@ -45,7 +45,7 @@ def preflight(): sha = source() if sha != os.environ['INPUT_SHA'] or sha != os.environ['GITHUB_SHA']: raise ValueError('dispatch must use the same reviewed branch head and full input SHA') - if os.environ['GITHUB_REF'] not in ('refs/heads/openclaw/main', 'refs/heads/openclaw/batch-1'): + if os.environ['GITHUB_REF'] not in ('refs/heads/openclaw/main', 'refs/heads/openclaw/batch-1', 'refs/heads/openclaw/release-5718a6ec'): raise ValueError('only an explicitly permitted owned branch may run this workflow') upstream = {l['label'] for l in lane_definitions()} wanted = [l['label'] for l in CONFIG['lanes']] diff --git a/.github/openclaw/qualification/als/jsc/fixtures/a-leaf.js b/.github/openclaw/qualification/als/jsc/fixtures/a-leaf.js new file mode 100644 index 0000000000000..369431868462a --- /dev/null +++ b/.github/openclaw/qualification/als/jsc/fixtures/a-leaf.js @@ -0,0 +1,3 @@ +export const before = $vm.asyncContext(); +await Promise.resolve(); +export const after = $vm.asyncContext(); diff --git a/.github/openclaw/qualification/als/jsc/fixtures/a-parent.js b/.github/openclaw/qualification/als/jsc/fixtures/a-parent.js new file mode 100644 index 0000000000000..1731800cbefbb --- /dev/null +++ b/.github/openclaw/qualification/als/jsc/fixtures/a-parent.js @@ -0,0 +1,2 @@ +import { before, after } from './a-leaf.js'; +export const contexts = [before, after, $vm.asyncContext()]; diff --git a/.github/openclaw/qualification/als/jsc/fixtures/b-leaf.js b/.github/openclaw/qualification/als/jsc/fixtures/b-leaf.js new file mode 100644 index 0000000000000..369431868462a --- /dev/null +++ b/.github/openclaw/qualification/als/jsc/fixtures/b-leaf.js @@ -0,0 +1,3 @@ +export const before = $vm.asyncContext(); +await Promise.resolve(); +export const after = $vm.asyncContext(); diff --git a/.github/openclaw/qualification/als/jsc/fixtures/b-parent.js b/.github/openclaw/qualification/als/jsc/fixtures/b-parent.js new file mode 100644 index 0000000000000..757c2e6a45a08 --- /dev/null +++ b/.github/openclaw/qualification/als/jsc/fixtures/b-parent.js @@ -0,0 +1,2 @@ +import { before, after } from './b-leaf.js'; +export const contexts = [before, after, $vm.asyncContext()]; diff --git a/.github/openclaw/qualification/als/jsc/fixtures/deferred-leaf.js b/.github/openclaw/qualification/als/jsc/fixtures/deferred-leaf.js new file mode 100644 index 0000000000000..ff0f293997641 --- /dev/null +++ b/.github/openclaw/qualification/als/jsc/fixtures/deferred-leaf.js @@ -0,0 +1,3 @@ +export const dependencyContext = $vm.asyncContext(); +await Promise.resolve(); +if ($vm.asyncContext() !== dependencyContext) throw Error('TLA context'); diff --git a/.github/openclaw/qualification/als/jsc/fixtures/deferred-parent.js b/.github/openclaw/qualification/als/jsc/fixtures/deferred-parent.js new file mode 100644 index 0000000000000..290856eef6dc7 --- /dev/null +++ b/.github/openclaw/qualification/als/jsc/fixtures/deferred-parent.js @@ -0,0 +1,2 @@ +export { dependencyContext } from './deferred-leaf.js'; +export const rootContext = $vm.asyncContext(); diff --git a/.github/openclaw/qualification/als/jsc/fixtures/throw.js b/.github/openclaw/qualification/als/jsc/fixtures/throw.js new file mode 100644 index 0000000000000..8a947b064f9e4 --- /dev/null +++ b/.github/openclaw/qualification/als/jsc/fixtures/throw.js @@ -0,0 +1,3 @@ +globalThis.contextLog.push($vm.asyncContext()); +$vm.setAsyncContext(['residue']); +throw Error('module throw'); diff --git a/.github/openclaw/qualification/als/jsc/fixtures/unscoped-leaf.js b/.github/openclaw/qualification/als/jsc/fixtures/unscoped-leaf.js new file mode 100644 index 0000000000000..54206e8c133a3 --- /dev/null +++ b/.github/openclaw/qualification/als/jsc/fixtures/unscoped-leaf.js @@ -0,0 +1,2 @@ +await Promise.resolve(); +export const value = 137; diff --git a/.github/openclaw/qualification/als/jsc/fixtures/unscoped-parent.js b/.github/openclaw/qualification/als/jsc/fixtures/unscoped-parent.js new file mode 100644 index 0000000000000..80fb64a54e9b0 --- /dev/null +++ b/.github/openclaw/qualification/als/jsc/fixtures/unscoped-parent.js @@ -0,0 +1 @@ +export { value } from './unscoped-leaf.js'; diff --git a/.github/openclaw/qualification/als/jsc/module-context.js b/.github/openclaw/qualification/als/jsc/module-context.js new file mode 100644 index 0000000000000..63dc9ead705b2 --- /dev/null +++ b/.github/openclaw/qualification/als/jsc/module-context.js @@ -0,0 +1,57 @@ +//@ requireOptions("--useDollarVM=1", "--useImportDefer=1") +function eq(actual, expected, label) { + if (actual !== expected) throw Error(label + ': expected ' + expected + ', got ' + actual); +} +function scoped(ctx, fn) { + const previous = $vm.asyncContext(); + $vm.setAsyncContext(ctx); + try { return fn(); } finally { $vm.setAsyncContext(previous); } +} +let complete = false; +// Exercise a raw import.defer payload before async-context tracking is enabled. +import.defer('./fixtures/unscoped-parent.js').then(ns => { + eq(ns.value, 137, 'deferred import without ALS'); + complete = true; +}, $vm.abort); +drainMicrotasks(); +eq(complete, true, 'unscoped defer completed'); +const A = ['A'], B = ['B']; +globalThis.contextLog = []; +let done = 0; +for (const [ctx, file] of [[A, 'a'], [B, 'b']]) { + scoped(ctx, () => import('./fixtures/' + file + '-parent.js').then(ns => { + eq($vm.asyncContext(), ctx, 'caller continuation'); + for (const value of ns.contexts) eq(value, ctx, 'graph ' + file); + done++; + }, $vm.abort)); +} +gc(); +drainMicrotasks(); +eq(done, 2, 'disjoint graphs completed'); +eq($vm.asyncContext(), undefined, 'outside graphs'); +let rejected = 0; +scoped(A, () => import('./fixtures/throw.js').then($vm.abort, e => { + eq(e.message, 'module throw', 'evaluation rejection'); + eq($vm.asyncContext(), A, 'rejection caller'); + rejected++; +})); +scoped(B, () => import('./fixtures/missing.js').then($vm.abort, () => { + eq($vm.asyncContext(), B, 'load rejection caller'); + rejected++; +})); +drainMicrotasks(); +eq(rejected, 2, 'rejections completed'); +eq(contextLog[0], A, 'throwing body'); +eq($vm.asyncContext(), undefined, 'outside rejection'); +let deferred = false; +scoped(A, () => import.defer('./fixtures/deferred-parent.js').then(ns => { + scoped(B, () => { + eq(ns.rootContext, B, 'deferred root evaluates at access'); + eq(ns.dependencyContext, A, 'TLA dependency evaluates in importer'); + }); + deferred = true; +}, $vm.abort)); +gc(); +drainMicrotasks(); +eq(deferred, true, 'scoped deferred graph completed'); +eq($vm.asyncContext(), undefined, 'outside deferred graph'); diff --git a/.github/openclaw/qualification/als/realm-require.mjs b/.github/openclaw/qualification/als/realm-require.mjs new file mode 100644 index 0000000000000..167bb2f5d940a --- /dev/null +++ b/.github/openclaw/qualification/als/realm-require.mjs @@ -0,0 +1,27 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import vm from 'node:vm'; +import { createRequire } from 'node:module'; +import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +const dir = mkdtempSync(join(tmpdir(), 'w137-realm-')); +const als = new AsyncLocalStorage(); +const A={name:'A'},B={name:'B'}; +globalThis.w137RealmRead=()=>als.getStore(); +function eq(a,b,label){if(a!==b)throw Error(label);} +try { + for (const name of ['a','b','sync']) { + writeFileSync(join(dir,name+'-leaf.mjs'), 'export const value=globalThis.w137RealmRead();\n'); + writeFileSync(join(dir,name+'.mjs'), `import {value} from './${name}-leaf.mjs'; export const captured=[value,globalThis.w137RealmRead()];\n`); + } + const context=vm.createContext({als}); + const run=file=>new vm.Script(`import(${JSON.stringify(pathToFileURL(join(dir,file+'.mjs')).href)})`,{importModuleDynamically:specifier=>import(specifier)}).runInContext(context); + const [a,b]=await Promise.all([als.run(A,()=>run('a')),als.run(B,()=>run('b'))]); + for(const x of a.captured)eq(x,A,'realm A'); + for(const x of b.captured)eq(x,B,'realm B'); + const required=als.run(A,()=>createRequire(import.meta.url)(join(dir,'sync.mjs'))); + for(const x of required.captured)eq(x,A,'synchronous require ESM'); + eq(als.getStore(),undefined,'outside realm'); + console.log(JSON.stringify({passed:true,vmGraphs:2,syncRequire:1})); +} finally { delete globalThis.w137RealmRead;rmSync(dir,{recursive:true,force:true}); } diff --git a/.github/openclaw/qualification/als/retention.mjs b/.github/openclaw/qualification/als/retention.mjs new file mode 100644 index 0000000000000..d978c94fb4aa4 --- /dev/null +++ b/.github/openclaw/qualification/als/retention.mjs @@ -0,0 +1,51 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import { heapStats } from 'bun:jsc'; +import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { setImmediate as turn } from 'node:timers/promises'; +const dir = mkdtempSync(join(tmpdir(), 'w137-retention-')); +const als = new AsyncLocalStorage(); +const refs = []; +const payloadRefs = []; +const metrics = []; +const bytesPerStore = 128 * 1024; +const batchSize = 256; +const batches = 6; +globalThis.w137ReadId = () => als.getStore()?.id; +function schedule(i) { + const store = { id: i, payload: new Uint8Array(bytesPerStore).fill(i & 255) }; + refs.push(new WeakRef(store)); + payloadRefs.push(new WeakRef(store.payload), new WeakRef(store.payload.buffer)); + const file = join(dir, i + '.mjs'); + const reject = i % 3 === 0; + writeFileSync(file, `await Promise.resolve(); const id=globalThis.w137ReadId(); if(id!==${i}) throw Error('context:'+id); ${reject ? "throw Error('expected-rejection');" : 'export const value=1;'}\n`); + return als.run(store, () => import(pathToFileURL(file).href)).then( + () => { if (reject) throw Error('missing rejection'); }, + error => { if (!reject || error.message !== 'expected-rejection') throw error; } + ); +} +async function collect() { + await turn(); Bun.gc(true); await turn(); Bun.gc(true); await turn(); +} +try { + await collect(); + metrics.push({ batch: 0, ...heapStats(), memory: process.memoryUsage() }); + for (let batch = 0; batch < batches; batch++) { + for (let offset = 0; offset < batchSize; offset += 16) + await Promise.all(Array.from({ length: 16 }, (_, j) => schedule(batch * batchSize + offset + j))); + await collect(); + const alive = refs.reduce((n, ref) => n + (ref.deref() !== undefined), 0); + const payloadsAlive = payloadRefs.reduce((n, ref) => n + (ref.deref() !== undefined), 0); + metrics.push({ batch: batch + 1, stores: refs.length, alive, payloadsAlive, ...heapStats(), memory: process.memoryUsage() }); + if (alive || payloadsAlive) throw Error(`retained ${alive}/${refs.length} contexts or ${payloadsAlive} payload objects`); + if (als.getStore() !== undefined) throw Error('context leaked outside import'); + } + const growth = metrics.at(-1).extraMemorySize - metrics[0].extraMemorySize; + if (growth > 8 * 1024 * 1024) throw Error('excessive external heap growth: ' + growth); + console.log(JSON.stringify({ passed: true, bytesPerStore, batchSize, batches, allocatedPayloadBytes: bytesPerStore*batchSize*batches, metrics }, null, 2)); +} catch (error) { + console.log(JSON.stringify({ passed: false, error: String(error), bytesPerStore, batchSize, batches, metrics }, null, 2)); + process.exitCode = 1; +} finally { delete globalThis.w137ReadId; rmSync(dir, {recursive:true,force:true}); } diff --git a/.github/openclaw/qualification/engine-limit-probe.cpp b/.github/openclaw/qualification/engine-limit-probe.cpp new file mode 100644 index 0000000000000..807fa542b6cd6 --- /dev/null +++ b/.github/openclaw/qualification/engine-limit-probe.cpp @@ -0,0 +1,56 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void evaluate(JSGlobalContextRef context, const char* text) +{ + auto source = JSStringCreateWithUTF8CString(text); + JSValueRef exception = nullptr; + JSEvaluateScript(context, source, nullptr, nullptr, 1, &exception); + JSStringRelease(source); + if (exception) { + std::fprintf(stderr, "probe evaluation failed\n"); + std::exit(2); + } +} + +static bool probe(const char* name, const char* source, size_t heapLimit, size_t nursery, bool expected) +{ + auto group = JSContextGroupCreate(); + auto context = JSGlobalContextCreateInGroup(group, nullptr); + auto& vm = toJS(context)->vm(); + bool exceeded; + { + JSC::JSLockHolder lock(vm); + vm.heap.collectNow(JSC::Sync, JSC::CollectionScope::Full); + vm.heap.setWorkerHeapLimits(heapLimit, nursery); + evaluate(context, source); + vm.heap.collectNow(JSC::Sync, JSC::CollectionScope::Full); + exceeded = vm.heap.heapLimitExceeded(); + std::printf("%s managed=%zu exceeded=%d expected=%d\n", name, + vm.heap.sizeAfterLastFullCollectionExcludingArrayBuffers(), exceeded, expected); + } + JSGlobalContextRelease(context); + JSContextGroupRelease(group); + return exceeded == expected; +} + +int main() +{ + constexpr size_t MB = 1024 * 1024; + bool passed = true; + passed &= probe("managed", "globalThis.held = new Array(8*1024*1024).fill(7)", 32*MB, 4*MB, true); + passed &= probe("oversize", "globalThis.held = new Uint8Array(128*1024*1024).fill(7)", 32*MB, 4*MB, false); + passed &= probe("fast", "globalThis.held=[];for(let i=0;i<65536;i++)held.push(new Uint8Array(2048).fill(7))", 32*MB, 4*MB, false); + passed &= probe("arraybuffer", "globalThis.held = new Uint8Array(new ArrayBuffer(128*1024*1024)).fill(7)", 32*MB, 4*MB, false); + passed &= probe("mixed", "globalThis.held=[new Uint8Array(128*1024*1024).fill(7),new Array(8*1024*1024).fill(7)]", 32*MB, 4*MB, true); + passed &= probe("unlimited-other-vm", "globalThis.held = new Array(8*1024*1024).fill(7)", 0, 0, false); + passed &= probe("young-only", "globalThis.held = new Array(8*1024*1024).fill(7)", 0, 4*MB, false); + return passed ? 0 : 1; +} diff --git a/.github/openclaw/qualification/patches/002-upstream-mimalloc.patch b/.github/openclaw/qualification/patches/002-upstream-mimalloc.patch new file mode 100644 index 0000000000000..51dfc60ff992c --- /dev/null +++ b/.github/openclaw/qualification/patches/002-upstream-mimalloc.patch @@ -0,0 +1,13 @@ +diff --git a/scripts/build/deps/mimalloc.ts b/scripts/build/deps/mimalloc.ts +index 2984efa9..76238c1f 100644 +--- a/scripts/build/deps/mimalloc.ts ++++ b/scripts/build/deps/mimalloc.ts +@@ -12,7 +12,7 @@ + + import type { Dependency, DirectBuild } from "../source.ts"; + +-const MIMALLOC_COMMIT = "eab09015a5850ae18fc43ccfaa5bbe8272992314"; ++const MIMALLOC_COMMIT = "7a828c116d96bb9a66f6c2e3417c739cbbfdb23c"; + + export const mimalloc: Dependency = { + name: "mimalloc", diff --git a/.github/openclaw/qualification/patches/009-allocation-sampling.patch b/.github/openclaw/qualification/patches/009-allocation-sampling.patch new file mode 100644 index 0000000000000..82921574e518e --- /dev/null +++ b/.github/openclaw/qualification/patches/009-allocation-sampling.patch @@ -0,0 +1,469 @@ +diff --git a/CHANGELOG.md b/CHANGELOG.md +index 29d1cea0..fc2372fa 100644 +--- a/CHANGELOG.md ++++ b/CHANGELOG.md +@@ -167,3 +167,5 @@ + - Validate `net`, `http`, and `https` listen ports synchronously before binding, preserving asynchronous bind errors and Node-compatible numeric string parsing. Adapts string routing from [oven-sh/bun#34083](https://github.com/oven-sh/bun/pull/34083). Thanks @robobun! + + - Match Node process property descriptors, including lazy `argv`/`execArgv` data properties and descriptor replacement in native argument readers. Adapts [oven-sh/bun#34229](https://github.com/oven-sh/bun/pull/34229) and [oven-sh/bun#44356](https://github.com/oven-sh/bun/pull/44356). Thanks @robobun! ++ ++- Support real byte-based `node:inspector` HeapProfiler allocation sampling with allocation-site trees, live profiles, and major/minor GC inclusion flags (requires the OpenClaw WebKit allocation sampler). +diff --git a/docs/runtime/debugger.mdx b/docs/runtime/debugger.mdx +index d83684a0..b2e009d1 100644 +--- a/docs/runtime/debugger.mdx ++++ b/docs/runtime/debugger.mdx +@@ -322,3 +322,25 @@ Error: here! + Error: here! + at /path/to/index.ts:18 + ``` ++ ++## In-process allocation sampling ++ ++`node:inspector` and `node:inspector/promises` support `HeapProfiler.startSampling`, ++`HeapProfiler.getSamplingProfile`, and `HeapProfiler.stopSampling`. A local session ++samples its own JavaScriptCore VM, including in a worker. The profile contains a ++`head` allocation-site tree with estimated allocated bytes in `selfSize`, and ++`samples` with `nodeId`, `size`, and unique `ordinal` fields. ++ ++`samplingInterval` is the mean byte interval and defaults to 32,768. The profiler ++samples JavaScriptCore cells, newly reported string backing storage, and GC-managed ++auxiliary storage; other native allocations and external buffer storage are outside ++this measurement. Engine object layouts ++and sampling randomness affect the estimates. ++ ++Getting or stopping a profile synchronously runs full garbage collection, matching Node. ++By default, collected allocations disappear from the profile. Set ++`includeObjectsCollectedByMajorGC` and `includeObjectsCollectedByMinorGC` to `true` ++to measure cumulative allocations across both collection types. Sampling does not ++retain the sampled objects. A repeated start keeps the current profile. Stop, ++`HeapProfiler.disable`, or disconnect releases sampling started by that session. ++Sampling through a worker's `connectToMainThread()` is not supported. +diff --git a/docs/runtime/nodejs-compat.mdx b/docs/runtime/nodejs-compat.mdx +index 0274c5b4..15f42e80 100644 +--- a/docs/runtime/nodejs-compat.mdx ++++ b/docs/runtime/nodejs-compat.mdx +@@ -294,7 +294,7 @@ Writes also reach the shared environment when value coercion creates the first w + + ### [`node:inspector`](https://nodejs.org/api/inspector.html) + +-🟑 Partially implemented. `Session` supports the `Profiler` domain (including precise coverage), `Runtime.enable` and `NodeTracing`, from both `node:inspector` and `node:inspector/promises`. `HeapProfiler.enable`, `HeapProfiler.disable`, and `HeapProfiler.collectGarbage` work in main-thread and worker sessions; collection runs after the current JavaScript job and completes before its callback. Heap allocation sampling and garbage collection through a worker's `connectToMainThread()` session are not implemented. After `open()`, `Session` also forwards `Debugger` configuration commands such as `Debugger.enable` and `Debugger.setBreakpointByUrl` to the inspector server. Their results, such as `breakpointId`, are not returned. Other `Session` commands such as `Runtime.evaluate` are not implemented. `open()`, `url()`, `close()` and `waitForDebugger()` are implemented. `open()` serves the `Debugger` and `Runtime` domains and throws in workers. Missing `Network`. ++🟑 Partially implemented. `Session` supports the `Profiler` domain (including precise coverage), `Runtime.enable` and `NodeTracing`, from both `node:inspector` and `node:inspector/promises`. `HeapProfiler.enable`, `HeapProfiler.disable`, and `HeapProfiler.collectGarbage` work in main-thread and worker sessions; collection runs after the current JavaScript job and completes before its callback. `HeapProfiler.startSampling`, `getSamplingProfile`, and `stopSampling` provide byte-based allocation-site profiles, including separate major/minor GC inclusion flags, through local main-thread and worker sessions. Sampling and garbage collection through a worker's `connectToMainThread()` session are not implemented. After `open()`, `Session` also forwards `Debugger` configuration commands such as `Debugger.enable` and `Debugger.setBreakpointByUrl` to the inspector server. Their results, such as `breakpointId`, are not returned. Other `Session` commands such as `Runtime.evaluate` are not implemented. `open()`, `url()`, `close()` and `waitForDebugger()` are implemented. `open()` serves the `Debugger` and `Runtime` domains and throws in workers. Missing `Network`. + + ### [`node:repl`](https://nodejs.org/api/repl.html) + +diff --git a/src/js/node/inspector.ts b/src/js/node/inspector.ts +index 392df660..27efa612 100644 +--- a/src/js/node/inspector.ts ++++ b/src/js/node/inspector.ts +@@ -25,6 +25,13 @@ const startPreciseCoverage = $newCppFunction("JSInspectorProfiler.cpp", "jsFunct + const stopPreciseCoverage = $newCppFunction("JSInspectorProfiler.cpp", "jsFunction_stopPreciseCoverage", 0); + const collectPreciseCoverage = $newCppFunction("JSInspectorProfiler.cpp", "jsFunction_collectPreciseCoverage", 0); + const collectInspectorGarbage = $newCppFunction("JSInspectorProfiler.cpp", "jsFunction_collectInspectorGarbage", 0); ++const startAllocationSampling = $newCppFunction("JSInspectorProfiler.cpp", "jsFunction_startAllocationSampling", 3); ++const stopAllocationSampling = $newCppFunction("JSInspectorProfiler.cpp", "jsFunction_stopAllocationSampling", 0); ++const getAllocationSamplingProfile = $newCppFunction( ++ "JSInspectorProfiler.cpp", ++ "jsFunction_getAllocationSamplingProfile", ++ 0, ++); + + // Native bindings for inspector.open(): they start Bun's debugger thread with a + // WebSocket server that speaks the V8 Chrome DevTools Protocol (see +@@ -428,6 +435,7 @@ class Session extends EventEmitter { + #preciseCoverageCallCount = false; + #preciseCoverageDetailed = false; + #forwardedDebugger = false; ++ #samplingAllocations = false; + #pendingCollections: Set<{ callback: (err: Error | null, result?: any) => void }> = new SafeSet(); + // Baseline for delta semantics: takePreciseCoverage must reset counters, but + // JSC has no counter-reset API, so subtract the previous take instead. +@@ -470,6 +478,10 @@ class Session extends EventEmitter { + + disconnect() { + if (!this.#connected) return; ++ if (this.#samplingAllocations) { ++ stopAllocationSampling(); ++ this.#samplingAllocations = false; ++ } + if (isCPUProfilerRunning()) stopCPUProfiler(); + if (this.#preciseCoverageEnabled) { + stopPreciseCoverage(); +@@ -514,10 +526,30 @@ class Session extends EventEmitter { + } + + if (method === "HeapProfiler.enable" || method === "HeapProfiler.disable") { ++ if (method === "HeapProfiler.disable" && this.#samplingAllocations) { ++ stopAllocationSampling(); ++ this.#samplingAllocations = false; ++ } + if (callback) this.#heapCallback(callback, {}); + return; + } + ++ if ( ++ !this.#connectedToMainThread && ++ (method === "HeapProfiler.startSampling" || ++ method === "HeapProfiler.stopSampling" || ++ method === "HeapProfiler.getSamplingProfile") ++ ) { ++ const result = this.#allocationSampling(method, params as Record | undefined); ++ if (callback) ++ this.#heapCallback( ++ callback, ++ result instanceof Error ? undefined : result, ++ result instanceof Error ? result : null, ++ ); ++ return; ++ } ++ + if (method === "HeapProfiler.collectGarbage" && !this.#connectedToMainThread) { + const collection = collectInspectorGarbage(); + if (callback) { +@@ -558,14 +590,44 @@ class Session extends EventEmitter { + } + } + +- #heapCallback(callback: (err: Error | null, result?: any) => void, result: any) { ++ #heapCallback(callback: (err: Error | null, result?: any) => void, result: any, error: Error | null = null) { + try { +- callback(null, result); ++ callback(error, result); + } catch (error) { + process.emitWarning(error as Error); + } + } + ++ #allocationSampling(method: string, params?: Record): object | Error { ++ if (method === "HeapProfiler.startSampling") { ++ const interval = params?.samplingInterval === undefined ? 32768 : params.samplingInterval; ++ const major = ++ params?.includeObjectsCollectedByMajorGC === undefined ? false : params.includeObjectsCollectedByMajorGC; ++ const minor = ++ params?.includeObjectsCollectedByMinorGC === undefined ? false : params.includeObjectsCollectedByMinorGC; ++ if ( ++ typeof interval !== "number" || ++ !Number.isFinite(interval) || ++ typeof major !== "boolean" || ++ typeof minor !== "boolean" ++ ) { ++ return $ERR_INSPECTOR_COMMAND("-32602: Invalid parameters"); ++ } ++ // V8 truncates the byte interval; sub-byte intervals cannot sample safely. ++ if (interval < 1) return $ERR_INSPECTOR_COMMAND("-32000: Invalid sampling interval"); ++ startAllocationSampling(Math.floor(interval), major, minor); ++ this.#samplingAllocations = true; ++ return {}; ++ } ++ const raw = getAllocationSamplingProfile(); ++ if (raw === null) return $ERR_INSPECTOR_COMMAND("-32000: V8 sampling heap profiler was not started."); ++ if (method === "HeapProfiler.stopSampling") { ++ stopAllocationSampling(); ++ this.#samplingAllocations = false; ++ } ++ return { profile: JSON.parse(raw) }; ++ } ++ + #handleMethod(method: string, params?: object): any { + switch (method) { + case "Runtime.enable": +diff --git a/src/jsc/bindings/JSInspectorProfiler.cpp b/src/jsc/bindings/JSInspectorProfiler.cpp +index 78ca21c9..35b3306f 100644 +--- a/src/jsc/bindings/JSInspectorProfiler.cpp ++++ b/src/jsc/bindings/JSInspectorProfiler.cpp +@@ -12,6 +12,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -22,6 +23,37 @@ using namespace JSC; + + extern "C" size_t Bun__gc(void* vm, bool sync); + ++JSC_DECLARE_HOST_FUNCTION(jsFunction_startAllocationSampling); ++JSC_DEFINE_HOST_FUNCTION(jsFunction_startAllocationSampling, (JSGlobalObject * globalObject, CallFrame* callFrame)) ++{ ++ globalObject->vm().ensureHeapProfiler().startAllocationSampling( ++ callFrame->argument(0).asNumber(), callFrame->argument(1).asBoolean(), callFrame->argument(2).asBoolean()); ++ return JSValue::encode(jsUndefined()); ++} ++ ++JSC_DECLARE_HOST_FUNCTION(jsFunction_stopAllocationSampling); ++JSC_DEFINE_HOST_FUNCTION(jsFunction_stopAllocationSampling, (JSGlobalObject * globalObject, CallFrame*)) ++{ ++ if (auto* profiler = globalObject->vm().heapProfiler()) ++ profiler->stopAllocationSampling(); ++ return JSValue::encode(jsUndefined()); ++} ++ ++JSC_DECLARE_HOST_FUNCTION(jsFunction_getAllocationSamplingProfile); ++JSC_DEFINE_HOST_FUNCTION(jsFunction_getAllocationSamplingProfile, (JSGlobalObject * globalObject, CallFrame*)) ++{ ++ auto& vm = globalObject->vm(); ++ auto* profiler = vm.heapProfiler(); ++ if (!profiler || !profiler->isSamplingAllocations()) ++ return JSValue::encode(jsNull()); ++ // Node's inspector sets V8's kSamplingForceGC before retrieving a profile. ++ Bun__gc(Bun::vm(globalObject), true); ++ profiler = vm.heapProfiler(); ++ if (!profiler || !profiler->isSamplingAllocations()) ++ return JSValue::encode(jsNull()); ++ return JSValue::encode(jsString(vm, profiler->allocationSamplingProfile())); ++} ++ + JSC_DECLARE_HOST_FUNCTION(jsFunction_collectInspectorGarbage); + JSC_DEFINE_HOST_FUNCTION(jsFunction_collectInspectorGarbage, (JSGlobalObject * globalObject, CallFrame*)) + { +diff --git a/src/jsc/bindings/JSInspectorProfiler.h b/src/jsc/bindings/JSInspectorProfiler.h +index 48f6640a..421077f1 100644 +--- a/src/jsc/bindings/JSInspectorProfiler.h ++++ b/src/jsc/bindings/JSInspectorProfiler.h +@@ -4,6 +4,9 @@ + #include + + JSC_DECLARE_HOST_FUNCTION(jsFunction_collectInspectorGarbage); ++JSC_DECLARE_HOST_FUNCTION(jsFunction_startAllocationSampling); ++JSC_DECLARE_HOST_FUNCTION(jsFunction_stopAllocationSampling); ++JSC_DECLARE_HOST_FUNCTION(jsFunction_getAllocationSamplingProfile); + JSC_DECLARE_HOST_FUNCTION(jsFunction_startCPUProfiler); + JSC_DECLARE_HOST_FUNCTION(jsFunction_stopCPUProfiler); + JSC_DECLARE_HOST_FUNCTION(jsFunction_setCPUSamplingInterval); +diff --git a/test/js/node/inspector/inspector-profiler.test.ts b/test/js/node/inspector/inspector-profiler.test.ts +index 268e362f..71ab6f40 100644 +--- a/test/js/node/inspector/inspector-profiler.test.ts ++++ b/test/js/node/inspector/inspector-profiler.test.ts +@@ -218,6 +218,163 @@ describe("node:inspector", () => { + }); + + describe("HeapProfiler", () => { ++ test("worker allocation sampling is independent of the main VM", async () => { ++ await using proc = Bun.spawn({ ++ cmd: [bunExe(), import.meta.dir + "/inspector-sampling-worker.fixture.cjs"], ++ env: bunEnv, ++ stdout: "pipe", ++ stderr: "pipe", ++ }); ++ const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); ++ expect({ stdout, stderr, exitCode }).toEqual({ ++ stdout: "worker allocation sampling passed\n", ++ stderr: "", ++ exitCode: 0, ++ }); ++ }); ++ ++ test("byte allocation sampling retains collected samples, tracks live samples, and matches Session lifecycle", async () => { ++ await using proc = Bun.spawn({ ++ cmd: [ ++ bunExe(), ++ "--input-type=module", ++ "-e", ++ ` ++import assert from 'node:assert/strict'; ++import { Session } from 'node:inspector/promises'; ++import { Session as CallbackSession } from 'node:inspector'; ++const session = new Session(); ++session.connect(); ++let retained; ++function allocateObjects(count) { ++ retained = Array.from({ length: count }, (_, index) => ({ index, x: index + 1, y: index + 2 })); ++} ++function allocateArrays(count) { ++ retained = Array.from({ length: count }, (_, index) => [index, index + 1, index + 2]); ++} ++function total(node, name) { ++ return (node.callFrame.functionName === name ? node.selfSize : 0) + node.children.reduce((sum, child) => sum + total(child, name), 0); ++} ++function validate(profile) { ++ const nodes = new Map(); ++ let bytes = 0; ++ const walk = node => { ++ assert(!nodes.has(node.id)); ++ nodes.set(node.id, node); ++ assert(node.selfSize >= 0); ++ assert.equal(typeof node.callFrame.functionName, 'string'); ++ assert.equal(typeof node.callFrame.scriptId, 'string'); ++ assert.equal(typeof node.callFrame.url, 'string'); ++ assert(Number.isInteger(node.callFrame.lineNumber)); ++ assert(Number.isInteger(node.callFrame.columnNumber)); ++ bytes += node.selfSize; ++ node.children.forEach(walk); ++ }; ++ walk(profile.head); ++ const ordinals = new Set(); ++ let sampleBytes = 0; ++ for (const sample of profile.samples) { ++ assert(nodes.has(sample.nodeId)); ++ assert(sample.size > 0); ++ assert(Number.isInteger(sample.ordinal) && sample.ordinal > 0); ++ assert(!ordinals.has(sample.ordinal)); ++ ordinals.add(sample.ordinal); ++ sampleBytes += sample.size; ++ } ++ if (profile.samples.length) { ++ assert(bytes > 0 && sampleBytes > 0); ++ assert(Math.abs(bytes - sampleBytes) < bytes / 4); ++ } ++ return bytes; ++} ++try { ++ await assert.rejects(session.post('HeapProfiler.getSamplingProfile'), { code: 'ERR_INSPECTOR_COMMAND' }); ++ for (let i = 0; i < 30; ++i) { allocateObjects(1000); allocateArrays(1000); } ++ retained = undefined; ++ await session.post('HeapProfiler.collectGarbage'); ++ await session.post('HeapProfiler.startSampling', { ++ samplingInterval: 1024, ++ includeObjectsCollectedByMajorGC: true, ++ includeObjectsCollectedByMinorGC: true, ++ }); ++ // A repeated start keeps existing samples and their ordinals. ++ allocateObjects(20000); ++ const first = (await session.post('HeapProfiler.getSamplingProfile')).profile; ++ assert(validate(first) > 100000); ++ await session.post('HeapProfiler.startSampling', { samplingInterval: 32768 }); ++ allocateArrays(20000); ++ retained = undefined; ++ await session.post('HeapProfiler.collectGarbage'); ++ const last = (await session.post('HeapProfiler.stopSampling')).profile; ++ assert(validate(last) >= validate(first)); ++ const byOrdinal = new Map(last.samples.map(sample => [sample.ordinal, sample])); ++ for (const sample of first.samples) assert.deepEqual(byOrdinal.get(sample.ordinal), sample); ++ assert(total(last.head, 'allocateObjects') + total(last.head, '') > 0); ++ assert(last.samples.length > first.samples.length); ++ await assert.rejects(session.post('HeapProfiler.stopSampling'), { code: 'ERR_INSPECTOR_COMMAND' }); ++ // Default flags discard dead allocations rather than strongly retaining them. ++ await session.post('HeapProfiler.startSampling', { samplingInterval: 512 }); ++ allocateObjects(30000); ++ const before = validate((await session.post('HeapProfiler.getSamplingProfile')).profile); ++ retained = undefined; ++ const liveAfter = validate((await session.post('HeapProfiler.getSamplingProfile')).profile); ++ assert(liveAfter < before / 3, \`live profile did not collect: \${before} -> \${liveAfter}\`); ++ const after = validate((await session.post('HeapProfiler.stopSampling')).profile); ++ assert(before > 100000); ++ assert(after < before / 3, \`dead samples remain: \${before} -> \${after}\`); ++ await session.post('HeapProfiler.startSampling'); ++ await session.post('HeapProfiler.disable'); ++ await assert.rejects(session.post('HeapProfiler.getSamplingProfile'), { code: 'ERR_INSPECTOR_COMMAND' }); ++ await session.post('HeapProfiler.startSampling'); ++ session.disconnect(); ++ session.connect(); ++ await assert.rejects(session.post('HeapProfiler.getSamplingProfile'), { code: 'ERR_INSPECTOR_COMMAND' }); ++} finally { session.disconnect(); } ++const callback = new CallbackSession(); ++callback.connect(); ++try { ++ for (const [params, message] of [ ++ [{ samplingInterval: 0 }, '-32000: Invalid sampling interval'], ++ [{ samplingInterval: -1 }, '-32000: Invalid sampling interval'], ++ [{ samplingInterval: '32' }, '-32602: Invalid parameters'], ++ [{ samplingInterval: null }, '-32602: Invalid parameters'], ++ [{ samplingInterval: NaN }, '-32602: Invalid parameters'], ++ [{ includeObjectsCollectedByMajorGC: 1 }, '-32602: Invalid parameters'], ++ [{ includeObjectsCollectedByMinorGC: null }, '-32602: Invalid parameters'], ++ ]) { ++ let called = false; ++ assert.equal(callback.post('HeapProfiler.startSampling', params, (error, result) => { ++ called = true; ++ assert.equal(error.code, 'ERR_INSPECTOR_COMMAND'); ++ assert.equal(error.message, 'Inspector error ' + message); ++ assert.equal(result, undefined); ++ }), undefined); ++ assert(called, 'callback was not synchronous'); ++ } ++ assert.equal(callback.post('HeapProfiler.startSampling'), undefined); ++ let completed = false; ++ callback.post('HeapProfiler.stopSampling', (error, { profile }) => { ++ assert.equal(error, null); ++ validate(profile); ++ completed = true; ++ }); ++ assert(completed); ++} finally { callback.disconnect(); } ++console.log('allocation sampling contract passed'); ++`, ++ ], ++ env: bunEnv, ++ stdout: "pipe", ++ stderr: "pipe", ++ }); ++ const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); ++ expect({ stdout, stderr, exitCode }).toEqual({ ++ stdout: "allocation sampling contract passed\n", ++ stderr: "", ++ exitCode: 0, ++ }); ++ }); ++ + test("enable and disable complete synchronously and return undefined", () => { + const session = new inspector.Session(); + session.connect(); +diff --git a/test/js/node/inspector/inspector-sampling-worker.fixture.cjs b/test/js/node/inspector/inspector-sampling-worker.fixture.cjs +new file mode 100644 +index 00000000..47ce7558 +--- /dev/null ++++ b/test/js/node/inspector/inspector-sampling-worker.fixture.cjs +@@ -0,0 +1,63 @@ ++const assert = require("node:assert/strict"); ++const { Worker, isMainThread, parentPort } = require("node:worker_threads"); ++const { Session } = require("node:inspector/promises"); ++async function run() { ++ const session = new Session(); ++ session.connect(); ++ try { ++ if (isMainThread) { ++ await session.post("HeapProfiler.startSampling", { samplingInterval: 1024 }); ++ const worker = new Worker(__filename); ++ try { ++ await new Promise((resolve, reject) => { ++ let result; ++ worker.on("error", reject); ++ worker.on("message", async message => { ++ try { ++ if (message === "ready") { ++ await session.post("HeapProfiler.stopSampling"); ++ worker.postMessage("finish"); ++ } else result = message; ++ } catch (error) { ++ reject(error); ++ } ++ }); ++ worker.once("exit", code => { ++ try { ++ assert.equal(code, 0); ++ assert(result?.samples > 0 && result?.bytes > 100000); ++ resolve(); ++ } catch (error) { ++ reject(error); ++ } ++ }); ++ }); ++ } finally { ++ await worker.terminate(); ++ } ++ console.log("worker allocation sampling passed"); ++ } else { ++ await session.post("HeapProfiler.startSampling", { ++ samplingInterval: 1024, ++ includeObjectsCollectedByMajorGC: true, ++ includeObjectsCollectedByMinorGC: true, ++ }); ++ parentPort.postMessage("ready"); ++ await new Promise(resolve => parentPort.once("message", resolve)); ++ let retained = Array.from({ length: 20000 }, (_, i) => ({ i, a: i + 1, b: i + 2 })); ++ assert.equal(retained.length, 20000); ++ retained = undefined; ++ await session.post("HeapProfiler.collectGarbage"); ++ const { profile } = await session.post("HeapProfiler.stopSampling"); ++ const sum = node => node.selfSize + node.children.reduce((n, child) => n + sum(child), 0); ++ parentPort.postMessage({ samples: profile.samples.length, bytes: sum(profile.head) }); ++ parentPort.close(); ++ } ++ } finally { ++ session.disconnect(); ++ } ++} ++run().catch(error => { ++ console.error(error); ++ process.exitCode = 1; ++}); diff --git a/.github/openclaw/qualification/patches/010-als-regressions.patch b/.github/openclaw/qualification/patches/010-als-regressions.patch new file mode 100644 index 0000000000000..84a2d518bfce4 --- /dev/null +++ b/.github/openclaw/qualification/patches/010-als-regressions.patch @@ -0,0 +1,234 @@ +diff --git a/test/js/node/async_hooks/AsyncLocalStorage.test.ts b/test/js/node/async_hooks/AsyncLocalStorage.test.ts +--- a/test/js/node/async_hooks/AsyncLocalStorage.test.ts ++++ b/test/js/node/async_hooks/AsyncLocalStorage.test.ts +@@ -1,7 +1,7 @@ + import { AsyncLocalStorage, AsyncResource } from "async_hooks"; + import { heapStats } from "bun:jsc"; + import { describe, expect, test } from "bun:test"; +-import { bunEnv, bunExe } from "harness"; ++import { bunEnv, bunExe, tempDir } from "harness"; + import http2 from "http2"; + + describe("AsyncLocalStorage", () => { +@@ -1801,3 +1801,32 @@ + expect(references.map(reference => reference.deref())).toEqual([undefined, undefined, undefined]); + expect(handles.map(handle => handle._destroyed)).toEqual([true, true, true]); + }); ++ ++ ++describe("module loading async context", () => { ++ const variants = [ ++ "static", "dynamic", "nested", "tla", "tla-dependency", "cjs", "cjs-direct", ++ "concurrent-distinct", "concurrent-shared", "concurrent-shared-reverse", ++ "concurrent-overlap", "cached", "unscoped", "throw", ++ ]; ++ for (const hooks of ["plugin", "native-hooks"]) { ++ test.each(variants)(`${hooks}: %s`, async variant => { ++ using dir = tempDir("module-loading-async-context", {}); ++ await using proc = Bun.spawn({ ++ cmd: [bunExe(), import.meta.dir + "/fixtures/module-context.fixture.mjs", variant, ...(hooks === "native-hooks" ? ["--native-hooks"] : [])], ++ env: { ...bunEnv, W136_FIXTURE_ROOT: String(dir) }, ++ stdout: "pipe", ++ stderr: "pipe", ++ }); ++ const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); ++ const result = JSON.parse(stdout); ++ expect({ variant: result.variant, failures: result.failures, error: result.error, stderr, exitCode }).toEqual({ ++ variant, ++ failures: [], ++ error: undefined, ++ stderr: "", ++ exitCode: 0, ++ }); ++ }); ++ } ++}); +diff --git a/test/js/node/async_hooks/fixtures/module-context.fixture.mjs b/test/js/node/async_hooks/fixtures/module-context.fixture.mjs +new file mode 100644 +--- /dev/null ++++ b/test/js/node/async_hooks/fixtures/module-context.fixture.mjs +@@ -0,0 +1,184 @@ ++import { AsyncLocalStorage } from 'node:async_hooks'; ++import { mkdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs'; ++import { createRequire, registerHooks } from 'node:module'; ++import path from 'node:path'; ++import { fileURLToPath, pathToFileURL } from 'node:url'; ++import { spawnSync } from 'node:child_process'; ++ ++const cases = ['static', 'dynamic', 'nested', 'tla', 'tla-dependency', 'cjs', 'cjs-direct', 'concurrent-distinct', 'concurrent-shared', 'concurrent-shared-reverse', 'concurrent-overlap', 'cached', 'unscoped', 'throw']; ++const variant = process.argv[2] === 'matrix' ? undefined : process.argv[2]; ++const nativeHooks = process.argv.includes('--native-hooks'); ++if (!variant) { ++ const results = cases.map(name => { ++ const child = spawnSync(process.execPath, [fileURLToPath(import.meta.url), name, ...process.argv.slice(3)], { encoding: 'utf8', timeout: 30000 }); ++ let result; ++ try { result = JSON.parse(child.stdout); } ++ catch { result = { error: 'Child did not emit JSON', stdout: child.stdout, spawnError: child.error?.message }; } ++ return { variant: name, exitCode: child.status, signal: child.signal, stderr: child.stderr, ...result }; ++ }); ++ await new Promise(resolve => process.stdout.write(JSON.stringify({ runtime: process.version, bun: process.versions.bun, results }, null, 2) + '\n', resolve)); ++ process.exit(results.every(row => row.exitCode === 0) ? 0 : 1); ++} ++if (!cases.includes(variant)) throw new Error('Unknown variant: ' + variant); ++ ++const root = realpathSync(process.env.W136_FIXTURE_ROOT); ++const als = new AsyncLocalStorage(); ++const trace = []; ++const record = (stage, file, parent) => trace.push({ stage, file, store: als.getStore() ?? null, ...(parent ? { parent } : {}) }); ++globalThis.w136Record = record; ++globalThis.w136Store = () => als.getStore() ?? null; ++const write = (file, source) => { ++ const name = path.join(root, file); ++ mkdirSync(path.dirname(name), { recursive: true }); ++ writeFileSync(name, source); ++}; ++const body = file => `globalThis.w136Record('body', ${JSON.stringify(file)}); export const store = globalThis.w136Store();`; ++const normalize = file => file.replace(pathToFileURL(root).href + '/', '').replace(root + '/', ''); ++const interesting = file => file.startsWith(root) || file.startsWith(pathToFileURL(root).href) || file.startsWith('./'); ++const importFile = file => import(pathToFileURL(path.join(root, file)).href); ++let hook; ++let values; ++let error; ++const failures = []; ++try { ++ write('package.json', '{"type":"module"}'); ++ als.run('REGISTRATION', () => { ++ if (process.versions.bun && !nativeHooks) { ++ Bun.plugin({ name: 'w136-observe-module-context', setup(build) { ++ build.onResolve({ filter: /.*/, namespace: 'file' }, ({ path: file, importer }) => { ++ if (interesting(file)) record('resolve', normalize(file), importer && normalize(importer)); ++ }); ++ build.onLoad({ filter: /\.mjs$/, namespace: 'file' }, ({ path: file }) => { ++ if (interesting(file)) record('load', normalize(file)); ++ return { contents: readFileSync(file, 'utf8'), loader: 'js' }; ++ }); ++ } }); ++ } else { ++ hook = registerHooks({ ++ resolve(file, context, next) { ++ if (interesting(file)) record('resolve', normalize(file), context.parentURL && normalize(context.parentURL)); ++ return next(file, context); ++ }, ++ load(file, context, next) { ++ if (interesting(file)) record('load', normalize(file)); ++ return next(file, context); ++ }, ++ }); ++ } ++ }); ++ write('leaf.mjs', body('leaf')); ++ write('nested.mjs', body('nested')); ++ let entry = body('entry'); ++ if (variant === 'static' || variant === 'unscoped' || variant === 'cached') ++ entry = `import './leaf.mjs'; ${entry}`; ++ if (variant === 'dynamic' || variant === 'nested') { ++ if (variant === 'nested') write('leaf.mjs', `${body('leaf')} await import('./nested.mjs'); globalThis.w136Record('after-nested', 'leaf');`); ++ entry = `${entry} await import('./leaf.mjs'); globalThis.w136Record('after-dynamic', 'entry');`; ++ } ++ if (variant === 'tla') entry = `${entry} await Promise.resolve(); globalThis.w136Record('after-await', 'entry'); await new Promise(r => setImmediate(r)); globalThis.w136Record('after-immediate', 'entry');`; ++ if (variant === 'tla-dependency') { ++ write('leaf.mjs', `${body('leaf')} await new Promise(r => setImmediate(r)); globalThis.w136Record('after-await', 'leaf');`); ++ entry = `import './leaf.mjs'; ${entry}`; ++ } ++ if (variant === 'cjs' || variant === 'cjs-direct') { ++ write('leaf.cjs', `globalThis.w136Record('body', 'cjs'); module.exports = globalThis.w136Store();`); ++ entry = `import {createRequire} from 'node:module'; ${entry} export const cjsStore = createRequire(import.meta.url)('./leaf.cjs');`; ++ } ++ if (variant === 'throw') entry = `${entry} throw new Error('expected-w136');`; ++ write('entry.mjs', entry); ++ const run = (store, file = 'entry.mjs') => als.run(store, async () => { ++ record('caller-before', file); ++ try { ++ const module = await importFile(file); ++ record('caller-after', file); ++ return { store: module.store, cjsStore: module.cjsStore }; ++ } catch (e) { ++ record('caller-catch', file); ++ if (variant !== 'throw' || e.message !== 'expected-w136') throw e; ++ return { error: e.message }; ++ } ++ }); ++ if (variant === 'cjs-direct') { ++ values = [als.run('A', () => createRequire(import.meta.url)(path.join(root, 'leaf.cjs')))]; ++ } else if (variant === 'concurrent-distinct') { ++ for (const name of ['a', 'b']) { ++ write(`${name}/leaf.mjs`, `${body(name + '/leaf')} await new Promise(r => setImmediate(r)); globalThis.w136Record('after-await', '${name}/leaf');`); ++ write(`${name}/entry.mjs`, `import './leaf.mjs'; ${body(name + '/entry')} await import('./nested.mjs');`); ++ write(`${name}/nested.mjs`, body(name + '/nested')); ++ } ++ values = await Promise.all([run('A', 'a/entry.mjs'), run('B', 'b/entry.mjs')]); ++ } else if (variant.startsWith('concurrent-shared') || variant === 'concurrent-overlap') { ++ let release; ++ let started; ++ globalThis.w136Gate = new Promise(r => { release = r; }); ++ const ready = new Promise(r => { started = r; }); ++ globalThis.w136Started = started; ++ write('leaf.mjs', `${body('leaf')} globalThis.w136Started(); await globalThis.w136Gate; globalThis.w136Record('after-await', 'leaf');`); ++ write('entry.mjs', `import './leaf.mjs'; ${body('entry')}`); ++ let otherStarted; ++ const otherReady = new Promise(r => { otherStarted = r; }); ++ globalThis.w136OtherStarted = otherStarted; ++ write('ready.mjs', `${body('ready')} globalThis.w136OtherStarted();`); ++ write('other.mjs', `import './leaf.mjs'; import './ready.mjs'; ${body('other')}`); ++ const stores = variant.endsWith('reverse') ? ['B', 'A'] : ['A', 'B']; ++ const first = run(stores[0]); ++ await Promise.race([ready, first]); ++ const second = run(stores[1], variant === 'concurrent-overlap' ? 'other.mjs' : 'entry.mjs'); ++ if (variant === 'concurrent-overlap') await Promise.race([otherReady, second]); ++ release(); ++ values = await Promise.all([first, second]); ++ } else if (variant === 'cached') { ++ values = [await run('A'), await run('B')]; ++ } else { ++ values = [await run(variant === 'unscoped' ? undefined : 'A')]; ++ } ++ record('outside', 'runner'); ++ const first = variant === 'unscoped' ? null : variant.endsWith('reverse') ? 'B' : 'A'; ++ let rootResolves = 0; ++ let callerBefore = 0; ++ let callerAfter = 0; ++ const repeated = variant.startsWith('concurrent-shared') || variant === 'cached'; ++ for (const event of trace) { ++ let expected = first; ++ if (event.stage === 'outside') expected = null; ++ else if (variant === 'concurrent-distinct') expected = (event.file.startsWith('b/') || event.parent?.startsWith('b/')) ? 'B' : 'A'; ++ else if (variant === 'concurrent-overlap' && (['other.mjs', 'ready', 'ready.mjs'].includes(event.file) || event.parent === 'other.mjs')) expected = 'B'; ++ else if (repeated) { ++ if (event.stage === 'resolve' && event.file === 'entry.mjs' && rootResolves++ > 0) expected = first === 'A' ? 'B' : 'A'; ++ if (event.stage === 'caller-before' && callerBefore++ > 0) expected = first === 'A' ? 'B' : 'A'; ++ if (event.stage === 'caller-after' && callerAfter++ > 0) expected = first === 'A' ? 'B' : 'A'; ++ } ++ if (event.store !== expected) failures.push({ ...event, expected }); ++ } ++ const bodyFiles = variant === 'concurrent-distinct' ? ['a/leaf', 'a/entry', 'a/nested', 'b/leaf', 'b/entry', 'b/nested'] ++ : variant === 'cjs-direct' ? ['cjs'] ++ : variant === 'cjs' ? ['entry', 'cjs'] ++ : variant === 'concurrent-overlap' ? ['leaf', 'entry', 'ready', 'other'] ++ : variant === 'nested' ? ['entry', 'leaf', 'nested'] ++ : ['tla', 'throw'].includes(variant) ? ['entry'] : ['leaf', 'entry']; ++ for (const file of bodyFiles) { ++ const count = trace.filter(e => e.stage === 'body' && e.file === file).length; ++ if (count !== 1) failures.push({ stage: 'body-count', file, count, expected: 1 }); ++ } ++ for (const stage of ['resolve', 'load']) { ++ for (const file of bodyFiles.map(file => file === 'cjs' ? 'leaf.cjs' : file + '.mjs')) { ++ // Keep require(CJS) native; the plugin's JS source loader changes its format. ++ if (stage === 'load' && file.endsWith('.cjs') && process.versions.bun && !nativeHooks) continue; ++ const seen = trace.some(event => { ++ if (event.stage !== stage) return false; ++ const resolved = event.file.startsWith('./') ++ ? path.posix.normalize(path.posix.join(path.posix.dirname(event.parent ?? ''), event.file)) ++ : event.file; ++ return resolved === file; ++ }); ++ if (!seen) failures.push({ stage: 'missing-hook', hook: stage, file }); ++ } ++ } ++ if (failures.length) process.exitCode = 1; ++} catch (e) { ++ error = { name: e.name, message: e.message, stack: e.stack }; ++ process.exitCode = 1; ++} finally { ++ hook?.deregister(); ++ console.log(JSON.stringify({ variant, values, error, failures, trace })); ++} diff --git a/.github/openclaw/qualification/stage-sync-artifact.py b/.github/openclaw/qualification/stage-sync-artifact.py new file mode 100644 index 0000000000000..7df12f77e9d29 --- /dev/null +++ b/.github/openclaw/qualification/stage-sync-artifact.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +"""Use the production resolver and checksum cache for an unpublished candidate.""" +import argparse +import hashlib +import json +import shutil +from pathlib import Path +p=argparse.ArgumentParser() +p.add_argument('archive',type=Path) +p.add_argument('--version',required=True) +p.add_argument('--output',required=True,type=Path) +a=p.parse_args() +with a.archive.open('rb') as f:digest=hashlib.file_digest(f,'sha256').hexdigest() +cache=a.output/'prefetch/by-sha256';cache.mkdir(parents=True,exist_ok=False) +shutil.copyfile(a.archive,cache/digest) +name=a.archive.name +manifest={'version':a.version,'artifacts':{name:{'url':f'https://github.com/openclaw/WebKit/releases/download/autobuild-{a.version}/{name}','sha256':digest}}} +(a.output/'artifacts.json').write_text(json.dumps(manifest,indent=2)+'\n') diff --git a/.github/openclaw/qualification/sync/selected.json b/.github/openclaw/qualification/sync/selected.json new file mode 100644 index 0000000000000..1811b7aae44d8 --- /dev/null +++ b/.github/openclaw/qualification/sync/selected.json @@ -0,0 +1,58 @@ +[ + "test/bundler/compile-node-compile-cache.test.ts", + "test/cli/run/run-process-env.test.ts", + "test/js/bun/dns/dns-interleave.test.ts", + "test/js/bun/dns/dns-prefetch.test.ts", + "test/js/bun/dns/resolve-dns.test.ts", + "test/js/bun/sqlite/sqlite.test.js", + "test/js/bun/util/bun-file-exists.test.js", + "test/js/bun/util/inspect.test.js", + "test/js/bun/util/sleepSync.test.ts", + "test/js/bun/wasm/compile-rss.test.ts", + "test/js/bun/wasm/wasi.test.js", + "test/js/node/buffer.test.js", + "test/js/node/child_process/child-process-exec.test.ts", + "test/js/node/child_process/child-process-stdio.test.js", + "test/js/node/fs/fs.test.ts", + "test/js/node/http/node-http-server-abort-events.test.ts", + "test/js/node/http/node-http-server-close-drain.test.ts", + "test/js/node/http/node-http-server-socket-end-drain.test.ts", + "test/js/node/http/node-http.test.ts", + "test/js/node/module/node-module-module.test.js", + "test/js/node/net/node-net-allowHalfOpen.test.js", + "test/js/node/net/node-net-server.test.ts", + "test/js/node/net/node-net.test.ts", + "test/js/node/tls/fetch-tls-cert.test.ts", + "test/js/node/tls/node-tls-cert.test.ts", + "test/js/node/tls/node-tls-connect-hostname-verification.test.ts", + "test/js/node/tls/node-tls-connect.test.ts", + "test/js/node/tls/node-tls-context.test.ts", + "test/js/node/tls/node-tls-create-secure-context-args.test.ts", + "test/js/node/tls/node-tls-duplex-close-throw-uaf.test.ts", + "test/js/node/tls/node-tls-duplex-end-verify.test.ts", + "test/js/node/tls/node-tls-duplex-write-throw-error-value.test.ts", + "test/js/node/tls/node-tls-ecdh-curve.test.ts", + "test/js/node/tls/node-tls-getpeercert-leak.test.ts", + "test/js/node/tls/node-tls-internals.test.ts", + "test/js/node/tls/node-tls-namedpipes.test.ts", + "test/js/node/tls/node-tls-no-cipher-match-error.test.ts", + "test/js/node/tls/node-tls-raw-end.test.ts", + "test/js/node/tls/node-tls-root-certs-concurrent-init.test.ts", + "test/js/node/tls/node-tls-rootcertificates-immutable.test.ts", + "test/js/node/tls/node-tls-server.test.ts", + "test/js/node/tls/node-tls-socket-allow-half-open-option.test.ts", + "test/js/node/tls/node-tls-upgrade.test.ts", + "test/js/node/tls/node-tls-wrapped-socket-close.test.ts", + "test/js/node/tls/renegotiation.test.ts", + "test/js/node/tls/ssl-ctx-cache.test.ts", + "test/js/node/tls/test-node-extra-ca-certs.test.ts", + "test/js/node/tls/test-system-ca-https.test.ts", + "test/js/node/tls/test-use-system-ca.test.ts", + "test/js/node/tls/tls-connect-socket-churn.test.ts", + "test/js/node/tls/tls-syscall-fault.test.ts", + "test/js/node/worker_threads/worker_threads.test.ts", + "test/js/web/atomics.test.ts", + "test/js/web/fetch/body-clone.test.ts", + "test/js/web/timers/timer-gc-roots.test.ts", + "test/regression/issue/17190.test.ts" +] diff --git a/.github/openclaw/qualification/sync/upstream-sync.patch b/.github/openclaw/qualification/sync/upstream-sync.patch new file mode 100644 index 0000000000000..464fb732ac0ac --- /dev/null +++ b/.github/openclaw/qualification/sync/upstream-sync.patch @@ -0,0 +1,2692 @@ +diff --git a/.github/UPSTREAM_SYNC.md b/.github/UPSTREAM_SYNC.md +index 8e96782e..089ab102 100644 +--- a/.github/UPSTREAM_SYNC.md ++++ b/.github/UPSTREAM_SYNC.md +@@ -90,3 +90,14 @@ A landed sync reaches OpenClaw through a fork release. Dispatching the release + workflow on `automation/sync-upstream` without a tag builds and smoke-tests every + release target as a dry run; the release cadence and security-patch policy are in + [OPENCLAW_RELEASE.md](OPENCLAW_RELEASE.md#upstream-sync-and-security-patches). ++ ++## Pending engine target after the October 2026 sync ++ ++Upstream Bun `1878660bb47a6a87ebe518cea0581ef4bf9b71f8` targets WebKit ++`5718a6ec579b98362ea7276a426deedcc6281ef5` for idle JSC-thread and ++`Atomics.wait` memory release. The fork continues to consume the checksum-pinned ++OpenClaw release `1ee09069fef6bdf26a899b3da06fc48714aa8a4d` from ++`scripts/build/deps/webkit-artifacts.json`. Updating this target requires a ++separately qualified and published OpenClaw WebKit release; upstream artifacts ++are never a fallback. The additional engine-side idle-release behavior is not ++in the currently pinned release. +diff --git a/CHANGELOG.md b/CHANGELOG.md +index cf69a85e..10801b41 100644 +--- a/CHANGELOG.md ++++ b/CHANGELOG.md +@@ -171,3 +171,8 @@ + - Switch the fork’s default WebKit source to immutable openclaw/WebKit releases with a committed SHA-256 manifest, verified extraction, and digest-keyed caches. + + - Allow Proxy objects in VM and main-realm global prototype chains with the pinned OpenClaw WebKit, including jsdom Window prototypes. Ports [oven-sh/bun#42347](https://github.com/oven-sh/bun/pull/42347). Thanks @robobun! ++ ++## Upstream sync β€” October 2026 (second sync) ++ ++- Merge oven-sh/bun through `1878660bb47a6a87ebe518cea0581ef4bf9b71f8`, retaining fork resolver, module-loader, stack, worker and VM fixes and the checksum-pinned OpenClaw WebKit release; record the newer upstream engine target separately. ++- Incorporate upstream TLS handshake shutdown, macOS split-DNS failover, file-body clone, Buffer write validation and mimalloc improvements while preserving fork idle accounting. +diff --git a/packages/bun-usockets/src/crypto/openssl.c b/packages/bun-usockets/src/crypto/openssl.c +index 2b5e6aa1..d39b12aa 100644 +--- a/packages/bun-usockets/src/crypto/openssl.c ++++ b/packages/bun-usockets/src/crypto/openssl.c +@@ -1163,6 +1163,10 @@ void us_socket_set_inline_reject(struct us_socket_t *s) { + SSL_set_verify(s_ssl(s), SSL_VERIFY_PEER, us_inline_reject_verify_callback); + } + ++void us_socket_set_first_flight_before_fin(struct us_socket_t *s) { ++ s->ssl_first_flight_before_fin = 1; ++} ++ + /* Drop the strdup'd passphrase. Called as soon as private-key load completes + * (the only consumer of the passwd_cb), so the secret never outlives ctx + * construction and SSL_CTX_free() is sufficient on every later path. Also +@@ -1778,6 +1782,8 @@ void us_internal_ssl_attach(struct us_socket_t *s, SSL_CTX *ctx, + s->ssl_in_use = 0; + s->ssl_pending_detach = 0; + s->ssl_pending_close_code = 0; ++ s->ssl_first_flight_before_fin = 0; ++ s->ssl_shutdown_after_first_flight = 0; + s->ssl_is_server = is_client ? 0 : 1; + s->ssl_inline_reject = 0; + s->ssl_verify_failed = 0; +@@ -2291,6 +2297,12 @@ struct us_socket_t *us_internal_ssl_on_open(struct us_socket_t *s, int is_client + /* Kick the handshake immediately β€” some peers stall waiting for ClientHello. */ + ssl_set_loop_data(result); + ssl_update_handshake(result, 1); ++ if (ssl_gone(result)) return result; ++ result->ssl_first_flight_before_fin = 0; ++ if (result->ssl_shutdown_after_first_flight) { ++ result->ssl_shutdown_after_first_flight = 0; ++ us_internal_ssl_shutdown(result); ++ } + return result; + } + +@@ -2882,6 +2894,10 @@ int us_internal_ssl_writev(struct us_socket_t *s, const struct us_iovec_t *iov, + + void us_internal_ssl_shutdown(struct us_socket_t *s) { + if (us_socket_is_closed(s) || us_internal_ssl_is_shut_down(s)) return; ++ if (s->ssl_first_flight_before_fin) { ++ s->ssl_shutdown_after_first_flight = 1; ++ return; ++ } + + /* Spilled ciphertext is data the layers above already count as written; + * a FIN/close_notify now would cut it off. Finish the shutdown from the +diff --git a/packages/bun-usockets/src/internal/internal.h b/packages/bun-usockets/src/internal/internal.h +index b580b7d9..af0274a7 100644 +--- a/packages/bun-usockets/src/internal/internal.h ++++ b/packages/bun-usockets/src/internal/internal.h +@@ -364,6 +364,10 @@ struct us_socket_t { + * inside a handshake callback must still RST, not FIN, when it is finally + * performed). */ + unsigned char ssl_pending_close_code : 2; ++ /* us_socket_set_first_flight_before_fin: the first handshake step is still due, and a shutdown waits for it. */ ++ unsigned char ssl_first_flight_before_fin : 1; ++ /* us_internal_ssl_shutdown held its FIN back for that step. */ ++ unsigned char ssl_shutdown_after_first_flight : 1; + /* Consecutive send() failures with an errno that is neither + * would-block/transient nor a known peer-gone error (see + * us_socket_write_check_error). Reset by any send that makes progress. +diff --git a/packages/bun-usockets/src/libusockets.h b/packages/bun-usockets/src/libusockets.h +index 2c4216df..430f8f1a 100644 +--- a/packages/bun-usockets/src/libusockets.h ++++ b/packages/bun-usockets/src/libusockets.h +@@ -389,6 +389,8 @@ void us_socket_start_tls_handshake(us_socket_r s) nonnull_fn_decl; + * server that fails verification. Must run before the handshake is driven + * (on_open, or between adopt_tls and start_tls_handshake). No-op otherwise. */ + void us_socket_set_inline_reject(us_socket_r s) nonnull_fn_decl; ++/* Call it from on_open. A shutdown before the first handshake step sends its FIN after that step. */ ++void us_socket_set_first_flight_before_fin(us_socket_r s) nonnull_fn_decl; + + /* ── Listen ─────────────────────────────────────────────────────────────── + * The listener owns: an embedded group for accepted sockets, the SSL_CTX +diff --git a/scripts/build/deps/mimalloc.ts b/scripts/build/deps/mimalloc.ts +index 2984efa9..76238c1f 100644 +--- a/scripts/build/deps/mimalloc.ts ++++ b/scripts/build/deps/mimalloc.ts +@@ -12,7 +12,7 @@ + + import type { Dependency, DirectBuild } from "../source.ts"; + +-const MIMALLOC_COMMIT = "eab09015a5850ae18fc43ccfaa5bbe8272992314"; ++const MIMALLOC_COMMIT = "7a828c116d96bb9a66f6c2e3417c739cbbfdb23c"; + + export const mimalloc: Dependency = { + name: "mimalloc", +diff --git a/src/js/node/tls.ts b/src/js/node/tls.ts +index f585b344..b5754b3d 100644 +--- a/src/js/node/tls.ts ++++ b/src/js/node/tls.ts +@@ -5,6 +5,7 @@ const Duplex = require("internal/streams/duplex"); + const EventEmitter = require("node:events"); + const addServerName = $newRustFunction("Listener.rs", "jsAddServerName", 3); + const setListenerSecureContext = $newRustFunction("Listener.rs", "jsSetSecureContext", 2); ++const firstFlightBeforeFin = $newRustFunction("runtime/socket/socket.rs", "jsFirstFlightBeforeFin", 1); + const { throwNotImplemented } = require("internal/shared"); + const { idnaToASCII } = require("internal/url"); + const { +@@ -1671,9 +1672,16 @@ function connect(...args) { + if (timeout) { + tlssock.setTimeout(timeout); + } ++ // https://github.com/nodejs/node/blob/v26.3.0/lib/internal/tls/wrap.js#L1795 ++ if (!options.socket) tlssock.once("connect", onConnectStart); + return tlssock.connect(normal); + } + ++// The engine sends the ClientHello when the listeners of 'connect' have run. An end() of theirs must not get ahead of it. ++function onConnectStart() { ++ firstFlightBeforeFin(this._handle); ++} ++ + function getCiphers() { + return getDefaultCiphers().split(":"); + } +diff --git a/src/jsc/bindings/JSBuffer.cpp b/src/jsc/bindings/JSBuffer.cpp +index 95986d0d..29fb49bb 100644 +--- a/src/jsc/bindings/JSBuffer.cpp ++++ b/src/jsc/bindings/JSBuffer.cpp +@@ -2373,11 +2373,20 @@ static JSC::EncodedJSValue jsBufferPrototypeFunction_SliceWithEncoding(JSC::JSGl + return jsBufferToString(lexicalGlobalObject, scope, castedThis, start, end - start, encoding); + } + ++// Node's native writers reject a non-string value and never coerce it (THROW_AND_RETURN_IF_NOT_STRING, src/node_errors.h). ++static JSString* stringArgumentOrThrow(JSC::ThrowScope& scope, JSC::JSGlobalObject* globalObject, JSValue value) ++{ ++ if (value.isString()) [[likely]] ++ return asString(value); ++ Bun::throwError(globalObject, scope, Bun::ErrorCode::ERR_INVALID_ARG_TYPE, "argument must be a string"_s); ++ return nullptr; ++} ++ + // https://github.com/nodejs/node/blob/v26.3.0/lib/internal/buffer.js#L962-L990 + // Only utf8Write/latin1Write/asciiWrite go through this strict JS wrapper in node; + // the other encodings use jsBufferPrototypeFunction_StringWriteWithEncoding below. + template +-static JSC::EncodedJSValue jsBufferPrototypeFunction_writeEncodingBody(JSC::VM& vm, JSC::JSGlobalObject* lexicalGlobalObject, JSArrayBufferView* castedThis, JSString* str, JSValue offsetValue, JSValue lengthValue) ++static JSC::EncodedJSValue jsBufferPrototypeFunction_writeEncodingBody(JSC::VM& vm, JSC::JSGlobalObject* lexicalGlobalObject, JSArrayBufferView* castedThis, JSValue stringValue, JSValue offsetValue, JSValue lengthValue) + { + auto scope = DECLARE_THROW_SCOPE(vm); + +@@ -2438,6 +2447,10 @@ static JSC::EncodedJSValue jsBufferPrototypeFunction_writeEncodingBody(JSC::VM& + maxLength = std::min(byteLength - safeOffset, static_cast(intLength)); + } + ++ // Node's JS wrapper checks the bounds first, so a bounds error wins over a non-string value. ++ JSString* str = stringArgumentOrThrow(scope, lexicalGlobalObject, stringValue); ++ RETURN_IF_EXCEPTION(scope, {}); ++ + RELEASE_AND_RETURN(scope, writeToBuffer(lexicalGlobalObject, castedThis, str, safeOffset, maxLength, encoding)); + } + +@@ -2449,10 +2462,7 @@ static JSC::EncodedJSValue jsBufferPrototypeFunctionWriteWithEncoding(JSC::JSGlo + + auto* castedThis = dynamicDowncast(callFrame->thisValue()); + +- auto arg0 = callFrame->argument(0); +- JSString* text = arg0.toStringOrNull(lexicalGlobalObject); +- RETURN_IF_EXCEPTION(scope, {}); +- ++ JSValue stringValue = callFrame->argument(0); + JSValue offsetValue = callFrame->argument(1); + JSValue lengthValue = callFrame->argument(2); + +@@ -2461,7 +2471,7 @@ static JSC::EncodedJSValue jsBufferPrototypeFunctionWriteWithEncoding(JSC::JSGlo + return {}; + } + +- RELEASE_AND_RETURN(scope, jsBufferPrototypeFunction_writeEncodingBody(vm, lexicalGlobalObject, castedThis, text, offsetValue, lengthValue)); ++ RELEASE_AND_RETURN(scope, jsBufferPrototypeFunction_writeEncodingBody(vm, lexicalGlobalObject, castedThis, stringValue, offsetValue, lengthValue)); + } + + // https://github.com/nodejs/node/blob/v26.3.0/src/node_buffer.cc#L711-L741 +@@ -2484,7 +2494,8 @@ static JSC::EncodedJSValue jsBufferPrototypeFunction_StringWriteWithEncoding(JSC + const JSValue offsetValue = callFrame->argument(1); + const JSValue lengthValue = callFrame->argument(2); + +- JSString* text = strValue.toStringOrNull(lexicalGlobalObject); ++ // The binding rejects the value before it reads offset or length. ++ JSString* text = stringArgumentOrThrow(scope, lexicalGlobalObject, strValue); + RETURN_IF_EXCEPTION(scope, {}); + + size_t offset = 0; +@@ -2492,9 +2503,9 @@ static JSC::EncodedJSValue jsBufferPrototypeFunction_StringWriteWithEncoding(JSC + return {}; + } + +- // toStringOrNull/toNumber only run user-overridable code for object arguments, and ++ // toNumber only runs user-overridable code for an object argument, and + // that code can detach or resize the view, so re-validate only when it could have run. +- if ((strValue.isObject() || offsetValue.isObject()) && castedThis->isDetached()) [[unlikely]] { ++ if (offsetValue.isObject() && castedThis->isDetached()) [[unlikely]] { + throwTypeError(lexicalGlobalObject, scope, "ArrayBufferView is detached"_s); + return {}; + } +diff --git a/src/jsc/webcore_types.rs b/src/jsc/webcore_types.rs +index 591d3e8e..cbb70e72 100644 +--- a/src/jsc/webcore_types.rs ++++ b/src/jsc/webcore_types.rs +@@ -791,6 +791,8 @@ pub mod store { + pub max_size: SizeType, + /// Milliseconds since ECMAScript epoch. + pub last_modified: crate::JSTimeType, ++ /// `st_mode` as a body's `clone()` saw it, kept apart from what the `Bun.file()` sharing this store answers from. ++ pub mode_seen_by_clone: Option, + } + + impl Default for File { +@@ -803,6 +805,7 @@ pub mod store { + seekable: None, + max_size: MAX_SIZE, + last_modified: crate::INIT_TIMESTAMP, ++ mode_seen_by_clone: None, + } + } + } +diff --git a/src/runtime/api/BunObject.rs b/src/runtime/api/BunObject.rs +index 29c34548..cd5203e6 100644 +--- a/src/runtime/api/BunObject.rs ++++ b/src/runtime/api/BunObject.rs +@@ -1025,9 +1025,15 @@ fn sleep_sync(global_object: &JSGlobalObject, callframe: &CallFrame) -> JsResult + ))); + } + ++ // mimalloc's scavenger sweeps this thread's heaps while it sleeps, as it does across the event loop's poll. ++ // SAFETY: nothing allocates or frees on this thread until `mi_on_thread_idle_end` below. ++ let handed_off = unsafe { bun_alloc::mimalloc::mi_on_thread_idle_start() }; + std::thread::sleep(core::time::Duration::from_millis( + u64::try_from(milliseconds).expect("int cast"), + )); ++ if handed_off { ++ bun_alloc::mimalloc::mi_on_thread_idle_end(); ++ } + Ok(JSValue::UNDEFINED) + } + +diff --git a/src/runtime/dns_jsc/dns.rs b/src/runtime/dns_jsc/dns.rs +index 5634a548..b06683e2 100644 +--- a/src/runtime/dns_jsc/dns.rs ++++ b/src/runtime/dns_jsc/dns.rs +@@ -118,7 +118,7 @@ bun_output::declare_scope!(DNSResolver, visible); + const IANA_DNS_PORT: i32 = 53; + + // ────────────────────────────────────────────────────────────────────────── +-// dns_sd (macOS): DNSServiceGetAddrInfo over one shared mDNSResponder connection, no per-lookup threads. ++// dns_sd (macOS): DNSServiceQueryRecord over one shared mDNSResponder connection, no per-lookup threads. + // ────────────────────────────────────────────────────────────────────────── + + #[cfg(target_os = "macos")] +@@ -1209,33 +1209,6 @@ impl GetAddrInfoRequest { + request + } + +- /// Reply callback (inside `DNSServiceProcessResult`): records state; completion happens in `on_readable`. +- /// # Safety +- /// `context` is the registered `*mut GetAddrInfoRequest`; `address`, if non-null, is a valid sockaddr. +- #[cfg(target_os = "macos")] +- pub(crate) unsafe extern "C" fn dns_sd_reply( +- _sd_ref: dns_sd::DNSServiceRef, +- flags: u32, +- _interface_index: u32, +- error_code: i32, +- _hostname: *const c_char, +- address: *const Sockaddr, +- ttl: u32, +- context: *mut c_void, +- ) { +- dns_sd::SharedConnection::note_reply(context); +- // SAFETY: context is the *mut GetAddrInfoRequest passed to start(). +- let this: *mut Self = context.cast(); +- // SAFETY: `this` is the live heap request (JS thread); `address` is valid per dns_sd.h. +- unsafe { +- (*this) +- .backend +- .as_dns_sd_mut() +- .query +- .record_reply(flags, error_code, address, ttl); +- } +- } +- + /// Complete a dns_sd-backed request; `this` is the live heap request, consumed on every path. + #[cfg(target_os = "macos")] + pub(crate) fn complete_dns_sd(this: *mut Self) { +@@ -2779,39 +2752,7 @@ pub(crate) mod internal { + query: dns_sd::QueryState::new(protocol), + }; + } +- let Some(_) = shared.start( +- dns_sd::Inflight::Internal(req), +- protocol, +- host, +- dns_sd_reply, +- req.cast::(), +- ) else { +- return false; +- }; +- +- true +- } +- +- #[cfg(target_os = "macos")] +- unsafe extern "C" fn dns_sd_reply( +- _sd_ref: dns_sd::DNSServiceRef, +- flags: u32, +- _interface_index: u32, +- error_code: i32, +- _hostname: *const c_char, +- address: *const Sockaddr, +- ttl: u32, +- context: *mut c_void, +- ) { +- dns_sd::SharedConnection::note_reply(context); +- let req: *mut Request = context.cast(); +- // SAFETY: `context` is the registered `req` (event-loop thread); `address` is valid per dns_sd.h. +- unsafe { +- (*req) +- .dns_sd +- .query +- .record_reply(flags, error_code, address, ttl) +- }; ++ shared.start(dns_sd::Inflight::Internal(req), protocol, host) + } + + /// Complete an internal request: build an addrinfo chain and reuse `process_results` (happy-eyeballs order). +diff --git a/src/runtime/dns_jsc/dns_sd.rs b/src/runtime/dns_jsc/dns_sd.rs +index a2a7d844..95d0b2a0 100644 +--- a/src/runtime/dns_jsc/dns_sd.rs ++++ b/src/runtime/dns_jsc/dns_sd.rs +@@ -1,9 +1,9 @@ +-//! macOS DNSServiceGetAddrInfo backend: all lookups share one mDNSResponder connection (see dns.rs banner). ++//! macOS DNSServiceQueryRecord backend: all lookups share one mDNSResponder connection (see dns.rs banner). + + use super::*; + use bun_collections::index_sort; + +-pub(crate) type DNSServiceRef = *mut c_void; ++type DNSServiceRef = *mut c_void; + type DNSServiceFlags = u32; + type DNSServiceErrorType = i32; + pub(crate) type DNSServiceProtocol = u32; +@@ -18,18 +18,23 @@ const FLAGS_TIMEOUT: DNSServiceFlags = 0x10000; + pub(crate) const PROTOCOL_IPV4: DNSServiceProtocol = 0x01; + pub(crate) const PROTOCOL_IPV6: DNSServiceProtocol = 0x02; + ++const TYPE_A: u16 = 1; ++const TYPE_AAAA: u16 = 28; ++const CLASS_IN: u16 = 1; ++ + pub(crate) const ERR_NO_ERROR: DNSServiceErrorType = 0; +-pub(crate) const ERR_NO_SUCH_RECORD: DNSServiceErrorType = -65554; + pub(crate) const ERR_TIMEOUT: DNSServiceErrorType = -65568; +-const ERR_DEFUNCT_CONNECTION: DNSServiceErrorType = -65569; + +-type GetAddrInfoReply = unsafe extern "C" fn( ++type QueryRecordReply = unsafe extern "C" fn( + sd_ref: DNSServiceRef, + flags: DNSServiceFlags, + interface_index: u32, + error_code: DNSServiceErrorType, +- hostname: *const c_char, +- address: *const Sockaddr, ++ fullname: *const c_char, ++ rrtype: u16, ++ rrclass: u16, ++ rdlen: u16, ++ rdata: *const c_void, + ttl: u32, + context: *mut c_void, + ); +@@ -40,35 +45,37 @@ unsafe extern "C" { + fn DNSServiceRefSockFD(sd_ref: DNSServiceRef) -> c_int; + fn DNSServiceProcessResult(sd_ref: DNSServiceRef) -> DNSServiceErrorType; + fn DNSServiceRefDeallocate(sd_ref: DNSServiceRef); +- fn DNSServiceGetAddrInfo( ++ fn DNSServiceQueryRecord( + sd_ref: *mut DNSServiceRef, + flags: DNSServiceFlags, + interface_index: u32, +- protocol: DNSServiceProtocol, +- hostname: *const c_char, +- callback: GetAddrInfoReply, ++ fullname: *const c_char, ++ rrtype: u16, ++ rrclass: u16, ++ callback: QueryRecordReply, + context: *mut c_void, + ) -> DNSServiceErrorType; + } + +-/// SPI: `DNSServiceGetAddrInfo` plus the attribute libinfo's getaddrinfo passes. Absent on macOS 12, so resolved at runtime. +-type GetAddrInfoExFn = unsafe extern "C" fn( ++/// `DNSServiceQueryRecord` plus an attribute. Absent on macOS 12, so resolved at runtime. ++type QueryRecordWithAttributeFn = unsafe extern "C" fn( + sd_ref: *mut DNSServiceRef, + flags: DNSServiceFlags, + interface_index: u32, +- protocol: DNSServiceProtocol, +- hostname: *const c_char, ++ fullname: *const c_char, ++ rrtype: u16, ++ rrclass: u16, + attr: *const DNSServiceAttribute, +- callback: GetAddrInfoReply, ++ callback: QueryRecordReply, + context: *mut c_void, + ) -> DNSServiceErrorType; + +-/// `DNSServiceGetAddrInfoEx` with `kDNSServiceAttrAllowFailover` (lets mDNSResponder fail a query over to +-/// scoped/supplemental resolvers, as getaddrinfo does), when this OS has both. +-fn getaddrinfo_ex() -> Option<(GetAddrInfoExFn, *const DNSServiceAttribute)> { ++/// libinfo's getaddrinfo call: only QueryRecord sends `kDNSServiceAttrAllowFailover` to the daemon (`DNSServiceGetAddrInfoEx` drops it). ++fn query_record_allowing_failover() ++-> Option<(QueryRecordWithAttributeFn, *const DNSServiceAttribute)> { + let f = bun_sys::dlsym_with_handle!( +- GetAddrInfoExFn, +- "DNSServiceGetAddrInfoEx", ++ QueryRecordWithAttributeFn, ++ "DNSServiceQueryRecordWithAttribute", + Some(libc::RTLD_DEFAULT) + )?; + let attr = bun_sys::dlsym_with_handle!( +@@ -150,41 +157,75 @@ enum Attempt { + Reissued, + } + ++/// The A and the AAAA subordinate of one lookup; null for a family that was not asked for. ++type FamilyRefs = [DNSServiceRef; 2]; ++ ++fn deallocate_refs(sd_refs: FamilyRefs) { ++ for sd_ref in sd_refs { ++ if !sd_ref.is_null() { ++ // SAFETY: FFI; a non-null entry is a live subordinate that nothing else releases. ++ unsafe { DNSServiceRefDeallocate(sd_ref) }; ++ } ++ } ++} ++ ++/// What dnssd_clientstub's `handle_addrinfo_response` builds: a link-local address is scoped to the interface it was seen on. ++fn address_from_record( ++ rrtype: u16, ++ rdata: &[u8], ++ interface_index: u32, ++) -> Option { ++ match rrtype { ++ TYPE_A => { ++ let ip = Ipv4Addr::from(<[u8; 4]>::try_from(rdata).ok()?); ++ Some(bun_dns::Address::from_ip(ip.into(), 0)) ++ } ++ TYPE_AAAA => { ++ let ip = Ipv6Addr::from(<[u8; 16]>::try_from(rdata).ok()?); ++ let mut address = bun_dns::Address::from_ip(ip.into(), 0); ++ if ip.is_unicast_link_local() { ++ address.set_scope_id(interface_index); ++ } ++ Some(address) ++ } ++ _ => None, ++ } ++} ++ + /// Per-query state shared by the JS `dns.lookup` path and the internal connect path. + pub(crate) struct QueryState { +- pub(crate) sd_ref: DNSServiceRef, ++ sd_refs: FamilyRefs, + pub(crate) results: bun_dns::ResultList, +- /// First hard error (NoSuchRecord/Timeout are per-family negatives, not errors). +- pub(crate) sd_error: DNSServiceErrorType, + /// A family timed out: an unsuppressed reissue would only wait out the timeout again. + saw_timeout: bool, + /// Last reply had `MoreComing` and no other request's reply followed: more is queued daemon-side. + awaiting_more: bool, +- /// Protocol bits with no reply yet; any family-tagged callback clears its bit. ++ /// Protocol bits with no reply yet; an address record or any error for a family's question clears its bit. + pub(crate) pending_proto: DNSServiceProtocol, + stragglers: Stragglers, + attempt: Attempt, + /// Kept so `finish()` can reissue the query for the retry. + hostname: bun::ZBox, +- callback: Option, + } + + impl QueryState { + pub(crate) fn new(protocol: DNSServiceProtocol) -> Self { + Self { +- sd_ref: ptr::null_mut(), ++ sd_refs: [ptr::null_mut(); 2], + results: Default::default(), +- sd_error: 0, + saw_timeout: false, + awaiting_more: false, + pending_proto: protocol, + stragglers: Stragglers::None, + attempt: Attempt::Plain, + hostname: bun::ZBox::from_bytes(b""), +- callback: None, + } + } + ++ fn deallocate_refs(&mut self) { ++ deallocate_refs(core::mem::replace(&mut self.sd_refs, [ptr::null_mut(); 2])); ++ } ++ + /// Back to a fresh in-flight state for the unsuppressed reissue. + pub(crate) fn reset_for_retry(&mut self, protocol: DNSServiceProtocol) { + self.attempt = Attempt::Reissued; +@@ -200,49 +241,40 @@ impl QueryState { + + /// A suppressed query that returned nothing at all gets one unsuppressed retry. + fn should_retry_unsuppressed(&self) -> bool { +- self.attempt == Attempt::Suppressed +- && self.results.is_empty() +- && self.sd_error == 0 +- && !self.saw_timeout ++ self.attempt == Attempt::Suppressed && self.results.is_empty() && !self.saw_timeout + } + +- /// Absorb one callback. SAFETY: `address`, if non-null, is a valid sockaddr (dnssd_clientstub guarantees it). +- pub(crate) unsafe fn record_reply( ++ /// Absorb one callback for the question `sd_ref`. ++ fn record_reply( + &mut self, ++ sd_ref: DNSServiceRef, + flags: DNSServiceFlags, ++ interface_index: u32, + error_code: DNSServiceErrorType, +- address: *const Sockaddr, ++ rrtype: u16, ++ rdata: &[u8], + ttl: u32, + ) { + self.awaiting_more = flags & FLAGS_MORE_COMING != 0; +- // Only PolicyDenied passes a null sockaddr; A/AAAA replies (incl. negatives) are family-tagged. +- if address.is_null() { +- if self.sd_error == 0 { +- self.sd_error = error_code; +- } +- return; ++ // As in libinfo, any error ends only its own family. The daemon's refusals carry no rrtype, so `sd_ref` names the family. ++ if error_code != ERR_NO_ERROR || matches!(rrtype, TYPE_A | TYPE_AAAA) { ++ self.pending_proto &= !if sd_ref == self.sd_refs[0] { ++ PROTOCOL_IPV4 ++ } else { ++ PROTOCOL_IPV6 ++ }; + } +- // SAFETY: caller contract. +- let fam = unsafe { (*address).sa_family } as i32; +- // Any reply retires the family's bit; completeness is tracked by `awaiting_more`. +- self.pending_proto &= !if fam == netc::AF_INET6 { +- PROTOCOL_IPV6 +- } else { +- PROTOCOL_IPV4 +- }; +- if error_code == ERR_NO_ERROR && flags & FLAGS_ADD != 0 { +- self.results.push(GetAddrInfoResult { +- // SAFETY: caller contract. +- address: unsafe { bun_dns::Address::init_posix(address.cast()) }, +- ttl: ttl as i32, +- }); +- } else if error_code == ERR_TIMEOUT { +- self.saw_timeout = true; +- } else if error_code != ERR_NO_ERROR +- && error_code != ERR_NO_SUCH_RECORD +- && self.sd_error == 0 +- { +- self.sd_error = error_code; ++ match error_code { ++ ERR_NO_ERROR if flags & FLAGS_ADD != 0 => { ++ if let Some(address) = address_from_record(rrtype, rdata, interface_index) { ++ self.results.push(GetAddrInfoResult { ++ address, ++ ttl: ttl as i32, ++ }); ++ } ++ } ++ ERR_TIMEOUT => self.saw_timeout = true, ++ _ => {} + } + self.stragglers = match (self.only_stragglers_left(), self.stragglers) { + (false, _) => Stragglers::None, +@@ -258,9 +290,7 @@ impl QueryState { + } + + pub(crate) fn is_ready(&self) -> bool { +- self.sd_error != 0 +- || self.stragglers == Stragglers::GaveUp +- || (self.pending_proto == 0 && !self.awaiting_more) ++ self.stragglers == Stragglers::GaveUp || (self.pending_proto == 0 && !self.awaiting_more) + } + + /// Deadline for giving up on stragglers (a silent second family, or a dangling `MoreComing`). +@@ -295,6 +325,21 @@ impl Inflight { + } + } + ++ /// The `on_reply` that turns `context()` back into this variant. ++ fn reply_callback(&self) -> QueryRecordReply { ++ match *self { ++ Inflight::Jsc(_) => on_reply::, ++ Inflight::Internal(_) => on_reply::, ++ } ++ } ++ ++ fn complete(self) { ++ match self { ++ Inflight::Jsc(r) => GetAddrInfoRequest::complete_dns_sd(r), ++ Inflight::Internal(r) => internal::dns_sd_complete(r), ++ } ++ } ++ + /// SAFETY: the request behind `self` is live (pinned in `inflight`); + /// the `&mut` derives from the stored raw pointer, not from a borrow. + unsafe fn query<'a>(self) -> &'a mut QueryState { +@@ -308,6 +353,41 @@ impl Inflight { + } + } + ++/// Records a reply; `on_readable` completes. SAFETY: `context` is a live request of the `INTERNAL` kind; `rdata` spans `rdlen` bytes. ++unsafe extern "C" fn on_reply( ++ sd_ref: DNSServiceRef, ++ flags: DNSServiceFlags, ++ interface_index: u32, ++ error_code: DNSServiceErrorType, ++ _fullname: *const c_char, ++ rrtype: u16, ++ _rrclass: u16, ++ rdlen: u16, ++ rdata: *const c_void, ++ ttl: u32, ++ context: *mut c_void, ++) { ++ SharedConnection::note_reply(context); ++ let owner = if INTERNAL { ++ Inflight::Internal(context.cast()) ++ } else { ++ Inflight::Jsc(context.cast()) ++ }; ++ // SAFETY: caller contract; the slice is only read before this callback returns. ++ let rdata = unsafe { bun::ffi::slice(rdata.cast::(), rdlen as usize) }; ++ // SAFETY: caller contract; event-loop thread, and no other borrow of the query is live here. ++ let query = unsafe { owner.query() }; ++ query.record_reply( ++ sd_ref, ++ flags, ++ interface_index, ++ error_code, ++ rrtype, ++ rdata, ++ ttl, ++ ); ++} ++ + /// One per event loop: owns the primary `DNSServiceRef` + `FilePoll`; lookups are ShareConnection subordinates. + pub(crate) struct SharedConnection { + main_ref: DNSServiceRef, +@@ -400,76 +480,88 @@ impl SharedConnection { + Self::current() + } + +- /// Start a subordinate query and track it (keeps the process alive); `None` if the daemon refused. ++ /// Start `owner`'s queries and track it (keeps the process alive); `false` if the daemon refused. + pub(crate) fn start( + &mut self, + owner: Inflight, + protocol: DNSServiceProtocol, + hostname: &ZStr, +- callback: GetAddrInfoReply, +- context: *mut c_void, +- ) -> Option { ++ ) -> bool { + let suppress = addrconfig_flags(protocol); +- let sub = self.issue(protocol, suppress, hostname, callback, context)?; ++ let Some(sd_refs) = self.issue(owner, protocol, suppress, hostname) else { ++ return false; ++ }; + if self.inflight.is_empty() { + let ctx = self.ctx; + self.file_poll().enable_keeping_process_alive(ctx); + } + // SAFETY: `owner` is the caller's live request, tracked here until `finish()`. + let q = unsafe { owner.query() }; +- q.sd_ref = sub; ++ q.sd_refs = sd_refs; + q.attempt = if suppress != 0 { + Attempt::Suppressed + } else { + Attempt::Plain + }; + q.hostname = bun::ZBox::from_bytes(hostname.as_bytes()); +- q.callback = Some(callback); + self.inflight.push(owner); +- Some(sub) ++ true + } + ++ /// One query per family in `protocol`, as libinfo does; `None`, with none left running, if the daemon refused one. + fn issue( + &mut self, ++ owner: Inflight, + protocol: DNSServiceProtocol, + suppress: DNSServiceFlags, + hostname: &ZStr, +- callback: GetAddrInfoReply, +- context: *mut c_void, +- ) -> Option { +- // ShareConnection requires `sub` to start as a copy of the primary ref. +- let mut sub: DNSServiceRef = self.main_ref; ++ ) -> Option { + let flags = FLAGS_SHARE_CONNECTION | FLAGS_TIMEOUT | FLAGS_RETURN_INTERMEDIATES | suppress; +- let hostname = hostname.as_ptr().cast::(); +- // SAFETY: FFI; `hostname` is NUL-terminated (copied by dns_sd); `context` is only stored. +- let err = unsafe { +- match getaddrinfo_ex() { +- Some((ex, attr)) => ex( +- &raw mut sub, +- flags, +- 0, +- protocol, +- hostname, +- attr, +- callback, +- context, +- ), +- None => DNSServiceGetAddrInfo( +- &raw mut sub, +- flags, +- 0, +- protocol, +- hostname, +- callback, +- context, +- ), ++ // Sent as given: the daemon tries search domains only for a name with no trailing dot. ++ let name = hostname.as_ptr().cast::(); ++ let (callback, context) = (owner.reply_callback(), owner.context()); ++ let mut sd_refs: FamilyRefs = [ptr::null_mut(); 2]; ++ let families = [(PROTOCOL_IPV4, TYPE_A), (PROTOCOL_IPV6, TYPE_AAAA)]; ++ for (i, (family, rrtype)) in families.into_iter().enumerate() { ++ if protocol & family == 0 { ++ continue; + } +- }; +- if err != ERR_NO_ERROR { +- bun_output::scoped_log!(dns, "DNSServiceGetAddrInfo failed: {}", err); +- return None; ++ // ShareConnection requires `sub` to start as a copy of the primary ref. ++ let mut sub: DNSServiceRef = self.main_ref; ++ // SAFETY: FFI; `name` is NUL-terminated (copied by dns_sd); `context` is only stored. ++ let err = unsafe { ++ match query_record_allowing_failover() { ++ Some((query_record, allow_failover)) => query_record( ++ &raw mut sub, ++ flags, ++ 0, ++ name, ++ rrtype, ++ CLASS_IN, ++ allow_failover, ++ callback, ++ context, ++ ), ++ None => DNSServiceQueryRecord( ++ &raw mut sub, ++ flags, ++ 0, ++ name, ++ rrtype, ++ CLASS_IN, ++ callback, ++ context, ++ ), ++ } ++ }; ++ if err != ERR_NO_ERROR { ++ bun_output::scoped_log!(dns, "DNSServiceQueryRecord failed: {}", err); ++ deallocate_refs(sd_refs); ++ return None; ++ } ++ sd_refs[i] = sub; + } +- Some(sub) ++ Some(sd_refs) + } + + /// Called first from every reply callback: a different `context` ends the previous request's `MoreComing` run. +@@ -507,7 +599,7 @@ impl SharedConnection { + let ready = core::mem::take(&mut this.inflight); + let detached = SHARED.replace(ptr::null_mut()); + for inf in ready { +- Self::finish(inf, Some(rc)); ++ Self::fail(inf); + } + // SAFETY: `detached` was just removed from SHARED and drained. + unsafe { Self::destroy(detached) }; +@@ -516,7 +608,7 @@ impl SharedConnection { + let ready = this.take_ready(|q| q.is_ready()); + this.arm_early_out(); + for inf in ready { +- Self::finish(inf, None); ++ Self::finish(inf); + } + } + +@@ -600,7 +692,7 @@ impl SharedConnection { + ready + }; + for inf in ready { +- Self::finish(inf, None); ++ Self::finish(inf); + } + } + +@@ -626,22 +718,23 @@ impl SharedConnection { + drop(conn); + } + +- /// `force_err` drops partial results so teardown rejects instead of resolving. +- fn finish(inf: Inflight, force_err: Option) { ++ fn finish(inf: Inflight) { + // SAFETY: `inf` is a live heap request just removed from `inflight`. + let q = unsafe { inf.query() }; +- // SAFETY: FFI; `sd_ref` is this request's live subordinate. +- unsafe { DNSServiceRefDeallocate(q.sd_ref) }; +- if let Some(e) = force_err { +- q.results.clear(); +- q.sd_error = e; +- } else if q.should_retry_unsuppressed() && Self::retry_unsuppressed(inf) { ++ q.deallocate_refs(); ++ if q.should_retry_unsuppressed() && Self::retry_unsuppressed(inf) { + return; + } +- match inf { +- Inflight::Jsc(r) => GetAddrInfoRequest::complete_dns_sd(r), +- Inflight::Internal(r) => internal::dns_sd_complete(r), +- } ++ inf.complete(); ++ } ++ ++ /// The connection is going away: drops partial results so the request rejects instead of resolving. ++ fn fail(inf: Inflight) { ++ // SAFETY: `inf` is a live heap request just removed from `inflight`. ++ let q = unsafe { inf.query() }; ++ q.deallocate_refs(); ++ q.results.clear(); ++ inf.complete(); + } + + /// Reissue `inf`'s query without SuppressUnusable; `false` if it couldn't be reissued. +@@ -652,11 +745,8 @@ impl SharedConnection { + // SAFETY: `inf` is a live heap request removed from `inflight` by the caller. + let q = unsafe { inf.query() }; + let (protocol, hostname) = (protocol_for_pending(q), q.hostname.clone()); +- let Some(callback) = q.callback else { +- return false; +- }; + q.reset_for_retry(protocol); +- let Some(sub) = this.issue(protocol, 0, &hostname, callback, inf.context()) else { ++ let Some(sd_refs) = this.issue(inf, protocol, 0, &hostname) else { + return false; + }; + bun_output::scoped_log!( +@@ -664,7 +754,7 @@ impl SharedConnection { + "retrying {} without SuppressUnusable", + bstr::BStr::new(hostname.as_bytes()) + ); +- q.sd_ref = sub; ++ q.sd_refs = sd_refs; + if this.inflight.is_empty() { + let ctx = this.ctx; + this.file_poll().enable_keeping_process_alive(ctx); +@@ -687,13 +777,12 @@ impl SharedConnection { + // have waiters on other threads (and its outcome is cached): this + // thread going away is not an answer. Finish it on the work pool. + Inflight::Internal(req) => { +- // SAFETY: `inf` is a live heap request just removed from `inflight`; +- // FFI releases this thread's subordinate for it. +- unsafe { DNSServiceRefDeallocate(inf.query().sd_ref) }; ++ // SAFETY: `inf` is a live heap request just removed from `inflight`. ++ unsafe { inf.query() }.deallocate_refs(); + internal::run_on_work_pool(req); + } + // A dns.lookup() from this thread's script: only this VM waits on it. +- Inflight::Jsc(_) => Self::finish(inf, Some(ERR_DEFUNCT_CONNECTION)), ++ Inflight::Jsc(_) => Self::fail(inf), + } + } + // SAFETY: `this` is detached and drained. +@@ -755,13 +844,7 @@ pub(crate) fn lookup( + let promise_value = unsafe { (*request).head.promise.value() }; + + let name_z = bun::ZBox::from_bytes(query.name.as_ref()); +- let Some(_) = shared.start( +- Inflight::Jsc(request), +- protocol, +- &name_z, +- GetAddrInfoRequest::dns_sd_reply, +- request.cast::(), +- ) else { ++ if !shared.start(Inflight::Jsc(request), protocol, &name_z) { + // SAFETY: request is exclusively owned; dns_sd never accepted it. + unsafe { + if let Some(pos) = (*request).pending_slot { +@@ -775,7 +858,7 @@ pub(crate) fn lookup( + drop(bun_core::heap::take(request)); + } + return lib_c::lookup(this, query, global_this, context); +- }; ++ } + + this.request_sent(this.vm()); + +diff --git a/src/runtime/socket/mod.rs b/src/runtime/socket/mod.rs +index bfe0c559..51aec89f 100644 +--- a/src/runtime/socket/mod.rs ++++ b/src/runtime/socket/mod.rs +@@ -120,8 +120,9 @@ pub(crate) use udp_socket::UDPSocket; + /// the name the generator expects rather than special-casing the generator. + pub(crate) mod socket { + pub(crate) use super::socket_body::{ +- js_create_socket_pair, js_get_buffered_amount, js_is_named_pipe_socket, +- js_set_socket_options, js_upgrade_duplex_to_tls, js_upgrade_tls_deferred, testing_ap_is, ++ js_create_socket_pair, js_first_flight_before_fin, js_get_buffered_amount, ++ js_is_named_pipe_socket, js_set_socket_options, js_upgrade_duplex_to_tls, ++ js_upgrade_tls_deferred, testing_ap_is, + }; + } + +diff --git a/src/runtime/socket/socket_body.rs b/src/runtime/socket/socket_body.rs +index 1bf3d1a7..84dfc230 100644 +--- a/src/runtime/socket/socket_body.rs ++++ b/src/runtime/socket/socket_body.rs +@@ -4656,6 +4656,20 @@ pub(crate) fn js_upgrade_tls_deferred( + Err(global.throw(format_args!("Expected a socket instance"))) + } + ++/// `tls.connect()`'s first 'connect' listener, where node sends the ClientHello. ++#[bun_jsc::host_fn] ++pub(crate) fn js_first_flight_before_fin( ++ _global: &JSGlobalObject, ++ callframe: &CallFrame, ++) -> JsResult { ++ jsc::mark_binding!(); ++ let [socket] = callframe.arguments_as_array::<1>(); ++ if let Some(this) = socket.as_class_ref::() { ++ this.socket.get().set_first_flight_before_fin(); ++ } ++ Ok(JSValue::UNDEFINED) ++} ++ + #[bun_jsc::host_fn] + pub(crate) fn js_upgrade_duplex_to_tls( + global: &JSGlobalObject, +diff --git a/src/runtime/webcore/Blob.rs b/src/runtime/webcore/Blob.rs +index f85d1129..fc47dea0 100644 +--- a/src/runtime/webcore/Blob.rs ++++ b/src/runtime/webcore/Blob.rs +@@ -1254,8 +1254,13 @@ impl BlobExt for Blob { + } + + fn get_exists_sync(&self) -> JSValue { +- if self.size.get() == MAX_SIZE { +- self.resolve_size(); ++ match self.store.get() { ++ _ if self.size.get() == MAX_SIZE => self.resolve_size(), ++ // A failed stat is not an answer to keep: the file may exist by now. ++ Some(store) if matches!(&store.data, store::Data::File(file) if file.seekable.is_none()) => { ++ resolve_file_stat(store) ++ } ++ _ => {} + } + + // If there's no store that means it's empty and we just return true +@@ -1996,7 +2001,8 @@ impl BlobExt for Blob { + } + + fn get_size_for_bindings(&self) -> u64 { +- if self.size.get() == MAX_SIZE { ++ let has_size = self.size.get() != MAX_SIZE; ++ if !has_size { + self.resolve_size(); + } + +@@ -2004,7 +2010,17 @@ impl BlobExt for Blob { + // signal that the size is unknown. + if let Some(store) = self.store.get() { + if let store::Data::File(file) = &store.data { +- if !file.seekable.unwrap_or(false) { ++ // A slice has its size without a stat, and `clone()` learns the mode without filling `seekable`. ++ let mode_seen_by_clone = file.mode_seen_by_clone.filter(|_| has_size); ++ if !file ++ .seekable ++ .or_else(|| mode_seen_by_clone.map(bun_sys::S::ISREG)) ++ .unwrap_or(false) ++ { ++ // Printing is not to leave the 0 of a failed stat behind: the body would read as empty. ++ if !has_size && file.seekable.is_none() { ++ self.size.set(MAX_SIZE); ++ } + return u64::MAX; + } + } +@@ -5836,6 +5852,34 @@ pub(crate) fn store_reads_repeatably(store: &RefPtr) -> bool { + } + } + ++/// Whether two Blobs over `store` would compete for its bytes (an fd, a pipe, a terminal). A directory or a closed fd has none: each Blob fails when read. ++pub(crate) fn store_yields_bytes_once(store: &RefPtr) -> bool { ++ if !matches!(store.data, store::Data::File(_)) { ++ return false; ++ } ++ let is_fd = Store::data_mut(store).as_file().pathlike.is_fd(); ++ // What the tee of an fd does next, so this costs no extra `fstat`. ++ if is_fd && Store::data_mut(store).as_file().seekable.is_none() { ++ resolve_file_stat(store); ++ } ++ let file = Store::data_mut(store).as_file_mut(); ++ let mode = if file.seekable.is_some() { ++ Some(file.mode) ++ } else if let PathOrFileDescriptor::Path(path) = &file.pathlike { ++ // Not `resolve_file_stat`: the `Bun.file()` sharing `store` answers from what that caches. ++ if file.mode_seen_by_clone.is_none() { ++ let mut buffer = bun_paths::path_buffer_pool::get(); ++ if let bun_sys::Result::Ok(stat) = bun_sys::stat(path.slice_z(&mut buffer)) { ++ file.mode_seen_by_clone = Some(stat.st_mode as bun_sys::Mode); ++ } ++ } ++ file.mode_seen_by_clone ++ } else { ++ None ++ }; ++ mode.is_some_and(|mode| !bun_sys::S::ISDIR(mode) && (is_fd || !bun_sys::S::ISREG(mode))) ++} ++ + // ────────────────────────────────────────────────────────────────────────── + // toStringWithBytes / toString / toJSON / toFormData / toArrayBuffer{View} + // ────────────────────────────────────────────────────────────────────────── +diff --git a/src/runtime/webcore/Body.rs b/src/runtime/webcore/Body.rs +index 66e2e131..8ede5c11 100644 +--- a/src/runtime/webcore/Body.rs ++++ b/src/runtime/webcore/Body.rs +@@ -127,8 +127,8 @@ impl Body { + unsafe { self.value.get_mut() } + } + +- pub(crate) fn len(&self) -> blob::SizeType { +- self.value_mut().size() ++ pub(crate) fn known_len(&self) -> Option { ++ self.value_mut().known_size() + } + } + +@@ -767,6 +767,14 @@ impl Value { + } + } + ++ /// `None` for a file that does not exist or is not seekable. ++ pub(crate) fn known_size(&mut self) -> Option { ++ match self.size() { ++ u64::MAX => None, ++ size => Some(size as usize), ++ } ++ } ++ + pub(crate) fn memory_cost(&self) -> usize { + match self { + Value::InternalBlob(b) => b.memory_cost(), +@@ -1571,9 +1579,7 @@ impl Value { + } + + if let Value::Blob(b) = self { +- if b.store() +- .is_some_and(|store| !blob::store_reads_repeatably(store)) +- { ++ if b.store().is_some_and(blob::store_yields_bytes_once) { + // A pipe or other fd yields its bytes once: read it as one + // stream and tee that. + self.to_readable_stream(cx)?; +diff --git a/src/runtime/webcore/Request.rs b/src/runtime/webcore/Request.rs +index 1fe691fb..ca3ae8ef 100644 +--- a/src/runtime/webcore/Request.rs ++++ b/src/runtime/webcore/Request.rs +@@ -540,12 +540,15 @@ impl Request { + } else { + "BunRequest" + }; +- writeln!( +- writer, +- "{} ({}) {{", +- class_label, +- bun_fmt::size(self.body_value_mut().size() as usize, Default::default()) +- )?; ++ match self.body_value_mut().known_size() { ++ Some(size) => writeln!( ++ writer, ++ "{} ({}) {{", ++ class_label, ++ bun_fmt::size(size, Default::default()) ++ )?, ++ None => writeln!(writer, "{} {{", class_label)?, ++ } + { + // RAII guard restores indent on every exit incl. `?` error paths. + // Shadows `formatter` for the block; auto-derefs to `&mut F`. +diff --git a/src/runtime/webcore/Response.rs b/src/runtime/webcore/Response.rs +index 8a301930..4a76eae6 100644 +--- a/src/runtime/webcore/Response.rs ++++ b/src/runtime/webcore/Response.rs +@@ -427,10 +427,6 @@ impl Response { + + impl Response { + #[inline] +- pub(crate) fn get_body_len(&self) -> usize { +- self.body.get().len() as usize +- } +- + pub(crate) fn get_form_data_encoding( + &self, + ) -> JsResult>> { +@@ -646,11 +642,14 @@ impl Response { + // `fmt::Error`. + let js_err = |_: JsError| core::fmt::Error; + +- writeln!( +- writer, +- "Response ({}) {{", +- bun_core::fmt::size(self.get_body_len(), Default::default()) +- )?; ++ match self.body.get().known_len() { ++ Some(len) => writeln!( ++ writer, ++ "Response ({}) {{", ++ bun_core::fmt::size(len, Default::default()) ++ )?, ++ None => writeln!(writer, "Response {{")?, ++ } + + { + let mut formatter = formatter.indented(); +diff --git a/src/uws_sys/socket.rs b/src/uws_sys/socket.rs +index f9faf344..32d37ee3 100644 +--- a/src/uws_sys/socket.rs ++++ b/src/uws_sys/socket.rs +@@ -572,6 +572,13 @@ impl NewSocketHandler { + } + } + ++ /// A shutdown before the first handshake step sends its FIN after that step. ++ pub fn set_first_flight_before_fin(&self) { ++ if let InternalSocket::Connected(s) = self.socket { ++ sock(s).set_first_flight_before_fin(); ++ } ++ } ++ + /// The session an SSLWrapper-backed socket got last from the new-session callback, borrowed. + pub fn wrapper_latest_session(&self) -> *mut bun_boringssl_sys::SSL_SESSION { + match self.socket { +diff --git a/src/uws_sys/us_socket_t.rs b/src/uws_sys/us_socket_t.rs +index db3284f2..09f8e8d8 100644 +--- a/src/uws_sys/us_socket_t.rs ++++ b/src/uws_sys/us_socket_t.rs +@@ -309,6 +309,11 @@ impl us_socket_t { + c::us_socket_set_inline_reject(self); + } + ++ /// A shutdown before the first handshake step sends its FIN after that step. ++ pub fn set_first_flight_before_fin(&mut self) { ++ c::us_socket_set_first_flight_before_fin(self); ++ } ++ + /// Feed bytes that were already read off the wire (e.g. a ClientHello the + /// plain-TCP layer consumed before the upgrade) through the same decrypt + /// path as bytes arriving from the kernel. +@@ -591,6 +596,7 @@ mod c { + ) -> *mut us_socket_t; + pub(super) safe fn us_socket_start_tls_handshake(s: &mut us_socket_t); + pub(super) safe fn us_socket_set_inline_reject(s: &mut us_socket_t); ++ pub(super) safe fn us_socket_set_first_flight_before_fin(s: &mut us_socket_t); + } + } + +diff --git a/test/js/bun/dns/mdnsresponder-fixture.ts b/test/js/bun/dns/mdnsresponder-fixture.ts +new file mode 100644 +index 00000000..fca1c897 +--- /dev/null ++++ b/test/js/bun/dns/mdnsresponder-fixture.ts +@@ -0,0 +1,222 @@ ++// A scripted stand-in for mDNSResponder. libsystem_dnssd connects to $DNSSD_UDS_PATH when it is set. ++// ++// bun mdnsresponder-fixture.ts ++// ++// Runs the client script in a child Bun, answers its queries from `answers`, and after the client has ++// gone prints every request it made as one line of JSON. The client's own stdout comes first. ++import { dlopen, ptr } from "bun:ffi"; ++import { closeSync, writeSync } from "node:fs"; ++ ++export type Answer = { ++ rrtype: number; ++ /** An IPv4 or IPv6 address as bytes, or any other rdata. */ ++ rdata?: number[]; ++ /** Defaults to kDNSServiceFlagsAdd. */ ++ flags?: number; ++ error?: number; ++ ifindex?: number; ++ ttl?: number; ++ /** Held back until a query for this name arrives. */ ++ after?: string; ++}; ++/** Keyed by `${name} ${rrtype}`. A query with no entry gets kDNSServiceErr_NoSuchRecord. */ ++export type Answers = Record; ++ ++const [socketPath, answersJSON, clientScript] = process.argv.slice(2); ++const answers: Answers = JSON.parse(answersJSON); ++ ++const { symbols: libc } = dlopen("libSystem.B.dylib", { ++ socket: { args: ["i32", "i32", "i32"], returns: "i32" }, ++ bind: { args: ["i32", "ptr", "u32"], returns: "i32" }, ++ listen: { args: ["i32", "i32"], returns: "i32" }, ++ accept: { args: ["i32", "ptr", "ptr"], returns: "i32" }, ++ poll: { args: ["ptr", "u32", "i32"], returns: "i32" }, ++ recvmsg: { args: ["i32", "ptr", "i32"], returns: "i64" }, ++}); ++ ++const AF_UNIX = 1; ++const SOCK_STREAM = 1; ++const SOL_SOCKET = 0xffff; ++const SCM_RIGHTS = 1; ++const POLLIN = 1; ++ ++const HEADER_SIZE = 28; ++const IPC_FLAGS_TRAILING_TLVS = 2; ++const ops = { 1: "connection", 8: "query", 15: "addrinfo", 63: "cancel" } as const; ++const QUERY_REPLY = 68; ++const ADDRINFO_REPLY = 72; ++ ++const FLAGS_ADD = 0x2; ++const ERR_NO_SUCH_RECORD = -65554; ++const ERR_UNSUPPORTED = -65544; ++const TYPE_A = 1; ++const TYPE_AAAA = 28; ++ ++function check(rc: number, what: string) { ++ if (rc < 0) throw new Error(`${what} failed`); ++ return rc; ++} ++ ++function listenOn(path: string) { ++ const pathBytes = Buffer.from(path); ++ if (pathBytes.length >= 104) throw new Error(`socket path is too long for sun_path: ${path}`); ++ const addr = new Uint8Array(106); ++ addr[0] = addr.length; ++ addr[1] = AF_UNIX; ++ addr.set(pathBytes, 2); ++ const fd = check(libc.socket(AF_UNIX, SOCK_STREAM, 0), "socket"); ++ check(libc.bind(fd, ptr(addr), addr.length), "bind"); ++ check(libc.listen(fd, 8), "listen"); ++ return fd; ++} ++ ++/** One recvmsg(): the bytes read (empty at EOF) and any descriptors that came with them. */ ++function receive(fd: number) { ++ const data = new Uint8Array(4096); ++ const control = new Uint8Array(64); ++ const iov = new BigUint64Array([BigInt(ptr(data)), BigInt(data.length)]); ++ const msg = new DataView(new ArrayBuffer(48)); ++ msg.setBigUint64(16, BigInt(ptr(iov)), true); ++ msg.setInt32(24, 1, true); ++ msg.setBigUint64(32, BigInt(ptr(control)), true); ++ msg.setUint32(40, control.length, true); ++ const n = Number(libc.recvmsg(fd, ptr(new Uint8Array(msg.buffer)), 0)); ++ const fds: number[] = []; ++ const cmsg = new DataView(control.buffer); ++ if (msg.getUint32(40, true) >= 16 && cmsg.getInt32(4, true) === SOL_SOCKET && cmsg.getInt32(8, true) === SCM_RIGHTS) { ++ for (let at = 12; at + 4 <= cmsg.getUint32(0, true); at += 4) fds.push(cmsg.getInt32(at, true)); ++ } ++ return { bytes: Buffer.from(data.subarray(0, Math.max(n, 0))), fds }; ++} ++ ++function reply(op: number, context: Buffer, name: string, qtype: number) { ++ const fullname = Buffer.from((name.endsWith(".") ? name : name + ".") + "\0"); ++ const list = answers[`${name} ${qtype}`] ?? [{ rrtype: qtype, error: ERR_NO_SUCH_RECORD }]; ++ return list.map(({ rrtype, rdata = [], flags = FLAGS_ADD, error = 0, ifindex = 0, ttl = 60, after }) => { ++ const body = Buffer.alloc(12 + fullname.length + 6 + rdata.length + 4); ++ body.writeUInt32BE(flags, 0); ++ body.writeUInt32BE(ifindex, 4); ++ body.writeInt32BE(error, 8); ++ let at = 12 + fullname.copy(body, 12); ++ at = body.writeUInt16BE(rrtype, at); ++ at = body.writeUInt16BE(1, at); ++ at = body.writeUInt16BE(rdata.length, at); ++ at += Buffer.from(rdata).copy(body, at); ++ body.writeUInt32BE(ttl, at); ++ const header = Buffer.alloc(HEADER_SIZE); ++ header.writeUInt32BE(1, 0); ++ header.writeUInt32BE(body.length, 4); ++ header.writeUInt32BE(op, 12); ++ context.copy(header, 16); ++ return { after, bytes: Buffer.concat([header, body]) }; ++ }); ++} ++ ++function cstring(data: Buffer, at: number) { ++ const end = data.indexOf(0, at); ++ return { value: data.toString("latin1", at, end), next: end + 1 }; ++} ++ ++function tlvs(data: Buffer, at: number) { ++ const out: Record = {}; ++ while (at + 4 <= data.length) { ++ const length = data.readUInt16BE(at + 2); ++ out[data.readUInt16BE(at)] = [...data.subarray(at + 4, at + 4 + length)]; ++ at += 4 + length; ++ } ++ return out; ++} ++ ++const requests: object[] = []; ++let held: { conn: number; after: string; bytes: Buffer }[] = []; ++ ++function handle(conn: number, header: Buffer, data: Buffer, errorFds: number[]) { ++ const op = header.readUInt32BE(12); ++ const context = header.subarray(16, 24); ++ const hasTLVs = (header.readUInt32BE(8) & IPC_FLAGS_TRAILING_TLVS) !== 0; ++ const status = Buffer.alloc(4); ++ ++ if (ops[op] === "connection") { ++ writeSync(conn, status); ++ return; ++ } ++ if (ops[op] === "cancel") return; ++ ++ // Byte 0 is the empty control path that marks a request whose status goes to a descriptor of its own. ++ const flags = data.readUInt32BE(1); ++ const ifindex = data.readUInt32BE(5); ++ const replies: ReturnType = []; ++ if (ops[op] === "query") { ++ const { value: name, next } = cstring(data, 9); ++ const rrtype = data.readUInt16BE(next); ++ for (const { conn, bytes } of held.filter(({ after }) => after === name)) writeSync(conn, bytes); ++ held = held.filter(({ after }) => after !== name); ++ requests.push({ op: "query", name, rrtype, flags, ifindex, tlvs: hasTLVs ? tlvs(data, next + 4) : {} }); ++ replies.push(...reply(QUERY_REPLY, context, name, rrtype)); ++ } else if (ops[op] === "addrinfo") { ++ const protocol = data.readUInt32BE(9); ++ const { value: name, next } = cstring(data, 13); ++ requests.push({ op: "addrinfo", name, protocol, flags, ifindex, tlvs: hasTLVs ? tlvs(data, next) : {} }); ++ if (protocol & 2) replies.push(...reply(ADDRINFO_REPLY, context, name, TYPE_AAAA)); ++ if (protocol & 1) replies.push(...reply(ADDRINFO_REPLY, context, name, TYPE_A)); ++ } else { ++ requests.push({ op }); ++ status.writeInt32BE(ERR_UNSUPPORTED); ++ } ++ ++ const errorFd = errorFds.shift(); ++ if (errorFd === undefined) throw new Error(`request with op ${op} came without a descriptor for its status`); ++ writeSync(errorFd, status); ++ closeSync(errorFd); ++ for (const { after, bytes } of replies) { ++ if (after === undefined) writeSync(conn, bytes); ++ else held.push({ conn, after, bytes }); ++ } ++} ++ ++const listener = listenOn(socketPath); ++const client = Bun.spawn({ ++ cmd: [process.execPath, "-e", clientScript], ++ env: { ...process.env, DNSSD_UDS_PATH: socketPath }, ++ stdio: ["ignore", "inherit", "inherit", "pipe"], ++}); ++// Only the client holds the other end, so this becomes readable when the client is gone. ++const clientGone = client.stdio[3] as number; ++ ++const conns = new Map(); ++serving: while (true) { ++ const fds = [clientGone, listener, ...conns.keys()]; ++ const pollfds = new DataView(new ArrayBuffer(fds.length * 8)); ++ fds.forEach((fd, i) => { ++ pollfds.setInt32(i * 8, fd, true); ++ pollfds.setInt16(i * 8 + 4, POLLIN, true); ++ }); ++ check(libc.poll(ptr(new Uint8Array(pollfds.buffer)), fds.length, -1), "poll"); ++ for (let i = 0; i < fds.length; i++) { ++ if (pollfds.getInt16(i * 8 + 6, true) === 0) continue; ++ const fd = fds[i]; ++ if (fd === clientGone) break serving; ++ if (fd === listener) { ++ conns.set(check(libc.accept(listener, null, null), "accept"), { pending: Buffer.alloc(0), errorFds: [] }); ++ continue; ++ } ++ const conn = conns.get(fd)!; ++ const { bytes, fds: passed } = receive(fd); ++ if (bytes.length === 0) { ++ closeSync(fd); ++ conns.delete(fd); ++ continue; ++ } ++ conn.errorFds.push(...passed); ++ conn.pending = Buffer.concat([conn.pending, bytes]); ++ while (conn.pending.length >= HEADER_SIZE) { ++ const size = HEADER_SIZE + conn.pending.readUInt32BE(4); ++ if (conn.pending.length < size) break; ++ handle(fd, conn.pending.subarray(0, HEADER_SIZE), conn.pending.subarray(HEADER_SIZE, size), conn.errorFds); ++ conn.pending = conn.pending.subarray(size); ++ } ++ } ++} ++ ++console.log(JSON.stringify(requests)); ++process.exitCode = await client.exited; +diff --git a/test/js/bun/dns/resolve-dns.test.ts b/test/js/bun/dns/resolve-dns.test.ts +index a9d7efde..6f2fec08 100644 +--- a/test/js/bun/dns/resolve-dns.test.ts ++++ b/test/js/bun/dns/resolve-dns.test.ts +@@ -1,9 +1,11 @@ + import { SystemError, dns } from "bun"; +-import { describe, expect, test } from "bun:test"; +-import { bunEnv, bunExe, isASAN, isWindows, withoutAggressiveGC } from "harness"; ++import { afterAll, beforeAll, describe, expect, test } from "bun:test"; ++import { bunEnv, bunExe, isASAN, isMacOS, isWindows, tempDir, withoutAggressiveGC } from "harness"; + import { isIP, isIPv4, isIPv6 } from "node:net"; + import { join } from "node:path"; ++import type { Answer, Answers } from "./mdnsresponder-fixture"; + ++const cc = Bun.which("cc") || Bun.which("clang"); + const backends = ["system", "libc", "c-ares"]; + const validHostnames = ["localhost", "example.com"]; + const invalidHostnames = ["adsfa.asdfasdf.asdf.com"]; // known invalid +@@ -298,4 +300,240 @@ describe("dns", () => { + ); + }); + }); ++ ++ // On macOS the system backend talks to mDNSResponder over a unix socket. These put a scripted responder ++ // behind that socket, so each test sees the requests Bun makes and chooses the replies Bun gets. ++ describe.skipIf(!isMacOS || !cc)("system backend over the mDNSResponder socket", () => { ++ // libsystem_dnssd has a DNSSD_UDS_PATH override of its own, but on macOS 26 only a setuid process reads it. ++ const redirectConnect = /* c */ ` ++ #include ++ #include ++ #include ++ #include ++ ++ int connect_nocancel(int, const struct sockaddr *, socklen_t) __asm("_connect$NOCANCEL"); ++ ++ static const struct sockaddr *redirect(const struct sockaddr *addr, struct sockaddr_un *to) { ++ const char *path = getenv("DNSSD_UDS_PATH"); ++ if (!path || addr->sa_family != AF_UNIX || ++ strcmp(((const struct sockaddr_un *)addr)->sun_path, "/var/run/mDNSResponder")) ++ return addr; ++ to->sun_len = sizeof(*to); ++ to->sun_family = AF_UNIX; ++ strlcpy(to->sun_path, path, sizeof(to->sun_path)); ++ return (const struct sockaddr *)to; ++ } ++ ++ static int redirected_connect(int fd, const struct sockaddr *addr, socklen_t len) { ++ struct sockaddr_un to; ++ return connect(fd, redirect(addr, &to), len); ++ } ++ ++ static int redirected_connect_nocancel(int fd, const struct sockaddr *addr, socklen_t len) { ++ struct sockaddr_un to; ++ return connect_nocancel(fd, redirect(addr, &to), len); ++ } ++ ++ __attribute__((used, section("__DATA,__interpose"))) static const struct { ++ const void *replacement, *original; ++ } interpose[] = { ++ {(const void *)redirected_connect, (const void *)connect}, ++ {(const void *)redirected_connect_nocancel, (const void *)connect_nocancel}, ++ }; ++ `; ++ ++ let dir: ReturnType; ++ let sockets = 0; ++ ++ beforeAll(async () => { ++ dir = tempDir("dns-sd", { "redirect.c": redirectConnect }); ++ await using proc = Bun.spawn({ ++ cmd: [cc!, "-dynamiclib", "-o", "redirect.dylib", "redirect.c"], ++ cwd: String(dir), ++ env: bunEnv, ++ stdout: "inherit", ++ stderr: "pipe", ++ }); ++ const [stderr, exitCode] = await Promise.all([proc.stderr.text(), proc.exited]); ++ if (exitCode !== 0) throw new Error(`could not compile redirect.c: ${stderr}`); ++ }); ++ ++ afterAll(() => dir?.[Symbol.dispose]()); ++ ++ const A = 1; ++ const CNAME = 5; ++ const AAAA = 28; ++ // kDNSServiceFlagsShareConnection | kDNSServiceFlagsTimeout | kDNSServiceFlagsReturnIntermediates ++ const baseFlags = 0x4000 | 0x10000 | 0x1000; ++ const suppressUnusable = 0x8000; ++ const alias: Answer = { rrtype: CNAME, rdata: [5, ...Buffer.from("alias"), 4, ...Buffer.from("test"), 0] }; ++ const a = (...rdata: number[]): Answer => ({ rrtype: A, rdata }); ++ const aaaa = (first: number, last: number, more: Partial = {}): Answer => ({ ++ rrtype: AAAA, ++ rdata: [first >> 8, first & 0xff, ...Buffer.alloc(13), last], ++ ...more, ++ }); ++ ++ /** Runs `script` with `answers` behind the socket: the one line of JSON it prints, and the requests it made. */ ++ async function exchange(answers: Answers, script: string) { ++ await using proc = Bun.spawn({ ++ cmd: [ ++ bunExe(), ++ join(import.meta.dir, "mdnsresponder-fixture.ts"), ++ join(String(dir), `${sockets++}.sock`), ++ JSON.stringify(answers), ++ script, ++ ], ++ env: { ...bunEnv, DYLD_INSERT_LIBRARIES: join(String(dir), "redirect.dylib"), NO_PROXY: "*", no_proxy: "*" }, ++ stdout: "pipe", ++ stderr: "inherit", ++ }); ++ const [stdout, exitCode] = await Promise.all([proc.stdout.text(), proc.exited]); ++ const [printed, requests] = stdout ++ .trim() ++ .split("\n") ++ .map(line => JSON.parse(line)); ++ return { ++ printed, ++ // TLV 4 is IPC_TLV_TYPE_SERVICE_ATTR_FAILOVER_POLICY; 1 is kDNSServiceFailoverPolicyAllow. ++ requests: requests.map(({ tlvs, ...request }) => ({ ...request, failoverPolicy: tlvs?.[4]?.at(-1) })), ++ exitCode, ++ }; ++ } ++ ++ const lookup = (name: string, options: object = {}) => ++ `console.log(JSON.stringify(await Bun.dns.lookup(${JSON.stringify(name)}, ${JSON.stringify(options)}).catch(e => e.code)))`; ++ ++ const queries = (name: string, flags: number, rrtypes = [A, AAAA]) => ++ rrtypes.map(rrtype => ({ op: "query", name, rrtype, flags, ifindex: 0, failoverPolicy: 1 })); ++ ++ const both = { ++ "host.corp.example 1": [alias, a(10, 0, 0, 10)], ++ "host.corp.example 28": [alias, aaaa(0xfd00, 0x10)], ++ }; ++ ++ // https://github.com/oven-sh/bun/issues/44075 ++ test.concurrent("lookup() queries each family and allows failover to another resolver", async () => { ++ expect(await exchange(both, lookup("host.corp.example"))).toEqual({ ++ printed: [ ++ { address: "fd00::10", family: 6, ttl: 60 }, ++ { address: "10.0.0.10", family: 4, ttl: 60 }, ++ ], ++ requests: queries("host.corp.example", baseFlags | suppressUnusable), ++ exitCode: 0, ++ }); ++ }); ++ ++ test.concurrent("lookup() makes the same requests as getaddrinfo()", async () => { ++ const [system, libc] = await Promise.all( ++ ["system", "libc"].map(backend => exchange(both, lookup("host.corp.example", { backend }))), ++ ); ++ expect(libc.requests).toHaveLength(2); ++ expect(system.requests).toEqual(libc.requests); ++ }); ++ ++ test.concurrent("fetch() queries each family and allows failover to another resolver", async () => { ++ const script = ` ++ using server = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: () => new Response("reached") }); ++ const response = await fetch("http://host.corp.example:" + server.port + "/"); ++ console.log(JSON.stringify(await response.text())); ++ `; ++ expect(await exchange({ "host.corp.example 1": [alias, a(127, 0, 0, 1)] }, script)).toEqual({ ++ printed: "reached", ++ requests: queries("host.corp.example", baseFlags | suppressUnusable), ++ exitCode: 0, ++ }); ++ }); ++ ++ test.concurrent.each([ ++ { family: 4, rrtype: A, address: "10.0.0.10" }, ++ { family: 6, rrtype: AAAA, address: "fd00::10" }, ++ ])("lookup() with family: $family queries that family alone", async ({ family, rrtype, address }) => { ++ expect(await exchange(both, lookup("host.corp.example", { family }))).toEqual({ ++ printed: [{ address, family, ttl: 60 }], ++ requests: queries("host.corp.example", baseFlags, [rrtype]), ++ exitCode: 0, ++ }); ++ }); ++ ++ // mDNSResponder tries the search domains for a single label only when the name has no trailing dot. ++ test.concurrent.each(["intranet", "host.corp.example."])("lookup(%j) sends the name as written", async name => { ++ expect(await exchange({ [`${name} 1`]: [a(10, 0, 0, 10)] }, lookup(name))).toEqual({ ++ printed: [{ address: "10.0.0.10", family: 4, ttl: 60 }], ++ requests: queries(name, baseFlags | suppressUnusable), ++ exitCode: 0, ++ }); ++ }); ++ ++ test.concurrent("lookup() scopes a link-local address to the interface it was seen on", async () => { ++ const answers = { "printer.local 28": [aaaa(0xfe80, 1, { ifindex: 7 }), aaaa(0xfd00, 1, { ifindex: 7 })] }; ++ expect(await exchange(answers, lookup("printer.local"))).toEqual({ ++ printed: [ ++ { address: "fe80::1%7", family: 6, ttl: 60 }, ++ { address: "fd00::1", family: 6, ttl: 60 }, ++ ], ++ requests: queries("printer.local", baseFlags | suppressUnusable), ++ exitCode: 0, ++ }); ++ }); ++ ++ test.concurrent("lookup() keeps only well-formed records that were added", async () => { ++ const removed = { ...a(10, 0, 0, 9), flags: 0 }; ++ const answers = { "host.corp.example 1": [a(10, 0, 0, 10), removed, a(10, 0, 0)] }; ++ expect(await exchange(answers, lookup("host.corp.example"))).toEqual({ ++ printed: [{ address: "10.0.0.10", family: 4, ttl: 60 }], ++ requests: queries("host.corp.example", baseFlags | suppressUnusable), ++ exitCode: 0, ++ }); ++ }); ++ ++ // kDNSServiceErr_NoAuth. The daemon reports a question it refuses to start with no record type. ++ const refused: Answer = { rrtype: 0, error: -65555 }; ++ ++ test.concurrent("lookup() waits for the other family when the daemon refuses one question", async () => { ++ const answers = { ++ "host.corp.example 1": [refused], ++ "host.corp.example 28": [aaaa(0xfd00, 0x10, { after: "second.test" })], ++ "first.test 1": [a(10, 0, 0, 1)], ++ "second.test 1": [a(10, 0, 0, 2)], ++ }; ++ // The answer for first.test is behind the refusal on the socket, so the refusal has been read once it resolves. ++ const script = ` ++ const host = Bun.dns.lookup("host.corp.example").catch(e => e.code); ++ await Bun.dns.lookup("first.test"); ++ await Bun.dns.lookup("second.test"); ++ console.log(JSON.stringify(await host)); ++ `; ++ expect(await exchange(answers, script)).toEqual({ ++ printed: [{ address: "fd00::10", family: 6, ttl: 60 }], ++ requests: ["host.corp.example", "first.test", "second.test"].flatMap(name => ++ queries(name, baseFlags | suppressUnusable), ++ ), ++ exitCode: 0, ++ }); ++ }); ++ ++ test.concurrent("lookup() reports ENOTFOUND when the daemon refuses both questions", async () => { ++ const answers = { "host.corp.example 1": [refused], "host.corp.example 28": [refused] }; ++ expect(await exchange(answers, lookup("host.corp.example"))).toEqual({ ++ printed: "DNS_ENOTFOUND", ++ requests: [ ++ ...queries("host.corp.example", baseFlags | suppressUnusable), ++ ...queries("host.corp.example", baseFlags), ++ ], ++ exitCode: 0, ++ }); ++ }); ++ ++ test.concurrent("lookup() asks once more without SuppressUnusable before it reports ENOTFOUND", async () => { ++ expect(await exchange({}, lookup("host.corp.example"))).toEqual({ ++ printed: "DNS_ENOTFOUND", ++ requests: [ ++ ...queries("host.corp.example", baseFlags | suppressUnusable), ++ ...queries("host.corp.example", baseFlags), ++ ], ++ exitCode: 0, ++ }); ++ }); ++ }); + }); +diff --git a/test/js/bun/util/bun-file-exists.test.js b/test/js/bun/util/bun-file-exists.test.js +index dc1a810c..f8526c84 100644 +--- a/test/js/bun/util/bun-file-exists.test.js ++++ b/test/js/bun/util/bun-file-exists.test.js +@@ -1,6 +1,7 @@ + import { write } from "bun"; + import { expect, test } from "bun:test"; +-import { unlinkSync } from "fs"; ++import { unlinkSync, writeFileSync } from "fs"; ++import { tempDir } from "harness"; + import { tmpdir } from "os"; + import { join } from "path"; + test("bun-file-exists", async () => { +@@ -18,3 +19,15 @@ test("bun-file-exists", async () => { + unlinkSync(temp); + expect(await Bun.file(temp).exists()).toBeFalse(); + }); ++ ++test.each([ ++ ["exists()", file => file.exists()], ++ ["size", file => file.size], ++])("one Bun.file() sees a file that is created after %s found none", async (_, look) => { ++ using dir = tempDir("bun-file-exists-later", {}); ++ const file = Bun.file(join(String(dir), "later.txt")); ++ await look(file); ++ expect(await file.exists()).toBeFalse(); ++ writeFileSync(file.name, "boop"); ++ expect(await file.exists()).toBeTrue(); ++}); +diff --git a/test/js/bun/util/inspect.test.js b/test/js/bun/util/inspect.test.js +index 931e45c7..3549700b 100644 +--- a/test/js/bun/util/inspect.test.js ++++ b/test/js/bun/util/inspect.test.js +@@ -141,6 +141,34 @@ it("Blob inspect", () => { + }`); + }); + ++it("Response and Request print no size for a Bun.file() body whose size is unknown", () => { ++ using dir = tempDir("inspect-unknown-size", {}); ++ const firstLines = file => [ ++ Bun.inspect(new Response(file)).split("\n")[0], ++ Bun.inspect(new Request("http://example.com/", { method: "POST", body: file })).split("\n")[0], ++ ]; ++ expect({ ++ missing: firstLines(Bun.file(join(String(dir), "missing.txt"))), ++ directory: firstLines(Bun.file(String(dir))), ++ }).toEqual({ ++ missing: ["Response {", "Request {"], ++ directory: ["Response {", "Request {"], ++ }); ++}); ++ ++it("printing a Response over a Bun.file() that does not exist yet leaves its body as it is", async () => { ++ using dir = tempDir("inspect-missing-then-created", {}); ++ const path = join(String(dir), "later.txt"); ++ const response = new Response(Bun.file(path)); ++ const before = Bun.inspect(response).split("\n")[0]; ++ await Bun.write(path, "created"); ++ expect({ before, after: Bun.inspect(response).split("\n")[0], text: await response.text() }).toEqual({ ++ before: "Response {", ++ after: "Response (7 bytes) {", ++ text: "created", ++ }); ++}); ++ + it("utf16 property name", () => { + var { Database } = require("bun:sqlite"); + const db = Database.open(":memory:"); +diff --git a/test/js/bun/util/sleepSync.test.ts b/test/js/bun/util/sleepSync.test.ts +index 45361c2b..99a9f820 100644 +--- a/test/js/bun/util/sleepSync.test.ts ++++ b/test/js/bun/util/sleepSync.test.ts +@@ -1,5 +1,6 @@ + import { sleepSync } from "bun"; + import { expect, it } from "bun:test"; ++import { bunEnv, bunExe, isASAN } from "harness"; + + it("sleepSync uses milliseconds", async () => { + const start = performance.now(); +@@ -28,3 +29,49 @@ it("sleepSync with negative number throws", async () => { + it("can map with sleepSync", async () => { + [1, 2, 3].map(sleepSync); + }); ++ ++// Free blocks inside pages that are still in use belong to the thread that owns the pages. They go back to the OS ++// when that thread tells mimalloc that it is about to block. ++it.skipIf(isASAN /* malloc is not mimalloc */)( ++ "sleepSync lets mimalloc release this thread's free memory", ++ async () => { ++ await using proc = Bun.spawn({ ++ cmd: [ ++ bunExe(), ++ "-e", ++ ` ++ const { heapStats } = require("bun:jsc"); ++ const purgeCalls = () => heapStats().mimalloc.purge_calls; ++ const spin = ms => { const start = performance.now(); while (performance.now() - start < ms); }; ++ ++ // the characters of these strings are allocated and freed by this thread ++ let strings = []; ++ for (let i = 0; i < 100000; i++) strings.push(Buffer.alloc(900 + (i % 5) * 8, 97).toString("latin1")); ++ // (far enough apart that whole OS pages are free in between, also where those are 16 KB) ++ strings = strings.filter((_, i) => i % 64 === 0); ++ Bun.gc(true); ++ ++ // what needs no idle thread settles first, without going idle ++ let before = purgeCalls(); ++ for (let stable = 0, tries = 0; stable < 3 && tries < 50; tries++) { ++ spin(60); ++ const now = purgeCalls(); ++ stable = now === before ? stable + 1 : 0; ++ before = now; ++ } ++ ++ let released = 0; ++ for (let i = 0; i < 20 && released < 500; i++) { ++ Bun.sleepSync(100); ++ released = purgeCalls() - before; ++ } ++ console.log(released >= 500, strings.length); ++ `, ++ ], ++ env: bunEnv, ++ stderr: "pipe", ++ }); ++ const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); ++ expect({ stdout, stderr, exitCode }).toEqual({ stdout: "true 1563\n", stderr: "", exitCode: 0 }); ++ }, ++); +diff --git a/test/js/bun/wasm/compile-rss-fixture.mjs b/test/js/bun/wasm/compile-rss-fixture.mjs +new file mode 100644 +index 00000000..c5ff8e8b +--- /dev/null ++++ b/test/js/bun/wasm/compile-rss-fixture.mjs +@@ -0,0 +1,145 @@ ++// Fixture for compile-rss.test.ts. Compiles a generated wasm module 6 times, discards each result, ++// then polls RSS until it falls below the target (argv[2], MiB) or the deadline passes. The idle ++// compiler threads exit after 10 s and release everything then, so the deadline stays well below ++// that. Prints one JSON line with the lowest RSS growth seen, relative to the RSS before the first ++// compile. ++ ++class Bytes { ++ constructor() { ++ this.buf = new Uint8Array(1 << 20); ++ this.len = 0; ++ } ++ push(...bytes) { ++ for (const b of bytes) this.byte(b); ++ } ++ byte(b) { ++ if (this.len === this.buf.length) { ++ const next = new Uint8Array(this.buf.length * 2); ++ next.set(this.buf); ++ this.buf = next; ++ } ++ this.buf[this.len++] = b; ++ } ++ leb(n) { ++ do { ++ let b = n & 0x7f; ++ n >>>= 7; ++ if (n !== 0) b |= 0x80; ++ this.byte(b); ++ } while (n !== 0); ++ } ++ append(other) { ++ for (let i = 0; i < other.len; i++) this.byte(other.buf[i]); ++ } ++ section(id, payload) { ++ this.byte(id); ++ this.leb(payload.len); ++ this.append(payload); ++ } ++ bytes() { ++ return this.buf.subarray(0, this.len); ++ } ++} ++ ++// A module shaped like a tree-sitter parser: a few dozen functions of 20 to 30 KB of control-flow ++// heavy code, plus one giant function. Compiling it makes each wasm compiler thread allocate and ++// free tens of MB of temporaries. Uniform modules of many small functions do not show the ++// retention: their pages end up fully free and mimalloc's scavenger purges them on its own. ++function makeModule({ functionCount = 35, opsPerFunction = 1400, giantOps = 30000 } = {}) { ++ const out = new Bytes(); ++ out.push(0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00); ++ ++ // type 0: (i32, i32) -> i32 ++ const types = new Bytes(); ++ types.push(1, 0x60, 2, 0x7f, 0x7f, 1, 0x7f); ++ out.section(1, types); ++ ++ const funcs = new Bytes(); ++ funcs.leb(functionCount); ++ for (let f = 0; f < functionCount; f++) funcs.byte(0); ++ out.section(3, funcs); ++ ++ const exports = new Bytes(); ++ exports.push(1, 1, 0x66, 0x00, 0); // export "f" = func 0 ++ out.section(7, exports); ++ ++ const code = new Bytes(); ++ code.leb(functionCount); ++ for (let f = 0; f < functionCount; f++) { ++ const ops = f === 0 ? giantOps : opsPerFunction + ((f * 7919) % 400); ++ const body = new Bytes(); ++ body.push(1, 1, 0x7f); // one local group: 1 x i32 ++ body.push(0x20, 0); // local.get 0 ++ for (let i = 0; i < ops; i++) { ++ // block ++ // local.get 1; i32.const k; i32.add; local.tee 2 ++ // br_table {0,0,0,0} 0 ++ // local.get 2; local.get 0; call g; drop ++ // end ++ // local.get 2; i32.xor ++ body.push(0x02, 0x40, 0x20, 1, 0x41); ++ body.leb((f * 31 + i * 7) & 0x3f); ++ body.push(0x6a, 0x22, 2, 0x0e, 4, 0, 0, 0, 0, 0, 0x20, 2, 0x20, 0, 0x10); ++ body.leb((f + 1) % functionCount); ++ body.push(0x1a, 0x0b, 0x20, 2, 0x73); ++ } ++ body.byte(0x0b); // end ++ code.leb(body.len); ++ code.append(body); ++ } ++ out.section(10, code); ++ return out.bytes(); ++} ++ ++const targetMiB = Number(process.argv[2]); ++if (!Number.isFinite(targetMiB)) throw new Error(`expected the target in MiB as argv[2], got ${process.argv[2]}`); ++// On Darwin mimalloc returns memory with MADV_FREE_REUSABLE, which the kernel keeps counted in RSS ++// until it reuses the pages. phys_footprint drops at once, so measure that there. memoryFootprint() ++// returns undefined when task_info fails, so fall back to RSS. ++const rss = () => ++ (process.platform === "darwin" ? Bun.unsafe.memoryFootprint?.() : undefined) ?? process.memoryUsage.rss(); ++const sleep = ms => new Promise(r => setTimeout(r, ms)); ++const bytes = makeModule(); ++ ++// Building the module leaves garbage behind. Wait until RSS stops falling before taking the ++// baseline: mimalloc hands freed memory back on its own schedule, and there is no signal for it. ++let base = Infinity; ++const baseDeadline = performance.now() + 2000; ++do { ++ Bun.gc(true); ++ await sleep(50); ++ const now = rss(); ++ if (now >= base - 1048576) break; ++ base = now; ++} while (performance.now() < baseDeadline); ++// Each compile adds to what the unfixed threads keep (8 threads: 18 to 27 MiB after 3 compiles, ++// 35 to 39 MiB after 6) and costs about 20 ms. The fixed build returns to the baseline either way. ++for (let i = 0; i < 6; i++) { ++ let mod = await WebAssembly.compile(bytes); ++ mod = null; ++} ++// Read before anything is freed: the test checks that the compiles grew RSS at all. ++const peak = rss() - base; ++// Free the modules. The main thread frees their metadata into the compiler threads' pages, and only ++// those threads can hand that memory back. They do so once they have worked and gone idle again, so ++// give each of them one trivial function to compile after the modules are gone. ++Bun.gc(true); ++await WebAssembly.compile(makeModule({ functionCount: 16, opsPerFunction: 1, giantOps: 1 })); ++Bun.gc(true); ++const afterCompiles = rss() - base; ++ ++const deadline = performance.now() + 3000; ++let min = Infinity; ++do { ++ await sleep(100); ++ Bun.gc(true); ++ min = Math.min(min, rss() - base); ++} while (performance.now() < deadline && min / 1048576 >= targetMiB); ++console.log( ++ JSON.stringify({ ++ moduleMiB: bytes.length / 1048576, ++ peakDeltaMiB: peak / 1048576, ++ afterCompilesDeltaMiB: afterCompiles / 1048576, ++ idleDeltaMiB: min / 1048576, ++ }), ++); +diff --git a/test/js/bun/wasm/compile-rss.test.ts b/test/js/bun/wasm/compile-rss.test.ts +new file mode 100644 +index 00000000..8987962c +--- /dev/null ++++ b/test/js/bun/wasm/compile-rss.test.ts +@@ -0,0 +1,38 @@ ++import { expect, test } from "bun:test"; ++import { bunEnv, bunExe, isASAN, isDebug } from "harness"; ++import path from "node:path"; ++ ++// Unfixed, the 8 idle wasm compiler threads keep 35 to 39 MiB of freed compile temporaries until ++// they exit after 10 s. Fixed, they release it about 100 ms after the last compile and RSS returns ++// to where it started. ++const idleTargetMiB = 10; ++ ++// Debug and ASAN builds link a JavaScriptCore that does not allocate through mimalloc, so the ++// per-thread retention this test checks for does not exist there. ++test.skipIf(isDebug || isASAN)( ++ "WebAssembly.compile does not retain memory in idle compiler threads (#41438)", ++ async () => { ++ await using proc = Bun.spawn({ ++ cmd: [bunExe(), path.join(import.meta.dir, "compile-rss-fixture.mjs"), String(idleTargetMiB)], ++ env: { ++ ...bunEnv, ++ // The retained amount scales with the compiler thread count. Pin it so the test does not ++ // depend on the core count of the machine. ++ BUN_JSC_numberOfWasmCompilerThreads: "8", ++ }, ++ stdout: "pipe", ++ stderr: "pipe", ++ }); ++ const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); ++ expect(stderr).toBe(""); ++ const result = JSON.parse(stdout.trim().split("\n").at(-1)!); ++ // The whole measurement is in the object so a failure prints it. The compiles have to grow RSS ++ // well past the target first, or the idle check means nothing. ++ expect({ ++ ...result, ++ grew: result.peakDeltaMiB > idleTargetMiB * 2, ++ released: result.idleDeltaMiB < idleTargetMiB, ++ }).toMatchObject({ grew: true, released: true }); ++ expect(exitCode).toBe(0); ++ }, ++); +diff --git a/test/js/node/buffer.test.js b/test/js/node/buffer.test.js +index 01ad7373..a2df8e1a 100644 +--- a/test/js/node/buffer.test.js ++++ b/test/js/node/buffer.test.js +@@ -4527,6 +4527,136 @@ describe("raw Slice / Write bindings match Node", () => { + expect(buf.toString("hex")).toBe(untouched); + }); + ++ // Node's native writers start with THROW_AND_RETURN_IF_NOT_STRING: they reject a value ++ // that is not a primitive string and never coerce it. ++ describe("with a value that is not a string", () => { ++ const NOT_A_STRING = expect.objectContaining({ ++ code: "ERR_INVALID_ARG_TYPE", ++ message: "argument must be a string", ++ }); ++ const nonStrings = () => [ ++ 123, ++ null, ++ undefined, ++ true, ++ 1n, ++ Symbol("s"), ++ {}, ++ [], ++ new String("ab"), ++ Buffer.from("ab"), ++ () => {}, ++ ]; ++ ++ it.each([...strict, ...clamping])("%s throws ERR_INVALID_ARG_TYPE", method => { ++ const buf = dest(); ++ for (const value of nonStrings()) { ++ expect(() => buf[method](value)).toThrow(NOT_A_STRING); ++ expect(() => buf[method](value, 0, 1)).toThrow(NOT_A_STRING); ++ } ++ expect(() => buf[method]()).toThrow(NOT_A_STRING); ++ expect(() => buf[method](123)).toThrow(TypeError); ++ expect(buf.toString("hex")).toBe(untouched); ++ }); ++ ++ it.each([...strict, ...clamping])("%s does not coerce an object value", method => { ++ const calls = []; ++ const value = { ++ toString() { ++ calls.push("toString"); ++ return source[method]; ++ }, ++ valueOf() { ++ calls.push("valueOf"); ++ return source[method]; ++ }, ++ [Symbol.toPrimitive]() { ++ calls.push("toPrimitive"); ++ return source[method]; ++ }, ++ }; ++ // The handler is a Proxy too, so a lookup of any trap is recorded. A message built ++ // from the value would show up here as a `get` of "constructor". ++ const proxy = new Proxy(value, new Proxy({}, { get: (_, trap) => void calls.push(`trap ${trap}`) })); ++ const buf = dest(); ++ expect(() => buf[method](value)).toThrow(NOT_A_STRING); ++ expect(() => buf[method](proxy)).toThrow(NOT_A_STRING); ++ expect(calls).toEqual([]); ++ expect(buf.toString("hex")).toBe(untouched); ++ }); ++ ++ it.each([...strict, ...clamping])("%s throws on an empty, a detached and a plain Uint8Array receiver", method => { ++ const detached = Buffer.from(new ArrayBuffer(9)); ++ structuredClone(detached.buffer, { transfer: [detached.buffer] }); ++ expect(() => Buffer.alloc(0)[method](123)).toThrow(NOT_A_STRING); ++ expect(() => detached[method](123)).toThrow(NOT_A_STRING); ++ expect(() => Buffer.prototype[method].call(new Uint8Array(9), 123)).toThrow(NOT_A_STRING); ++ }); ++ ++ // utf8Write/latin1Write/asciiWrite check the bounds in a JS wrapper first. The native ++ // writer sees the value only after that, so a bounds error wins. ++ it.each(strict)("%s reports an out-of-bounds offset or length first", method => { ++ const buf = dest(); ++ expect(() => buf[method](123, -1)).toThrow(OUT_OF_BOUNDS); ++ expect(() => buf[method](123, 10)).toThrow(OUT_OF_BOUNDS); ++ expect(() => buf[method](123, Infinity)).toThrow(OUT_OF_BOUNDS); ++ expect(() => buf[method](123, 0, -1)).toThrow(OUT_OF_BOUNDS); ++ expect(() => buf[method](123, 0, 10)).toThrow(OUT_OF_BOUNDS); ++ expect(() => buf[method](123, 6, 4)).toThrow(OUT_OF_BOUNDS); ++ // In bounds, including the ranges that leave nothing to write. ++ expect(() => buf[method](123, 9)).toThrow(NOT_A_STRING); ++ expect(() => buf[method](123, 0, 0)).toThrow(NOT_A_STRING); ++ expect(() => buf[method](123, NaN)).toThrow(NOT_A_STRING); ++ expect(() => buf[method](123, NaN, NaN)).toThrow(NOT_A_STRING); ++ expect(() => buf[method](123, "abc")).toThrow(NOT_A_STRING); ++ expect(() => buf[method](123, 1.5)).toThrow(NOT_A_STRING); ++ }); ++ ++ it.each(strict)("%s converts offset and length before it rejects the value", method => { ++ const converted = []; ++ const arg = (name, result) => ({ ++ valueOf() { ++ if (!converted.includes(name)) converted.push(name); ++ return result; ++ }, ++ }); ++ expect(() => dest()[method](123, arg("offset", 0), arg("length", 1))).toThrow(NOT_A_STRING); ++ expect(converted).toEqual(["offset", "length"]); ++ ++ const throwing = message => ({ ++ valueOf() { ++ throw new Error(message); ++ }, ++ }); ++ expect(() => dest()[method](123, throwing("offset valueOf"))).toThrow("offset valueOf"); ++ expect(() => dest()[method](123, 0, throwing("length valueOf"))).toThrow("length valueOf"); ++ }); ++ ++ // base64/base64url/hex/ucs2 are the raw binding. It rejects the value before it reads ++ // offset or length, so the value error wins and neither argument is converted. ++ it.each(clamping)("%s rejects the value before it reads offset and length", method => { ++ const buf = dest(); ++ expect(() => buf[method](123, -1)).toThrow(NOT_A_STRING); ++ expect(() => buf[method](123, 10)).toThrow(NOT_A_STRING); ++ expect(() => buf[method](123, Infinity)).toThrow(NOT_A_STRING); ++ expect(() => buf[method](123, 0, -1)).toThrow(NOT_A_STRING); ++ expect(() => buf[method](123, 9, 1)).toThrow(NOT_A_STRING); ++ expect(() => buf[method](123, 0, 0)).toThrow(NOT_A_STRING); ++ expect(() => buf[method](123, NaN, NaN)).toThrow(NOT_A_STRING); ++ ++ const converted = []; ++ const arg = name => ({ ++ valueOf() { ++ converted.push(name); ++ throw new Error(`${name} valueOf`); ++ }, ++ }); ++ expect(() => buf[method](123, arg("offset"), arg("length"))).toThrow(NOT_A_STRING); ++ expect(converted).toEqual([]); ++ expect(buf.toString("hex")).toBe(untouched); ++ }); ++ }); ++ + it("the documented write() wrapper is unchanged", () => { + const buf = dest(); + expect(() => buf.write("hello", 6, 1000)).toThrow(expect.objectContaining({ code: "ERR_OUT_OF_RANGE" })); +diff --git a/test/js/node/tls/node-tls-duplex-end-verify.test.ts b/test/js/node/tls/node-tls-duplex-end-verify.test.ts +index 81dd7b37..f4a8c29d 100644 +--- a/test/js/node/tls/node-tls-duplex-end-verify.test.ts ++++ b/test/js/node/tls/node-tls-duplex-end-verify.test.ts +@@ -981,3 +981,154 @@ test("a bad record behind the client's Finished does not make a server accept an + // Node reports the bad record. Its code depends on the cipher, so only the class of the error is fixed. + assert.match(events[0], isBun ? /^tlsClientError DEPTH_ZERO_SELF_SIGNED_CERT$/ : /^tlsClientError ERR_SSL_/); + }); ++ ++// A client calls end() before the first step of its handshake. Returns the ordered events of the client. ++async function endBeforeClientHello( ++ when, ++ maxVersion, ++ rejectUnauthorized, ++ { trusted = false, servername = "agent1" } = {}, ++) { ++ const events = []; ++ const { promise, resolve } = Promise.withResolvers(); ++ const server = tls.createServer({ key, cert, maxVersion }, socket => socket.on("error", () => {})); ++ server.on("tlsClientError", () => {}); ++ await new Promise(listening => server.listen(0, "127.0.0.1", listening)); ++ const client = tls.connect({ ++ port: server.address().port, ++ host: "127.0.0.1", ++ servername, ++ rejectUnauthorized, ++ maxVersion, ++ ...(trusted && { ca: serverCA }), ++ }); ++ if (when === "in the same tick") client.end(); ++ else if (when === "in the next tick") process.nextTick(() => client.end()); ++ else client.on("connect", () => client.end()); ++ for (const event of ["finish", "secureConnect", "end"]) client.on(event, () => events.push(event)); ++ client.on("error", err => events.push(`error ${err.code}`)); ++ client.on("close", () => { ++ events.push("close"); ++ resolve(); ++ }); ++ await promise; ++ server.close(); ++ return events; ++} ++ ++for (const when of ["in the same tick", "in the next tick", "inside 'connect'"]) { ++ test(`TLSv1.3: end() ${when} still refuses an untrusted certificate`, async () => { ++ assert.deepStrictEqual(await endBeforeClientHello(when, "TLSv1.3", true), [ ++ "finish", ++ "error UNABLE_TO_VERIFY_LEAF_SIGNATURE", ++ "close", ++ ]); ++ }); ++ ++ test(`TLSv1.3: end() ${when} still refuses a certificate for another name`, async () => { ++ const wrongName = { trusted: true, servername: "another.name" }; ++ assert.deepStrictEqual(await endBeforeClientHello(when, "TLSv1.3", true, wrongName), [ ++ "finish", ++ "error ERR_TLS_CERT_ALTNAME_INVALID", ++ "close", ++ ]); ++ }); ++ ++ test(`TLSv1.3: end() ${when} still completes the handshake`, async () => { ++ assert.deepStrictEqual(await endBeforeClientHello(when, "TLSv1.3", false), [ ++ "finish", ++ "secureConnect", ++ "end", ++ "close", ++ ]); ++ }); ++ ++ test(`TLSv1.2: end() ${when} reports the handshake that the server cannot complete`, async () => { ++ // The client cannot send its second flight after the FIN, so the handshake ends when the server closes. ++ assert.deepStrictEqual(await endBeforeClientHello(when, "TLSv1.2", false), [ ++ "finish", ++ "end", ++ "error ECONNRESET", ++ "close", ++ ]); ++ }); ++} ++ ++test("end() inside 'connect' still reports a ClientHello that the client cannot build", async () => { ++ const events = []; ++ const { promise, resolve } = Promise.withResolvers(); ++ const server = tls.createServer({ key, cert }, socket => socket.on("error", () => {})); ++ server.on("tlsClientError", () => {}); ++ await new Promise(listening => server.listen(0, "127.0.0.1", listening)); ++ // No protocol version is inside this window, so the first step of the handshake fails. ++ const client = tls.connect({ ++ port: server.address().port, ++ host: "127.0.0.1", ++ rejectUnauthorized: false, ++ minVersion: "TLSv1.3", ++ maxVersion: "TLSv1.2", ++ }); ++ client.on("connect", () => client.end()); ++ client.on("secureConnect", () => events.push("secureConnect")); ++ client.on("error", err => events.push(`error ${err.code}`)); ++ client.on("close", () => { ++ events.push("close"); ++ resolve(); ++ }); ++ await promise; ++ server.close(); ++ // OpenSSL and BoringSSL name the reason differently. ++ assert.match(events.join(", "), /^error ERR_SSL_NO_(PROTOCOLS_AVAILABLE|SUPPORTED_VERSIONS_ENABLED), close$/); ++}); ++ ++test("end() inside 'connect' sends the ClientHello before the FIN", async () => { ++ const { promise, resolve, reject } = Promise.withResolvers(); ++ let accepted; ++ const server = net.createServer({ allowHalfOpen: true }, socket => { ++ accepted = socket; ++ const received = []; ++ socket.on("error", reject); ++ socket.on("data", chunk => received.push(chunk)); ++ socket.on("end", () => resolve(Buffer.concat(received))); ++ }); ++ await new Promise(listening => server.listen(0, "127.0.0.1", listening)); ++ const client = tls.connect({ port: server.address().port, host: "127.0.0.1", rejectUnauthorized: false }); ++ client.on("error", () => {}); ++ client.on("connect", () => client.end()); ++ const beforeFin = await promise; ++ client.destroy(); ++ accepted.destroy(); ++ server.close(); ++ // One complete handshake record: the ClientHello. ++ assert.deepStrictEqual( ++ { type: beforeFin[0], complete: beforeFin.length >= 5 && beforeFin.length === 5 + beforeFin.readUInt16BE(3) }, ++ { type: 22, complete: true }, ++ ); ++}); ++ ++test("end() after a second connect() of the same socket sends no ClientHello", async () => { ++ // Only tls.connect() starts a handshake: https://github.com/nodejs/node/blob/v26.3.0/lib/internal/tls/wrap.js#L1795 ++ const received = []; ++ const second = Promise.withResolvers(); ++ const server = net.createServer(socket => { ++ let bytes = 0; ++ socket.on("error", second.reject); ++ socket.on("data", chunk => (bytes += chunk.length)); ++ socket.on("end", () => received.push(bytes) === 2 && second.resolve()); ++ }); ++ await new Promise(listening => server.listen(0, "127.0.0.1", listening)); ++ const where = { port: server.address().port, host: "127.0.0.1" }; ++ const client = tls.connect({ ...where, rejectUnauthorized: false }); ++ client.on("error", () => {}); ++ client.end(); ++ await new Promise(closed => client.once("close", closed)); ++ client.connect(where); ++ client.end(); ++ try { ++ await second.promise; ++ assert.strictEqual(received[1], 0); ++ } finally { ++ client.destroy(); ++ server.close(); ++ } ++}); +diff --git a/test/js/web/atomics.test.ts b/test/js/web/atomics.test.ts +index a36b5cf8..673821d3 100644 +--- a/test/js/web/atomics.test.ts ++++ b/test/js/web/atomics.test.ts +@@ -1,4 +1,5 @@ + import { describe, expect, test } from "bun:test"; ++import { bunEnv, bunExe, isASAN } from "harness"; + + describe("Atomics", () => { + describe("basic operations", () => { +@@ -307,3 +308,118 @@ describe("Atomics", () => { + }); + }); + }); ++ ++// Free blocks inside pages that are still in use belong to the thread that owns the pages. They go back to the OS ++// when that thread tells mimalloc that it is idle, which a wait that takes a while does. ++test.skipIf(isASAN /* malloc is not mimalloc */)( ++ "Atomics.wait lets mimalloc release this thread's free memory", ++ async () => { ++ await using proc = Bun.spawn({ ++ cmd: [ ++ bunExe(), ++ "-e", ++ ` ++ const { heapStats } = require("bun:jsc"); ++ const purgeCalls = () => heapStats().mimalloc.purge_calls; ++ const spin = ms => { const start = performance.now(); while (performance.now() - start < ms); }; ++ ++ // the characters of these strings are allocated and freed by this thread ++ let strings = []; ++ for (let i = 0; i < 100000; i++) strings.push(Buffer.alloc(900 + (i % 5) * 8, 97).toString("latin1")); ++ // (far enough apart that whole OS pages are free in between, also where those are 16 KB) ++ strings = strings.filter((_, i) => i % 64 === 0); ++ Bun.gc(true); ++ ++ // what needs no idle thread settles first, without going idle ++ let before = purgeCalls(); ++ for (let stable = 0, tries = 0; stable < 3 && tries < 50; tries++) { ++ spin(60); ++ const now = purgeCalls(); ++ stable = now === before ? stable + 1 : 0; ++ before = now; ++ } ++ ++ const view = new Int32Array(new SharedArrayBuffer(4)); ++ let released = 0; ++ for (let i = 0; i < 10 && released < 500; i++) { ++ if (Atomics.wait(view, 0, 0, 250) !== "timed-out") throw new Error("unexpected result"); ++ released = purgeCalls() - before; ++ } ++ console.log(released >= 500, strings.length); ++ `, ++ ], ++ env: bunEnv, ++ stderr: "pipe", ++ }); ++ const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); ++ expect({ stdout, stderr, exitCode }).toEqual({ stdout: "true 1563\n", stderr: "", exitCode: 0 }); ++ }, ++); ++ ++// 100 ms into a wait the waiter drops the lock of the waiter list to release its memory, and takes it again. ++test.skipIf(isASAN /* malloc is not mimalloc */)( ++ "a notify that arrives while Atomics.wait releases this thread's free memory is not lost", ++ async () => { ++ await using proc = Bun.spawn({ ++ cmd: [ ++ bunExe(), ++ "-e", ++ ` ++ const view = new Int32Array(new SharedArrayBuffer(16)); ++ const [VALUE, WAITING, LAST] = [0, 1, 2]; ++ const worker = new Worker( ++ URL.createObjectURL( ++ new Blob( ++ [ ++ \` ++ self.onmessage = event => { ++ const view = new Int32Array(event.data); ++ const results = []; ++ for (;;) { ++ // Free memory in between what is in use, so that the release has something to do and takes a while. ++ let strings = []; ++ for (let i = 0; i < 100000; i++) strings.push(Buffer.alloc(900 + (i % 5) * 8, 97).toString("latin1")); ++ strings = strings.filter((_, i) => i % 64 === 0); ++ Bun.gc(true); ++ Atomics.store(view, 1, 1); ++ // A waiter that misses its notification is off the list already: it sleeps for the whole ++ // timeout and then still answers "ok". ++ const start = performance.now(); ++ const result = Atomics.wait(view, 0, 0, 10000); ++ results.push(performance.now() - start > 5000 ? "late" : result); ++ Atomics.store(view, 0, 0); ++ if (Atomics.load(view, 2)) return postMessage(results); ++ } ++ }; ++ \`, ++ ], ++ { type: "application/javascript" }, ++ ), ++ ), ++ ); ++ worker.postMessage(view.buffer); ++ const results = new Promise(resolve => (worker.onmessage = event => resolve(event.data))); ++ const delays = []; ++ for (let delay = 100; delay <= 107; delay += 1) delays.push(delay); ++ for (const delay of delays) { ++ while (Atomics.load(view, WAITING) !== 1); ++ Atomics.store(view, WAITING, 0); ++ const start = performance.now(); ++ while (performance.now() - start < delay); ++ if (delay === delays.at(-1)) Atomics.store(view, LAST, 1); ++ Atomics.store(view, VALUE, 1); ++ Atomics.notify(view, VALUE); ++ } ++ // ("not-equal" if the worker was held up for that long before it got to wait) ++ console.log(JSON.stringify((await results).filter(result => result !== "ok" && result !== "not-equal")), delays.length); ++ process.exit(0); ++ `, ++ ], ++ env: bunEnv, ++ stderr: "pipe", ++ }); ++ const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); ++ expect({ stdout, stderr, exitCode }).toEqual({ stdout: "[] 8\n", stderr: "", exitCode: 0 }); ++ }, ++ 30_000, ++); +diff --git a/test/js/web/fetch/body-clone.test.ts b/test/js/web/fetch/body-clone.test.ts +index 2b1a4094..19e732ce 100644 +--- a/test/js/web/fetch/body-clone.test.ts ++++ b/test/js/web/fetch/body-clone.test.ts +@@ -1,5 +1,6 @@ + import { describe, expect, test } from "bun:test"; +-import { bunEnv, bunExe, isASAN, isDebug, isWindows, tempDirWithFiles } from "harness"; ++import { bunEnv, bunExe, isASAN, isDebug, isLinux, isWindows, tempDirWithFiles } from "harness"; ++import { closeSync, openSync, unlinkSync, utimesSync, writeFileSync } from "node:fs"; + import net from "node:net"; + import { join } from "node:path"; + +@@ -1171,6 +1172,18 @@ describe("clone() of a body over an unread native stream keeps the Blob behind i + expect(await cloneInChild("Bun.stdin")).toEqual(bothBodiesReadStdin); + }); + ++ // Reads of one fd share its offset, so the second Blob would start where the first one ended. ++ test("a body over a regular file given by file descriptor is teed, not duped", async () => { ++ const fd = openSync(join(tempDirWithFiles("body-clone-fd", { "a.txt": "hello world" }), "a.txt"), "r"); ++ try { ++ const original = new Response(Bun.file(fd)); ++ const clone = original.clone(); ++ expect(await Promise.all([original.text(), clone.text()])).toEqual(["hello world", "hello world"]); ++ } finally { ++ closeSync(fd); ++ } ++ }); ++ + // The same store kind reached by path: stat says it is not a regular file. + test.skipIf(isWindows)("a body over a FIFO opened by path is still teed", async () => { + const fifo = join(tempDirWithFiles("body-clone-fifo", {}), "body.fifo"); +@@ -1183,6 +1196,159 @@ describe("clone() of a body over an unread native stream keeps the Blob behind i + expect(await cloneInChild("Bun.file(process.argv.at(-1)).stream()", [fifo])).toEqual(bothBodiesReadStdin); + expect(await writer.exited).toBe(0); + }); ++ ++ test.skipIf(isWindows)("a body over a FIFO given by path as the Blob itself is teed, not duped", async () => { ++ const fifo = join(tempDirWithFiles("body-clone-fifo-blob", {}), "body.fifo"); ++ expect(Bun.spawnSync({ cmd: ["mkfifo", fifo] }).exitCode).toBe(0); ++ await using writer = Bun.spawn({ ++ cmd: ["sh", "-c", `printf 'hello world' > "$1"`, "sh", fifo], ++ stdout: "ignore", ++ stderr: "inherit", ++ }); ++ expect(await cloneInChild("Bun.file(process.argv.at(-1))", [fifo])).toEqual(bothBodiesReadStdin); ++ expect(await writer.exited).toBe(0); ++ }); ++ ++ // The body shares its store with the Bun.file() it was made from. A stat cached ++ // there is what `size`, `lastModified`, `exists()` and a later `.body` answer from. ++ describe("clone() of a Bun.file() body does not change what the Bun.file() answers", () => { ++ const post = (body: Bun.BunFile) => new Request("http://example.com/", { method: "POST", body }); ++ const cloners: Array<[string, (file: Bun.BunFile) => unknown]> = [ ++ ["Response.clone()", file => new Response(file).clone()], ++ ["Request.clone()", file => post(file).clone()], ++ ["new Request(request)", file => new Request(post(file))], ++ ["new Request(request, init)", file => new Request(post(file), { headers: { "x-test": "1" } })], ++ // @ts-expect-error Bun takes a Response as init and clones its body. ++ ["new Request(url, response)", file => new Request("http://example.com/", new Response(file))], ++ ]; ++ ++ async function afterClone( ++ clone: (file: Bun.BunFile) => unknown, ++ change: (path: string) => void, ++ read: (file: Bun.BunFile) => T | Promise, ++ ) { ++ const path = join(tempDirWithFiles("body-clone-answers", { "log.txt": "12345" }), "log.txt"); ++ const file = Bun.file(path); ++ clone(file); ++ change(path); ++ return await read(file); ++ } ++ const grow = (path: string) => writeFileSync(path, "123456789"); ++ ++ test.each(cloners)("%s: size and a later body see a file that grew", async (_, clone) => { ++ expect({ ++ size: await afterClone(clone, grow, file => file.size), ++ body: await afterClone(clone, grow, file => Bun.readableStreamToText(new Response(file).body!)), ++ }).toEqual({ size: 9, body: "123456789" }); ++ }); ++ ++ test("lastModified and exists() see a file that changed", async () => { ++ const clone = cloners[0][1]; ++ const mtime = new Date("2033-05-18T03:33:20.000Z"); ++ expect({ ++ lastModified: await afterClone( ++ clone, ++ path => utimesSync(path, mtime, mtime), ++ file => file.lastModified, ++ ), ++ exists: await afterClone(clone, unlinkSync, file => file.exists()), ++ }).toEqual({ lastModified: mtime.getTime(), exists: false }); ++ }); ++ ++ test("both bodies see a file that grew", async () => { ++ const path = join(tempDirWithFiles("body-clone-grew", { "log.txt": "12345" }), "log.txt"); ++ const original = new Response(Bun.file(path)); ++ const clone = original.clone(); ++ grow(path); ++ expect(await Promise.all([original.body, clone.body].map(body => Bun.readableStreamToText(body!)))).toEqual([ ++ "123456789", ++ "123456789", ++ ]); ++ }); ++ ++ test("exists() sees a file that was created before the clone", async () => { ++ const path = join(tempDirWithFiles("body-clone-created", {}), "later.txt"); ++ const file = Bun.file(path); ++ expect(await file.exists()).toBe(false); ++ writeFileSync(path, "12345"); ++ new Response(file).clone(); ++ expect(await file.exists()).toBe(true); ++ }); ++ ++ test("the size console.log prints for the clone", () => { ++ const sizeOf = (change: (path: string) => void, body: (file: Bun.BunFile) => Blob) => { ++ const path = join(tempDirWithFiles("body-clone-label", { "log.txt": "12345" }), "log.txt"); ++ const clone = new Response(body(Bun.file(path))).clone(); ++ change(path); ++ return [Bun.inspect(clone), Bun.inspect(clone)].map(printed => printed.split("\n")[0]).join(" "); ++ }; ++ expect({ ++ whole: sizeOf( ++ () => {}, ++ file => file, ++ ), ++ slice: sizeOf( ++ () => {}, ++ file => file.slice(1, 4), ++ ), ++ grew: sizeOf(grow, file => file), ++ deleted: sizeOf(unlinkSync, file => file), ++ }).toEqual({ ++ whole: "Response (5 bytes) { Response (5 bytes) {", ++ slice: "Response (3 bytes) { Response (3 bytes) {", ++ grew: "Response (9 bytes) { Response (9 bytes) {", ++ deleted: "Response { Response {", ++ }); ++ }); ++ ++ // procfs reports `st_size == 0` for a file that has content, so with that stat cached all three read "". ++ test.skipIf(!isLinux)("a procfs file is read whole", async () => { ++ const file = Bun.file("/proc/sys/kernel/ostype"); ++ const original = new Response(file); ++ const clone = original.clone(); ++ expect({ ++ original: await original.text(), ++ clone: await clone.text(), ++ file: await file.text(), ++ }).toEqual({ original: "Linux\n", clone: "Linux\n", file: "Linux\n" }); ++ }); ++ }); ++ ++ // No bytes to compete for: it is duped like a regular file, and each body fails when it is read. ++ describe("clone() of a body that cannot be read does not throw", () => { ++ const sizeRead = (file: Bun.BunFile) => (file.size, file); ++ const directory = () => tempDirWithFiles("body-clone-dir", {}); ++ ++ async function codes(file: Bun.BunFile) { ++ const original = new Response(file); ++ const clone = original.clone(); ++ const results = await Promise.allSettled([original.text(), clone.text()]); ++ return results.map(result => (result.status === "rejected" ? result.reason?.code : result.status)); ++ } ++ ++ test("a directory by path", async () => { ++ expect({ ++ fresh: await codes(Bun.file(directory())), ++ sizeRead: await codes(sizeRead(Bun.file(directory()))), ++ }).toEqual({ fresh: ["EISDIR", "EISDIR"], sizeRead: ["EISDIR", "EISDIR"] }); ++ }); ++ ++ test.skipIf(isWindows)("a directory by file descriptor", async () => { ++ const fd = openSync(directory(), "r"); ++ try { ++ expect({ ++ fresh: await codes(Bun.file(fd)), ++ sizeRead: await codes(sizeRead(Bun.file(fd))), ++ }).toEqual({ fresh: ["EISDIR", "EISDIR"], sizeRead: ["EISDIR", "EISDIR"] }); ++ } finally { ++ closeSync(fd); ++ } ++ }); ++ ++ test.skipIf(isWindows)("a file descriptor that is not open", async () => { ++ expect(await codes(Bun.file(1_000_000))).toEqual(["EBADF", "EBADF"]); ++ }); ++ }); + }); + + test("Blob type from a consumed Response keeps the original content-type after clones with different content-types are consumed", async () => { +diff --git a/test/js/web/timers/timer-gc-roots.test.ts b/test/js/web/timers/timer-gc-roots.test.ts +index 6f5576f2..a88f55d3 100644 +--- a/test/js/web/timers/timer-gc-roots.test.ts ++++ b/test/js/web/timers/timer-gc-roots.test.ts +@@ -133,6 +133,9 @@ describe.concurrent("AbortSignal.timeout is released when its wrapper is collect + await new Promise(r => setTimeout(r, 10)); + } + } ++ // RSS still climbs to its plateau in the first rounds ++ await round(); ++ await round(); + await round(); + const before = process.memoryUsage().rss; + await round(); +diff --git a/test/regression/issue/17190.test.ts b/test/regression/issue/17190.test.ts +new file mode 100644 +index 00000000..ff045e32 +--- /dev/null ++++ b/test/regression/issue/17190.test.ts +@@ -0,0 +1,26 @@ ++// https://github.com/oven-sh/bun/issues/17190 ++import { expect, test } from "bun:test"; ++import { tls as selfSigned } from "harness"; ++import { once } from "node:events"; ++import type { AddressInfo } from "node:net"; ++import tls from "node:tls"; ++ ++test("tls.connect() followed at once by end() reports a self-signed certificate", async () => { ++ const server = tls.createServer(selfSigned, socket => socket.on("error", () => {})); ++ server.on("tlsClientError", () => {}); ++ await once(server.listen(0, "127.0.0.1"), "listening"); ++ try { ++ const socket = tls.connect({ ++ host: "127.0.0.1", ++ port: (server.address() as AddressInfo).port, ++ rejectUnauthorized: true, ++ }); ++ const { promise, resolve } = Promise.withResolvers(); ++ socket.on("error", err => resolve(`error ${(err as NodeJS.ErrnoException).code}`)); ++ socket.on("close", () => resolve("close with no error")); ++ socket.end(); ++ expect(await promise).toBe("error DEPTH_ZERO_SELF_SIGNED_CERT"); ++ } finally { ++ server.close(); ++ } ++}); diff --git a/.github/openclaw/qualification/verify-sync.py b/.github/openclaw/qualification/verify-sync.py new file mode 100644 index 0000000000000..30829451c9eab --- /dev/null +++ b/.github/openclaw/qualification/verify-sync.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Require complete upstream-sync and patch evidence before release assembly.""" +import hashlib,json,re,sys +from pathlib import Path +root=Path(sys.argv[1]);out=root/'sync' +selected=json.loads((out/'selected.json').read_text()) +rows=json.loads((out/'results.json').read_text()) +assert selected and len(selected)==len(set(selected)) +assert set(selected)<={r['testPath'] for r in rows} +assert rows and all(r['ok'] for r in rows),'sync CI failures' +assert {'test/js/web/atomics.test.ts','test/js/bun/wasm/compile-rss.test.ts'}<=set(selected) +counts={} +for name,minimum in [('memory-release',3),('proxy',4),('resource-limits',14),('namespace',41),('inspector',78),('als-suites',223)]: + text=(out/(name+'.log')).read_text() + passes=re.findall(r'(\d+) pass',text) + assert passes and int(passes[-1])>=minimum,name+' incomplete' + assert re.search(r'\b0 fail\b',text),name+' failed' + if name=='memory-release':assert not re.search(r'\b[1-9]\d* skip\b',text),'memory tests skipped' + counts[name]=int(passes[-1]) +for name in ['als-plugin','als-native','als-gc']: + matrix=json.loads((out/(name+'.json')).read_text())['results'] + assert len(matrix)==14 and all(r['exitCode']==0 for r in matrix),name +assert json.loads((out/'als-retention.json').read_text())['passed'] is True +assert len((out/'engine-limits.log').read_text().splitlines())==7 +engine=json.loads((root/'engine-gate.json').read_text()) +assert engine['passed'] and engine['source']==sys.argv[2] +record={'passed':True,'base_commit':sys.argv[3],'prepared_tree':sys.argv[4], + 'selected_files':len(selected),'result_rows':len(rows),'regressions':0,'tests':counts, + 'adapter_sha256':hashlib.sha256((root/'bun-sync-adapters.patch').read_bytes()).hexdigest()} +(root/'sync-gate.json').write_text(json.dumps(record,indent=2)+'\n') +gate=json.loads((root/'gate.json').read_text());gate['upstream_sync']=record;gate['engine']=engine +(root/'gate.json').write_text(json.dumps(gate,indent=2)+'\n') diff --git a/.github/openclaw/qualify-engine.sh b/.github/openclaw/qualify-engine.sh new file mode 100644 index 0000000000000..a45ae797c1fb3 --- /dev/null +++ b/.github/openclaw/qualify-engine.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail +: "${ARTIFACT_DIR:?}" "${QUALIFICATION_DIR:?}" +mkdir -p "$QUALIFICATION_DIR/engine" +QUALIFICATION_DIR=$(realpath "$QUALIFICATION_DIR") +source_sha=$(git rev-parse HEAD) +tar -xzf "$ARTIFACT_DIR/bun-webkit-linux-amd64.tar.gz" -C "$QUALIFICATION_DIR/engine" +jsc="$QUALIFICATION_DIR/engine/bun-webkit/bin/jsc" +"$jsc" --useDollarVM=true -e 'if(typeof $vm.moduleNamespaceForRequire!=="function")throw Error("namespace hook absent")' +Tools/Scripts/run-jsc-stress-tests JSTests/stress --jsc "$jsc" --child-processes 16 \ + --filter 'promise|microtask|dynamic-import|bun-async-context|module-loader|top-level-await|async-module|module-namespace|intl-segmenter' \ + --output-dir "$QUALIFICATION_DIR/jsc-stress" --no-retry > "$QUALIFICATION_DIR/jsc-stress.log" 2>&1 +python3 .github/openclaw/verify-jsc-results.py "$QUALIFICATION_DIR/jsc-stress" +Tools/Scripts/run-jsc-stress-tests JSTests/modules.yaml --jsc "$jsc" --child-processes 16 \ + --output-dir "$QUALIFICATION_DIR/jsc-modules" --no-retry > "$QUALIFICATION_DIR/jsc-modules.log" 2>&1 +"$jsc" --useDollarVM=true --useImportDefer=true .github/openclaw/qualification/als/jsc/module-context.js > "$QUALIFICATION_DIR/jsc-context.log" 2>&1 +fixture=JSTests/stress/heap-allocation-sampling.js +common=(--useDollarVM=true --useGenerationalGC=true --forceMiniVMMode=false) +"$jsc" "${common[@]}" "$fixture" > "$QUALIFICATION_DIR/sampling-default.log" 2>&1 +"$jsc" "${common[@]}" --useJIT=false "$fixture" > "$QUALIFICATION_DIR/sampling-interpreter.log" 2>&1 +"$jsc" "${common[@]}" --useDFGJIT=false --useFTLJIT=false "$fixture" > "$QUALIFICATION_DIR/sampling-baseline.log" 2>&1 +"$jsc" "${common[@]}" --useConcurrentJIT=false --thresholdForOptimizeAfterWarmUp=10 --thresholdForFTLOptimizeAfterWarmUp=20 "$fixture" > "$QUALIFICATION_DIR/sampling-ftl.log" 2>&1 +python3 - "$QUALIFICATION_DIR" "$source_sha" <<'PY' +import json,sys +from pathlib import Path +root=Path(sys.argv[1]);counts={} +for name,minimum in [('jsc-stress',1400),('jsc-modules',1600)]: + lines=(root/name/'results').read_text().splitlines() + assert len(lines)>=minimum,(name,len(lines)) + assert all(line.endswith(': PASS') for line in lines),name+' failures' + counts[name]=len(lines) +(root/'engine-gate.json').write_text(json.dumps({'passed':True,'source':sys.argv[2],'stress_counts':counts,'sampling_tiers':4},indent=2)+'\n') +PY diff --git a/.github/openclaw/qualify-sync.sh b/.github/openclaw/qualify-sync.sh new file mode 100644 index 0000000000000..1e232ad308b41 --- /dev/null +++ b/.github/openclaw/qualify-sync.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +set -euo pipefail +pipeline=$(pwd) +inputs="$pipeline/.github/openclaw/qualification" +: "${ARTIFACT_DIR:?}" "${QUALIFICATION_DIR:?}" +QUALIFICATION_DIR=$(realpath "$QUALIFICATION_DIR") +SOURCE_SHA=$(git rev-parse HEAD) +BUN_BASE=d2d2a26ef973cdd97b37953f5dba58052acad74f +SYNC_TREE=d9ec052fccc81afbf43a5344e52bc980f439d587 +bootstrap=$(command -v bun) +export CI=1 PUPPETEER_SKIP_DOWNLOAD=1 +repo="$QUALIFICATION_DIR/bun-sync" +results="$QUALIFICATION_DIR/sync" +mkdir -p "$results/home" "$results/tmp" "$results/bin" +chmod 700 "$results/home" "$results/tmp" +git clone --depth=1 --filter=blob:none --no-checkout https://github.com/openclaw/bun.git "$repo" +cd "$repo" +git fetch --depth=1 origin "$BUN_BASE" +git checkout --detach "$BUN_BASE" +git apply --index "$inputs/sync/upstream-sync.patch" +test "$(git write-tree)" = "$SYNC_TREE" +# The complete W150 prepared tree is verified before qualification-only adapters. +for patch in "$inputs"/patches/00[3-7]-*.patch; do + git apply --index --exclude=CHANGELOG.md --exclude='docs/*' --exclude='test/js/node/worker_threads/*' "$patch" +done +for patch in 008-namespace.patch 009-allocation-sampling.patch 010-als-regressions.patch; do + git apply --index --exclude=CHANGELOG.md --exclude='docs/*' "$inputs/patches/$patch" +done +cp "$inputs/worker-resource-limits.test.ts" test/js/node/worker_threads/openclaw-resource-limits.test.ts +git add test/js/node/worker_threads/openclaw-resource-limits.test.ts +python3 "$inputs/stage-sync-artifact.py" "$ARTIFACT_DIR/bun-webkit-linux-amd64.tar.gz" --version "$SOURCE_SHA" --output "$QUALIFICATION_DIR/sync-input" +cp "$QUALIFICATION_DIR/sync-input/artifacts.json" scripts/build/deps/webkit-artifacts.json +git add scripts/build/deps/webkit-artifacts.json +git diff --check HEAD +git diff --binary HEAD > "$QUALIFICATION_DIR/bun-sync-adapters.patch" +"$bootstrap" install --frozen-lockfile +(cd test && "$bootstrap" install --frozen-lockfile) +export BUN_BUILD_PREFETCH_DIR="$QUALIFICATION_DIR/sync-input/prefetch" +export BUN_BUILD_CACHE_DIR="$QUALIFICATION_DIR/cache-sync" +env -u GITHUB_SHA -u BUILDKITE_COMMIT -u GIT_SHA "$bootstrap" run build:release --lto=off --buildDir="$repo/build/qualify-sync" --timings > "$QUALIFICATION_DIR/build-sync.log" 2>&1 +candidate="$repo/build/qualify-sync/bun" +"$candidate" -e 'if(process.versions.webkit!==process.argv[1] || Bun.revision!==process.argv[2]) throw new Error("wrong engine or Bun source revision");' "$SOURCE_SHA" "$BUN_BASE" +sha256sum "$candidate" > "$QUALIFICATION_DIR/bun-sync.sha256" +"$candidate" --revision > "$results/revision.log" +"$candidate" -e 'const server=Bun.serve({port:0,fetch(){return new Response("ready")}});try{if(await(await fetch(server.url)).text()!=="ready")throw Error("startup");console.log("build/startup/http passed")}finally{await server.stop(true)}' > "$results/startup.log" 2>&1 +unset BUN_BUILD_PREFETCH_DIR BUN_BUILD_CACHE_DIR BUN_WEBKIT_ARTIFACT_MANIFEST +ln -s "$candidate" "$results/bin/bun" +export HOME="$results/home" TMPDIR="$results/tmp" PATH="$results/bin:$PATH" OPENCLAW_CI_PLATFORM=linux +# Recompute the engine/nightly selection and retain every W150 upstream-sync case. +"$candidate" --eval 'import {selectTests} from "./scripts/openclaw-ci/tests.ts"; import{execFileSync}from"node:child_process";import{readFileSync,writeFileSync,mkdirSync}from"node:fs";const tracked=execFileSync("git",["ls-files","-z","test"],{encoding:"utf8"}).split("\0").filter(Boolean);const original=JSON.parse(readFileSync(process.argv[1],"utf8"));const selected=[...new Set([...original,...selectTests(["scripts/build/deps/webkit.ts"],tracked,true,"linux")])].sort();for(const file of selected)if(!tracked.includes(file))throw Error("missing selected test "+file);mkdirSync("build/openclaw-ci",{recursive:true});writeFileSync("build/openclaw-ci/selected.json",JSON.stringify(selected,null,2)+"\n");' "$inputs/sync/selected.json" +cp build/openclaw-ci/selected.json "$results/selected.json" +"$candidate" scripts/openclaw-ci/tests.ts test > "$results/tests.log" 2>&1 +cp build/openclaw-ci/results.json "$results/results.json" +# Run these unchanged upstream tests explicitly as well, so skips cannot satisfy the gate. +"$candidate" test test/js/web/atomics.test.ts test/js/bun/wasm/compile-rss.test.ts -t 'Atomics.wait lets mimalloc|a notify that arrives while Atomics.wait|WebAssembly.compile does not retain' > "$results/memory-release.log" 2>&1 +"$candidate" "$inputs/segmenter.js" > "$results/segmenter.log" 2>&1 +"$candidate" test test/js/node/vm/vm.test.ts -t 'a Proxy in the prototype chain' > "$results/proxy.log" 2>&1 +"$candidate" test test/js/node/worker_threads/openclaw-resource-limits.test.ts -t resourceLimits > "$results/resource-limits.log" 2>&1 +"$candidate" test test/js/bun/resolve/esModule.test.ts test/js/bun/namespace-prototype-pollution.test.ts test/js/bun/resolve/require-esm-evaluating-cycle.test.ts test/js/bun/import-attributes/import-attributes.test.ts test/js/node/module/require-extensions.test.ts > "$results/namespace.log" 2>&1 +"$candidate" test test/js/node/inspector/inspector-profiler.test.ts test/js/node/inspector/inspector.test.ts > "$results/inspector.log" 2>&1 +"$candidate" test test/js/node/async_hooks test/js/bun/resolve/import-defer.test.ts test/js/bun/resolve/dynamic-import-tla-cycle.test.ts test/js/bun/resolve/require-esm-gc-roots.test.ts test/js/bun/resolve/require-esm-microtask-order.test.ts test/js/bun/resolve/require-esm-evaluating-cycle.test.ts test/js/bun/resolve/require-esm-transitive-tla.test.ts > "$results/als-suites.log" 2>&1 +"$candidate" "$inputs/module-context.mjs" matrix > "$results/als-plugin.json" +"$candidate" "$inputs/module-context.mjs" matrix --native-hooks > "$results/als-native.json" +BUN_JSC_collectContinuously=1 "$candidate" "$inputs/module-context.mjs" matrix > "$results/als-gc.json" +"$candidate" "$inputs/als/realm-require.mjs" > "$results/als-realm.log" 2>&1 +"$candidate" "$inputs/als/retention.mjs" > "$results/als-retention.json" 2>&1 +WK="$QUALIFICATION_DIR/engine/bun-webkit" +clang++-23 -std=c++23 -O2 -DNDEBUG -DBUILDING_WITH_CMAKE=1 -DHAVE_CONFIG_H=1 \ + -I"$WK/include" -I"$WK/include/JavaScriptCore" -I"$WK/include/wtf" \ + "$inputs/engine-limit-probe.cpp" "$repo/build/qualify-sync/obj/vendor/mimalloc/src/static.c.o" \ + -Wl,--start-group "$WK"/lib/*.a -Wl,--end-group -lpthread -ldl -lm -latomic \ + -o "$results/engine-limit-probe" > "$results/engine-limit-build.log" 2>&1 +"$results/engine-limit-probe" > "$results/engine-limits.log" 2>&1 +python3 "$inputs/verify-sync.py" "$QUALIFICATION_DIR" "$SOURCE_SHA" "$BUN_BASE" "$SYNC_TREE" diff --git a/.github/openclaw/qualify.sh b/.github/openclaw/qualify.sh index fb7fb33a22832..a99f800511136 100644 --- a/.github/openclaw/qualify.sh +++ b/.github/openclaw/qualify.sh @@ -23,6 +23,7 @@ bootstrap=$(command -v bun) python3 "$inputs/stage-local-artifact.py" "$ARTIFACT_DIR/bun-webkit-linux-amd64.tar.gz" --version "$SOURCE_SHA" --output "$QUALIFICATION_DIR/candidate-input" for arm in baseline candidate; do if test "$arm" = candidate; then + git apply "$inputs/patches/002-upstream-mimalloc.patch" for patch in "$inputs/patches/001-proxy.patch" "$inputs"/patches/00[3-7]-*.patch; do git apply --exclude=CHANGELOG.md --exclude='docs/*' --exclude='test/js/node/worker_threads/*' "$patch" done diff --git a/.github/openclaw/release-notes.md b/.github/openclaw/release-notes.md new file mode 100644 index 0000000000000..153f8cf150ab8 --- /dev/null +++ b/.github/openclaw/release-notes.md @@ -0,0 +1,14 @@ +This release rebases the qualified OpenClaw engine patches onto upstream `5718a6ec579b98362ea7276a426deedcc6281ef5`. + +- Release free allocator memory from idle compiler threads and long Atomics waits; thanks @Jarred-Sumner. +- Preserve Segmenter surrogate boundaries, opt-in Proxy global prototypes, module-loader AsyncLocalStorage context, and per-VM worker heap/stack budgets; thanks @steipete and @robobun. +- Give require(esm) a separate live namespace with its own `__esModule` marker while preserving unmarked import namespaces; thanks @robobun. +- Add VM-owned byte-based allocation sampling, allocating-stack profiles, and independent major/minor GC inclusion. + +Qualification includes all nine artifact builds, Linux JSC module/promise/namespace/Segmenter and allocation-sampling regressions, the W113 paired Bun selection, and a separate build of the prepared upstream-synced Bun source with the complete fork selection, patch regressions, startup checks, and unchanged upstream memory-release tests. Other targets have build/provenance proof; native runtime qualification is Linux x64. + +The new engine requires Bun's updated mimalloc idle hook and matching embedding adapters. Consumers must rebuild with the manifest's exact headers and libraries; an older executable is not interchangeable. + +Source and licenses: LICENSE-SOURCES.txt. Checksums: SHA256SUMS and manifest.json. Build and qualification evidence: provenance.tar.gz. + +Rollback by restoring the prior Bun manifest and WebKit pin together, then rebuilding. Prior releases remain available. diff --git a/.github/openclaw/release.py b/.github/openclaw/release.py index 9d057ee3cc2ee..9768e85acb658 100644 --- a/.github/openclaw/release.py +++ b/.github/openclaw/release.py @@ -27,6 +27,11 @@ def assemble(inputs,out,source): gate=json.loads(gate_path.read_text()) if gate.get('passed') is not True or gate['source']!=source or gate['bun_commit']!=config['bun_commit']: raise ValueError('missing or mismatched qualification gate') + sync=gate.get('upstream_sync',{}) + engine=gate.get('engine',{}) + if (sync.get('passed') is not True or any(sync.get(k)!=config['upstream_sync'][k] for k in ('base_commit','prepared_tree')) + or engine.get('passed') is not True or engine.get('source')!=source): + raise ValueError('missing or mismatched upstream-sync/engine qualification') artifacts={}; lanes=[] for lane in config['lanes']: label=lane['label']; name=label+'.tar.gz'; directory=inputs/label @@ -75,7 +80,7 @@ def authorize(source,run_id): if not api('-H','Cache-Control: max-age=0',f'repos/{REPO}/immutable-releases')['enabled']: raise ValueError('immutable releases must be enabled') run=api('-H','Cache-Control: max-age=0',f'repos/{REPO}/actions/runs/{run_id}') - if (run['head_sha']!=source or run['head_branch'] not in ('openclaw/main','openclaw/batch-1') + if (run['head_sha']!=source or run['head_branch'] not in ('openclaw/main','openclaw/batch-1','openclaw/release-5718a6ec') or run['event']!='workflow_dispatch' or run['path'].split('@',1)[0]!='.github/workflows/openclaw-artifacts.yml'): raise ValueError('publication run does not match the reviewed source/workflow') jobs=api('-H','Cache-Control: max-age=0',f'repos/{REPO}/actions/runs/{run_id}/jobs?filter=latest&per_page=100') @@ -106,11 +111,11 @@ def validate_authorization(source): def publish(directory,source): manifest=verify(directory,source);tag=manifest['tag'] validate_authorization(source) - if os.environ.get('GITHUB_REF') not in ('refs/heads/openclaw/main','refs/heads/openclaw/batch-1') or os.environ.get('GITHUB_SHA')!=source: + if os.environ.get('GITHUB_REF') not in ('refs/heads/openclaw/main','refs/heads/openclaw/batch-1','refs/heads/openclaw/release-5718a6ec') or os.environ.get('GITHUB_SHA')!=source: raise ValueError('publishing requires reviewed source on an explicitly permitted owned branch') # Tag creation fails atomically if the name exists. Neither existing drafts nor releases are resumed. api('-X','POST',f'repos/{REPO}/git/refs','-f','ref=refs/tags/'+tag,'-f','sha='+source) - body=f'OpenClaw-built WebKit at `{source}`. All selected lanes and Linux qualification passed.\n\nBatch 1: Segmenter surrogate boundaries; opt-in Proxy global prototypes; module-loader AsyncLocalStorage propagation; per-VM worker heap and stack budgets. Thanks @steipete and @robobun for the source changes and upstream work.\n\nSource and licenses: LICENSE-SOURCES.txt. Checksums: SHA256SUMS and manifest.json. Full build and test provenance: provenance.tar.gz.\n\nRollback by restoring the prior Bun manifest and WebKit pin together; prior releases remain available.\n' + body=f'OpenClaw-built WebKit at `{source}`. All nine lanes and Linux qualification passed.\n\n'+(ROOT/'.github/openclaw/release-notes.md').read_text() draft=api('-X','POST',f'repos/{REPO}/releases','-f','tag_name='+tag,'-f','name='+tag,'-f','body='+body,'-F','draft=true') release_id=draft['id'] execute('gh','release','upload',tag,'-R',REPO,*[str(p) for p in sorted(directory.iterdir())]) diff --git a/.github/openclaw/test_release.py b/.github/openclaw/test_release.py index a91e747ca5a06..36d561ea6c8d6 100644 --- a/.github/openclaw/test_release.py +++ b/.github/openclaw/test_release.py @@ -17,7 +17,7 @@ def setUp(self): self.root=Path(self.temp.name);self.inputs=self.root/'inputs';self.inputs.mkdir() self.out=self.root/'release' config=json.loads((release.ROOT/'.github/openclaw/variants.json').read_text()) - q=self.inputs/'qualification';q.mkdir();(q/'gate.json').write_text(json.dumps({'passed':True,'source':SOURCE,'bun_commit':config['bun_commit']})) + q=self.inputs/'qualification';q.mkdir();(q/'gate.json').write_text(json.dumps({'passed':True,'source':SOURCE,'bun_commit':config['bun_commit'],'upstream_sync':{'passed':True,**config['upstream_sync']},'engine':{'passed':True,'source':SOURCE}})) for lane in config['lanes']: label=lane['label'];d=self.inputs/label;d.mkdir();(d/'provenance').mkdir() f=d/(label+'.tar.gz');f.write_bytes(label.encode()) @@ -94,6 +94,24 @@ def test_authorization_accepts_protected_batch_branch(self): with patch.object(release,'api',side_effect=[{'enabled':True},run,jobs,pending,{}]),patch.object(release,'execute') as execute: release.authorize(SOURCE,42) self.assertEqual(execute.call_count,1) + def test_missing_upstream_sync_gate_rejected(self): + p=self.inputs/'qualification/gate.json';m=json.loads(p.read_text());del m['upstream_sync'];p.write_text(json.dumps(m)) + with self.assertRaisesRegex(ValueError,'upstream-sync'):self.assemble() + def test_wrong_sync_tree_rejected(self): + p=self.inputs/'qualification/gate.json';m=json.loads(p.read_text());m['upstream_sync']['prepared_tree']='b'*40;p.write_text(json.dumps(m)) + with self.assertRaisesRegex(ValueError,'upstream-sync'):self.assemble() + def test_wrong_engine_gate_source_rejected(self): + p=self.inputs/'qualification/gate.json';m=json.loads(p.read_text());m['engine']['source']='b'*40;p.write_text(json.dumps(m)) + with self.assertRaisesRegex(ValueError,'upstream-sync'):self.assemble() + def test_authorization_accepts_rebased_release_branch(self): + run={'head_sha':SOURCE,'head_branch':'openclaw/release-5718a6ec','event':'workflow_dispatch','path':'.github/workflows/openclaw-artifacts.yml','run_attempt':1} + config=json.loads((release.ROOT/'.github/openclaw/variants.json').read_text()) + names=['preflight','test','assemble']+['build ('+l['label']+')' for l in config['lanes']] + jobs={'jobs':[{'name':n,'conclusion':'success','run_attempt':1} for n in names]} + pending=[{'environment':{'name':'openclaw-artifacts','id':99},'current_user_can_approve':True}] + with patch.object(release,'api',side_effect=[{'enabled':True},run,jobs,pending,{}]),patch.object(release,'execute') as execute: + release.authorize(SOURCE,42) + self.assertEqual(execute.call_count,1) def test_authorization_rejects_other_branch(self): run={'head_sha':SOURCE,'head_branch':'feature/unreviewed','event':'workflow_dispatch','path':'.github/workflows/openclaw-artifacts.yml','run_attempt':1} with patch.object(release,'api',side_effect=[{'enabled':True},run]),patch.object(release,'execute') as execute: diff --git a/.github/openclaw/variants.json b/.github/openclaw/variants.json index 5220f2da23951..f6075d745dd8d 100644 --- a/.github/openclaw/variants.json +++ b/.github/openclaw/variants.json @@ -1,17 +1,49 @@ { "bun_repository": "openclaw/bun", "bun_commit": "b83b544ef744b6ab0dc1feadb59652261d64291d", - "base": "1600131e46b5af48bbda3559af8d8a3327230b6e", + "base": "5718a6ec579b98362ea7276a426deedcc6281ef5", "lanes": [ - {"label":"bun-webkit-linux-amd64","consumer":"openclaw-build-test.yml linux-x64: build:release --lto=off"}, - {"label":"bun-webkit-macos-arm64","consumer":"openclaw-build-test.yml darwin-arm64: build:release --lto=off"}, - {"label":"bun-webkit-linux-amd64-lto","consumer":"openclaw-release.yml default linux-x64"}, - {"label":"bun-webkit-linux-arm64-lto","consumer":"openclaw-release.yml default linux-arm64"}, - {"label":"bun-webkit-macos-arm64-lto","consumer":"openclaw-release.yml default darwin-arm64"}, - {"label":"bun-webkit-macos-amd64-lto","consumer":"openclaw-release.yml default darwin-x64"}, - {"label":"bun-webkit-linux-amd64-musl-lto","consumer":"openclaw-release.yml optional linux-x64-musl"}, - {"label":"bun-webkit-linux-arm64-musl-lto","consumer":"openclaw-release.yml optional linux-arm64-musl"}, - {"label":"bun-webkit-windows-amd64-lto","consumer":"openclaw-release.yml optional windows-x64"} + { + "label": "bun-webkit-linux-amd64", + "consumer": "openclaw-build-test.yml linux-x64: build:release --lto=off" + }, + { + "label": "bun-webkit-macos-arm64", + "consumer": "openclaw-build-test.yml darwin-arm64: build:release --lto=off" + }, + { + "label": "bun-webkit-linux-amd64-lto", + "consumer": "openclaw-release.yml default linux-x64" + }, + { + "label": "bun-webkit-linux-arm64-lto", + "consumer": "openclaw-release.yml default linux-arm64" + }, + { + "label": "bun-webkit-macos-arm64-lto", + "consumer": "openclaw-release.yml default darwin-arm64" + }, + { + "label": "bun-webkit-macos-amd64-lto", + "consumer": "openclaw-release.yml default darwin-x64" + }, + { + "label": "bun-webkit-linux-amd64-musl-lto", + "consumer": "openclaw-release.yml optional linux-x64-musl" + }, + { + "label": "bun-webkit-linux-arm64-musl-lto", + "consumer": "openclaw-release.yml optional linux-arm64-musl" + }, + { + "label": "bun-webkit-windows-amd64-lto", + "consumer": "openclaw-release.yml optional windows-x64" + } ], - "excluded": "Windows arm64, Android, FreeBSD, debug and ASAN are not selected by the fork's active OpenClaw workflows. There is no separate x64 baseline WebKit archive." + "excluded": "Windows arm64, Android, FreeBSD, debug and ASAN are not selected by the fork's active OpenClaw workflows. There is no separate x64 baseline WebKit archive.", + "upstream_sync": { + "base_commit": "d2d2a26ef973cdd97b37953f5dba58052acad74f", + "upstream_commit": "1878660bb47a6a87ebe518cea0581ef4bf9b71f8", + "prepared_tree": "d9ec052fccc81afbf43a5344e52bc980f439d587" + } } diff --git a/.github/workflows/openclaw-artifacts.yml b/.github/workflows/openclaw-artifacts.yml index e0c05f03f62d5..41249f59d89f7 100644 --- a/.github/workflows/openclaw-artifacts.yml +++ b/.github/workflows/openclaw-artifacts.yml @@ -17,7 +17,7 @@ concurrency: cancel-in-progress: false jobs: preflight: - if: github.repository == 'openclaw/WebKit' && (github.ref == 'refs/heads/openclaw/main' || github.ref == 'refs/heads/openclaw/batch-1') + if: github.repository == 'openclaw/WebKit' && (github.ref == 'refs/heads/openclaw/main' || github.ref == 'refs/heads/openclaw/batch-1' || github.ref == 'refs/heads/openclaw/release-5718a6ec') runs-on: blacksmith-32vcpu-ubuntu-2404 timeout-minutes: 10 outputs: @@ -93,7 +93,14 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: persist-credentials: false - sparse-checkout: .github + sparse-checkout-cone-mode: false + sparse-checkout: | + /.github/ + /Tools/ + /JSTests/stress/ + /JSTests/modules/ + /JSTests/*.yaml + /JSTests/BUN-TEST-DIFFERENCES.md - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 with: node-version: '26.3.0' @@ -109,7 +116,7 @@ jobs: curl -fsSL --retry 5 https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null echo 'deb https://apt.llvm.org/noble/ llvm-toolchain-noble-23 main' | sudo tee /etc/apt/sources.list.d/llvm.list sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends cmake ninja-build nasm ccache golang-go clang-23 lld-23 llvm-23 unzip + sudo apt-get install -y --no-install-recommends cmake ninja-build nasm ccache golang-go clang-23 lld-23 llvm-23 unzip ruby ruby-highline libjson-pp-perl rustup toolchain install nightly-2026-09-15 --profile minimal --component rust-src echo 'RUSTUP_TOOLCHAIN=nightly-2026-09-15' >> "$GITHUB_ENV" echo '/usr/lib/llvm-23/bin' >> "$GITHUB_PATH" @@ -120,7 +127,10 @@ jobs: env: ARTIFACT_DIR: ${{ runner.temp }}/linux-artifact QUALIFICATION_DIR: ${{ runner.temp }}/qualification - run: bash .github/openclaw/qualify.sh + run: | + bash .github/openclaw/qualify-engine.sh + bash .github/openclaw/qualify.sh + bash .github/openclaw/qualify-sync.sh - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a if: always() with: @@ -134,6 +144,9 @@ jobs: ${{ runner.temp }}/qualification/baseline/*.log ${{ runner.temp }}/qualification/candidate/*.json ${{ runner.temp }}/qualification/candidate/*.log + ${{ runner.temp }}/qualification/sync/*.json + ${{ runner.temp }}/qualification/sync/*.log + ${{ runner.temp }}/qualification/jsc-*/results if-no-files-found: error retention-days: 7 assemble: @@ -164,7 +177,7 @@ jobs: if-no-files-found: error retention-days: 7 publish: - if: inputs.publish == true && (github.ref == 'refs/heads/openclaw/main' || github.ref == 'refs/heads/openclaw/batch-1') + if: inputs.publish == true && (github.ref == 'refs/heads/openclaw/main' || github.ref == 'refs/heads/openclaw/batch-1' || github.ref == 'refs/heads/openclaw/release-5718a6ec') needs: [build, test, assemble] environment: openclaw-artifacts permissions: diff --git a/.github/workflows/openclaw-checks.yml b/.github/workflows/openclaw-checks.yml index d41891ab71558..beda0ee11d813 100644 --- a/.github/workflows/openclaw-checks.yml +++ b/.github/workflows/openclaw-checks.yml @@ -1,7 +1,7 @@ name: OpenClaw engine checks on: pull_request: - branches: [openclaw/main] + branches: [openclaw/main, openclaw/release-5718a6ec] workflow_dispatch: permissions: contents: read @@ -28,6 +28,9 @@ jobs: !/ManualTests/ !/WebDriverTests/ /JSTests/stress/ + /JSTests/modules/ + /JSTests/*.yaml + /JSTests/BUN-TEST-DIFFERENCES.md - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 with: node-version: '26.3.0' @@ -51,31 +54,14 @@ jobs: curl -fSL --retry 5 https://github.com/oven-sh/bun/releases/download/bun-v1.4.2/bun-linux-x64.zip -o "$RUNNER_TEMP/bun.zip" unzip -q "$RUNNER_TEMP/bun.zip" -d "$RUNNER_TEMP/bootstrap" echo "$RUNNER_TEMP/bootstrap/bun-linux-x64" >> "$GITHUB_PATH" - - name: Exercise module namespace engine regressions - run: | - mkdir "$RUNNER_TEMP/jsc-artifact" - tar -xzf "$RUNNER_TEMP/linux-artifact/bun-webkit-linux-amd64.tar.gz" -C "$RUNNER_TEMP/jsc-artifact" - # $vm is a JavaScript identifier. - # shellcheck disable=SC2016 - "$RUNNER_TEMP/jsc-artifact/bun-webkit/bin/jsc" --useDollarVM=1 -e 'if (typeof $vm.moduleNamespaceForRequire !== "function") throw new Error("namespace test hook is missing");' - Tools/Scripts/run-jsc-stress-tests JSTests/stress --jsc "$RUNNER_TEMP/jsc-artifact/bun-webkit/bin/jsc" --child-processes 16 --filter 'module-namespace|module-loader-security-thenable-namespace' --output-dir "$RUNNER_TEMP/jsc-results" --no-retry - python3 .github/openclaw/verify-jsc-results.py "$RUNNER_TEMP/jsc-results" - - name: Exercise allocation sampling across execution tiers - shell: bash - run: | - set -euo pipefail - jsc="$RUNNER_TEMP/jsc-artifact/bun-webkit/bin/jsc" - fixture=JSTests/stress/heap-allocation-sampling.js - common=(--useDollarVM=true --useGenerationalGC=true --forceMiniVMMode=false) - "$jsc" "${common[@]}" "$fixture" - "$jsc" "${common[@]}" --useJIT=false "$fixture" - "$jsc" "${common[@]}" --useDFGJIT=false --useFTLJIT=false "$fixture" - "$jsc" "${common[@]}" --useConcurrentJIT=false --thresholdForOptimizeAfterWarmUp=10 --thresholdForFTLOptimizeAfterWarmUp=20 "$fixture" - name: Qualify the paired Bun adapter and fork selection env: ARTIFACT_DIR: ${{ runner.temp }}/linux-artifact QUALIFICATION_DIR: ${{ runner.temp }}/qualification - run: bash .github/openclaw/qualify.sh + run: | + bash .github/openclaw/qualify-engine.sh + bash .github/openclaw/qualify.sh + bash .github/openclaw/qualify-sync.sh - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a if: always() with: @@ -89,7 +75,8 @@ jobs: ${{ runner.temp }}/qualification/baseline/*.log ${{ runner.temp }}/qualification/candidate/*.json ${{ runner.temp }}/qualification/candidate/*.log - ${{ runner.temp }}/jsc-results/results - ${{ runner.temp }}/jsc-results/*.log + ${{ runner.temp }}/qualification/sync/*.json + ${{ runner.temp }}/qualification/sync/*.log + ${{ runner.temp }}/qualification/jsc-*/results if-no-files-found: error retention-days: 7 diff --git a/OPENCLAW.md b/OPENCLAW.md index 126c4c51e848e..8fa72b4c68dd0 100644 --- a/OPENCLAW.md +++ b/OPENCLAW.md @@ -1,7 +1,7 @@ # OpenClaw WebKit artifacts This fork owns the WebKit artifacts consumed by `openclaw/bun`. The engine base is -`1600131e46b5af48bbda3559af8d8a3327230b6e`; each engine change is a separate +`5718a6ec579b98362ea7276a426deedcc6281ef5`; each engine change is a separate commit. Upstream `.github/scripts/lanes.mjs`, Dockerfiles, ICU inputs and macOS cross tools remain unchanged. No workflow publishes toolchain images. @@ -33,7 +33,7 @@ base, then review the whole batch. Changing the upstream recipe requires a separate qualification and an explicit base/recipe update. Dispatch **OpenClaw WebKit artifacts** from `openclaw/main` (or the protected -`openclaw/batch-1` publication branch) with its full current +`openclaw/batch-1` or `openclaw/release-5718a6ec` publication branch) with its full current commit SHA and `publish: false`. It builds the selected lanes on Linux x64 with Docker Buildx, at most two lanes at once. Each lane records the exact source, commit patches, recipe hashes, locally built toolchain image ID, BuildKit @@ -51,7 +51,7 @@ this gate does not claim native runtime qualification on every architecture. ## Publish and verify Batch 1 is published from `openclaw/batch-1` so concurrently reviewed batch-2 -engine changes can land on `openclaw/main` without entering this release. Both +engine changes can land on `openclaw/main` without entering this release. All publication branches require the same protected environment and full-SHA/green-job checks. After a green build/test dispatch, dispatch the same full SHA with `publish: @@ -130,3 +130,40 @@ entrypoint. - Add real byte-based JSC heap allocation sampling with weighted allocating-stack profiles and independent major/minor collection inclusion. - Check allocation sampling across interpreter and JIT tiers before merging engine pull requests. + +## Rebased release qualification + +- Rebase the four batch-1 engine patches and the landed namespace and allocation-sampling patches onto oven-sh/WebKit `5718a6ec579b98362ea7276a426deedcc6281ef5` (upstream #768). Thanks @Jarred-Sumner for idle compiler-thread and Atomics.wait memory release. +- Retain the published batch-1 release. Publish this separately qualified batch only through the protected `openclaw/release-5718a6ec` environment policy; updating the Bun main manifest is a separate integration task. + +The existing qualification baseline remains upstream `1600131e46b5af48bbda3559af8d8a3327230b6e` with its committed archive checksums. The source/provenance base advances to `5718a6ec579b98362ea7276a426deedcc6281ef5`; the Docker and lane recipes are unchanged. Unlanded engine work is not included in this rebase. + + +The rebased release is frozen on `openclaw/release-5718a6ec`; only patches merged +into engine main and explicitly included before the freeze enter this batch. +Later main changes require a new source commit and qualification. The published +release notes come from `.github/openclaw/release-notes.md`. + +The original W113 comparison retains its pinned Bun source and 44 selected files +plus two dependency checks. Its candidate additionally takes the upstream +mimalloc bump needed by `mi_on_thread_idle`. This is a matched engine/allocator +comparison, not an engine-only performance measurement. + +A separate qualification build reproduces W150's prepared upstream-sync tree +`d9ec052fccc81afbf43a5344e52bc980f439d587` exactly from Bun +`d2d2a26ef973cdd97b37953f5dba58052acad74f` before applying recorded qualification +adapters and replacing the local checksum manifest. The production manifest +resolver stays intact, using its SHA-256 prefetch cache. No Bun branch is pushed. +This build runs the union of the original 56-file sync selection and the current +engine/nightly selection, requires every result to pass, and explicitly executes +the unchanged Atomics purge, notification-race and idle Wasm RSS tests without +skips. Runtime source/engine identities, adapter bytes and results are retained +in provenance. Release assembly requires this gate in addition to W113. + +The shared test job also checks the namespace and Segmenter stress cases, +promise/microtask stress and complete module manifest, allocation sampling in +four execution tiers, seven worker heap counters, fourteen worker cases, the +ALS hook/continuous-GC matrices, realm/retention probes, and Bun inspector tests. +The upstream delta changes only waiting/compiler-thread allocator release and +its bridge; module-loader, namespace, managed-heap and inspector patches remain +patch-identical on the new base. diff --git a/Source/JavaScriptCore/runtime/WaiterListManager.cpp b/Source/JavaScriptCore/runtime/WaiterListManager.cpp index 3e7009b3671fc..dbeb59150bd60 100644 --- a/Source/JavaScriptCore/runtime/WaiterListManager.cpp +++ b/Source/JavaScriptCore/runtime/WaiterListManager.cpp @@ -132,8 +132,29 @@ WaiterListManager::WaitSyncResult WaiterListManager::waitSyncImpl(VM& vm, ValueT dataLogLnIf(WaiterListsManagerInternal::verbose, " added a new SyncWaiter=", syncWaiter.get(), " to a waiterList for ptr ", RawPointer(ptr)); syncWaiter->setParkedList(list.copyRef()); - while (syncWaiter->isOnList() && time.now() < time && !shouldStopWaitingForTermination(vm)) +#if USE(MIMALLOC) + // Only the owning thread can return the memory its thread-local heap holds, so do that once in a wait that + // takes a while. A wait that is notified soon pays nothing. + MonotonicTime releaseFreeMemoryTime = MonotonicTime::now() + 100_ms; + bool didReleaseFreeMemory = false; +#endif + while (syncWaiter->isOnList() && time.now() < time && !shouldStopWaitingForTermination(vm)) { +#if USE(MIMALLOC) + if (!didReleaseFreeMemory && releaseFreeMemoryTime < time) { + if (MonotonicTime::now() < releaseFreeMemoryTime) { + syncWaiter->condition().waitUntil(list->lock, releaseFreeMemoryTime.approximate()); + continue; + } + didReleaseFreeMemory = true; + // A notification in the meantime takes us off the list, and the loop tests for that (and for a + // termination request) under the lock again before it waits. + DropLockForScope dropLock { listLocker }; + WTF::releaseFastMallocFreeMemoryForIdleThread(); + continue; + } +#endif syncWaiter->condition().waitUntil(list->lock, time.approximate()); + } syncWaiter->setParkedList(nullptr); diff --git a/Source/JavaScriptCore/shell/CMakeLists.txt b/Source/JavaScriptCore/shell/CMakeLists.txt index b1a05d5972bf8..fe6073d65a08e 100644 --- a/Source/JavaScriptCore/shell/CMakeLists.txt +++ b/Source/JavaScriptCore/shell/CMakeLists.txt @@ -21,7 +21,10 @@ endif () if (USE_EXTERNAL_MIMALLOC) # The shipped static archives intentionally leave mi_* unresolved so the # consumer links its own mimalloc. The jsc shell still needs a working - # allocator, so link the vendored mimalloc object library here only. + # allocator, so link the vendored mimalloc object library here only, plus + # the entry points the archives expect from the consumer's mimalloc that + # the vendored one does not have (ExternalMimallocShims.cpp). + list(APPEND jsc_SOURCES ExternalMimallocShims.cpp) list(APPEND jsc_LIBRARIES $) endif () @@ -94,6 +97,7 @@ if (USE_BUN_JSC_ADDITIONS) # Same as jsc above: the shipped static archives leave mi_* unresolved # under USE_EXTERNAL_MIMALLOC, so any executable that links the # framework must supply the vendored mimalloc objects itself. + list(APPEND testFFI_SOURCES ExternalMimallocShims.cpp) list(APPEND testFFI_LIBRARIES $) endif () diff --git a/Source/JavaScriptCore/shell/ExternalMimallocShims.cpp b/Source/JavaScriptCore/shell/ExternalMimallocShims.cpp new file mode 100644 index 0000000000000..a4b74b4123b2f --- /dev/null +++ b/Source/JavaScriptCore/shell/ExternalMimallocShims.cpp @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2026 Oven, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +// Under USE_EXTERNAL_MIMALLOC the static archives call into the consumer's mimalloc +// (oven-sh/mimalloc), which has entry points the vendored copy does not. An executable in this +// repository that links the archives against the vendored mimalloc-obj (the jsc shell, testFFI) +// supplies those entry points here, in terms of the vendored API. + +#include + +// The consumer's hook for a thread that goes idle: collect the heap, discard the free runs inside +// still-used pages, hand the arena purge to a scavenger thread. The vendored mimalloc can only do +// the first part. +extern "C" void mi_on_thread_idle(void) noexcept +{ + mi_theap_collect(mi_theap_get_default(), /* force */ false); +} diff --git a/Source/WTF/wtf/AutomaticThread.cpp b/Source/WTF/wtf/AutomaticThread.cpp index 048180f219bf3..27cc52d63b9a7 100644 --- a/Source/WTF/wtf/AutomaticThread.cpp +++ b/Source/WTF/wtf/AutomaticThread.cpp @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -37,6 +38,11 @@ WTF_MAKE_TZONE_ALLOCATED_IMPL(AutomaticThread); static constexpr bool verbose = false; +#if USE(MIMALLOC) +// How long a thread waits for more work before it releases its allocator's free memory. +static constexpr Seconds idleReleaseDelay = 100_ms; +#endif + Ref AutomaticThreadCondition::create() { return adoptRef(*new AutomaticThreadCondition); @@ -48,6 +54,16 @@ AutomaticThreadCondition::~AutomaticThreadCondition() = default; void AutomaticThreadCondition::notifyOne(const AbstractLocker& locker) { +#if USE(MIMALLOC) + // A thread that is releasing its free memory only sees the notification when it is done with that. + for (auto& thread : m_threads) { + if (thread->isWaiting(locker) && !thread->m_isReleasingFreeMemory) { + thread->notify(locker); + return; + } + } +#endif + for (auto& thread : m_threads) { if (thread->isWaiting(locker)) { thread->notify(locker); @@ -215,6 +231,11 @@ void AutomaticThread::start(const AbstractLocker&) stopImpl(locker); }; +#if USE(MIMALLOC) + // Whether this thread released its allocator's free memory since it last worked. + bool didReleaseFreeMemory = false; +#endif + for (;;) { { Locker locker { *m_lock }; @@ -228,8 +249,39 @@ void AutomaticThread::start(const AbstractLocker&) // Shut the thread down after a timeout. m_isWaiting = true; +#if USE(MIMALLOC) + // Only the owning thread can return the memory its thread-local heap holds. + // A compiler thread frees tens of MB of temporaries after a large compile, + // and without this they stay resident until the thread times out and exits. + // Wait a little first, so that a thread which is notified again right away + // (the usual case between tasks) does not pay for the release. + // + // poll() runs once per notify: JITWorklistThread counts a thread as active + // from the notify until its poll() returns Wait. So the thread stays in the + // waiting state through the release, and polls again only if a notify + // arrived meanwhile (notify() clears m_isWaiting, which is checked under the + // lock, so a notify during the release is not lost). + Seconds timeout = m_timeout; + if (!didReleaseFreeMemory && m_timeout > idleReleaseDelay) { + m_waitCondition.waitFor(*m_lock, idleReleaseDelay); + if (!m_isWaiting) + continue; + didReleaseFreeMemory = true; + m_isReleasingFreeMemory = true; + { + DropLockForScope dropLock { locker }; + releaseFastMallocFreeMemoryForIdleThread(); + } + m_isReleasingFreeMemory = false; + if (!m_isWaiting) + continue; + timeout -= idleReleaseDelay; + } +#else + Seconds timeout = m_timeout; +#endif bool awokenByNotify = - m_waitCondition.waitFor(*m_lock, m_timeout); + m_waitCondition.waitFor(*m_lock, timeout); if (verbose && !awokenByNotify && !m_isWaiting) dataLog(RawPointer(this), ": waitFor timed out, but notified via m_isWaiting flag!\n"); if (m_isWaiting && shouldSleep(locker)) { @@ -244,6 +296,9 @@ void AutomaticThread::start(const AbstractLocker&) } } +#if USE(MIMALLOC) + didReleaseFreeMemory = false; +#endif WorkResult result = work(); if (result == WorkResult::Stop) { Locker locker { *m_lock }; diff --git a/Source/WTF/wtf/AutomaticThread.h b/Source/WTF/wtf/AutomaticThread.h index f665a24883729..cda144334cff3 100644 --- a/Source/WTF/wtf/AutomaticThread.h +++ b/Source/WTF/wtf/AutomaticThread.h @@ -203,6 +203,7 @@ class WTF_EXPORT_PRIVATE AutomaticThread : public ThreadSafeRefCounted