diff --git a/bench/runtime/_pin_/apple_m4.txt b/bench/runtime/_pin_/apple_m4.txt index 569fea73f..5a99edded 100644 --- a/bench/runtime/_pin_/apple_m4.txt +++ b/bench/runtime/_pin_/apple_m4.txt @@ -1,20 +1,20 @@ -# 2026-09-11 b20509fd bun gates/perf.ts (the COMPILER column: medians of three runs, by hand) +# 2026-10-08 the SEQ-CPU, PAR-CPU and PAR-GPU times are the medians of 5 gate runs (each cell timed by tm, the median of three runs under 2 s) at the commit each was pinned at: 35dbdb75; 386b7fc1 for bitonic, kmeans and nbody SEQ and PAR; d04cf05e for histogram. COMPILER and the spaces as 2026-09-11 b20509fd bun gates/perf.ts (the COMPILER column: medians of three runs, by hand) | bench | COMPILER | SEQ-CPU | PAR-CPU | PAR-GPU | OUTPUT | |--------------|----------|------------------|------------------|------------------|------------| -| bfs | 0.578s | 3.508s 1.9M | 0.507s 2.6M | 0.956s 13.9M | 651176970 | -| editdist | 0.562s | 1.926s 2.0M | 0.397s 3.0M | 0.481s 13.8M | 2229810577 | -| gameoflife | 0.553s | 6.927s 2.0M | 1.154s 2.6M | 0.100s 13.9M | 2016151040 | -| hashmap | 0.661s | 3.293s 2.3M | 0.517s 9.0M | 1.548s 14.0M | 1307803744 | -| histogram | 1.033s | 1.717s 67.7M | 0.367s 74.8M | 1.481s 80.0M | 122845198 | -| kmeans | 1.094s | 3.969s 23.5M | 0.835s 21.0M | 0.700s 15.8M | 1616398086 | -| lexer | 0.646s | 2.323s 1.9M | 0.429s 2.9M | 3.891s 14.1M | 2401049475 | -| mandelbrot | 0.637s | 4.451s 2.2M | 0.685s 2.8M | 0.068s 13.6M | 3101455856 | -| merkle | 0.597s | 4.171s 130M | 0.777s 131M | 0.081s 13.9M | 3104235417 | -| nbody | 0.671s | 5.899s 2.1M | 0.850s 2.6M | 0.065s 14.2M | 3516450380 | -| queens | 0.602s | 4.768s 2.0M | 0.762s 2.5M | 2.683s 13.9M | 2063750025 | -| raytrace | 0.652s | 4.353s 2.0M | 0.734s 2.6M | 0.395s 14.0M | 1924309504 | -| symreg | 0.686s | 2.750s 2.0M | 0.473s 3.0M | 0.814s 14.3M | 2383953211 | -| terrain | 0.622s | 2.150s 2.0M | 0.364s 3.3M | 0.291s 13.9M | 2572468224 | -| tree-bitonic | 0.666s | 5.847s 146M | 1.246s 159M | 1.987s 14.3M | 3787129428 | -| tree-matmul | 1.473s | 1.855s 2.8M | 0.377s 15.3M | 0.999s 16.8M | 3797651056 | -| tree-radix | 0.668s | 3.785s 651M | 0.759s 615M | 1.028s 14.5M | 1998173798 | +| bfs | 0.578s | 3.501s 1.9M | 0.500s 2.6M | 0.894s 13.9M | 651176970 | +| editdist | 0.562s | 1.916s 2.0M | 0.393s 3.0M | 0.476s 13.8M | 2229810577 | +| gameoflife | 0.553s | 6.919s 2.0M | 1.151s 2.6M | 0.085s 13.9M | 2016151040 | +| hashmap | 0.661s | 3.281s 2.3M | 0.509s 9.0M | 1.543s 14.0M | 1307803744 | +| histogram | 1.033s | 1.466s 67.7M | 0.312s 74.8M | 1.477s 80.0M | 122845198 | +| kmeans | 1.094s | 3.772s 23.5M | 0.826s 21.0M | 0.684s 15.8M | 1616398086 | +| lexer | 0.646s | 2.292s 1.9M | 0.427s 2.9M | 3.923s 14.1M | 2401049475 | +| mandelbrot | 0.637s | 4.444s 2.2M | 0.677s 2.8M | 0.067s 13.6M | 3101455856 | +| merkle | 0.597s | 4.166s 130M | 0.771s 131M | 0.080s 13.9M | 3104235417 | +| nbody | 0.671s | 5.894s 2.1M | 0.842s 2.6M | 0.061s 14.2M | 3516450380 | +| queens | 0.602s | 4.745s 2.0M | 0.770s 2.5M | 2.701s 13.9M | 2063750025 | +| raytrace | 0.652s | 4.346s 2.0M | 0.730s 2.6M | 0.391s 14.0M | 1924309504 | +| symreg | 0.686s | 2.744s 2.0M | 0.466s 3.0M | 0.807s 14.3M | 2383953211 | +| terrain | 0.622s | 2.139s 2.0M | 0.360s 3.3M | 0.313s 13.9M | 2572468224 | +| tree-bitonic | 0.666s | 5.490s 146M | 1.185s 159M | 1.966s 14.3M | 3787129428 | +| tree-matmul | 1.473s | 1.861s 2.8M | 0.371s 15.3M | 1.013s 16.8M | 3797651056 | +| tree-radix | 0.668s | 3.780s 651M | 0.738s 615M | 1.025s 14.5M | 1998173798 | diff --git a/gates/perf.ts b/gates/perf.ts index 5fae62e68..8f3da6047 100644 --- a/gates/perf.ts +++ b/gates/perf.ts @@ -10,14 +10,20 @@ // -O3 of bend -o: no -lm, no -fmodules, no -fno-slp-vectorize, so the // gate grades the binary bend builds) and runs it with the flags the // pins were measured with (PAR on the power of two under the core -// count, GPU over the bench's span), one warm run and one timed -// by a microsecond clock around /usr/bin/time -l, whose own 10 ms tick -// cannot grade a 50 ms GPU cell (its RSS is the space). A cell passes at -// 1.15x or under; --pin runs every -// cell three times and writes the medians as the new pins (a space in -// tenths of a megabyte, so a 2.5 MB program is not graded against "2M"), -// refusing to write while any cell is unmeasured; --gate prints only the -// verdict. +// count, GPU over the bench's span), one warm run and one timed, or three +// and their median when the warm run took under SHORT (a 60 ms GPU cell +// read 59.7-73.6 ms from one run after its builds). tm times each run and +// gives its RSS (the space): one posix_spawn and wait4, where the perl +// clocks around /usr/bin/time -l cost 6 ms of that cell. A cell passes at +// 1.15x or under; --pin runs the +// gate PIN_RUNS times, PIN_REST seconds apart, each run dispatched and +// measured as a graded one, prints every run's times, and writes each +// cell's median as its pin, a space in tenths of a +// megabyte (so a 2.5 MB program is not graded against "2M"), refusing to +// write while any cell is unmeasured. Three copies of each cell in one +// run share that run's moment: mandelbrot's PAR-GPU pinned at 0.060 s, +// then read 0.060-0.070 s at the commit it was pinned at. --gate prints +// only the verdict. import * as child from "node:child_process"; import * as fs from "node:fs"; @@ -79,6 +85,41 @@ const MARK = "@@B4"; const CLOCK = "perl -MTime::HiRes=time -e 'print time'"; +const SHORT = 2; + +const PIN_RUNS = 5; + +const PIN_REST = 2; + +// tm FILE CMD ARGS... runs CMD and writes its exit status, wall seconds and +// maximum RSS in bytes to FILE. +const TM = `#include +#include +#include +#include +#include +extern char** environ; +int main(int argc, char** argv) { + struct timespec a, b; + struct rusage ru; + pid_t pid; + int st = 0; + clock_gettime(CLOCK_MONOTONIC, &a); + if (argc < 3 || posix_spawn(&pid, argv[2], NULL, NULL, argv + 2, environ) + || wait4(pid, &st, 0, &ru) < 0) { + return 127; + } + clock_gettime(CLOCK_MONOTONIC, &b); + FILE* f = fopen(argv[1], "w"); + if (f == NULL) { + return 127; + } + fprintf(f, "%d %.6f %ld\\n", WIFEXITED(st) ? WEXITSTATUS(st) + : 128 + WTERMSIG(st), (double)(b.tv_sec - a.tv_sec) + + (b.tv_nsec - a.tv_nsec) / 1e9, (long)ru.ru_maxrss); + return fclose(f); +}`; + const VIEW: string[] = []; let drawn = 0; @@ -231,15 +272,19 @@ function cell_script(c: Cell): string { + ` t1=$(${CLOCK}); [ $b = 0 ] && { ${lib.BUN} bend2/main.ts ${src}` + ` -o main.c >> build.txt 2>&1 && ${BUILD[c.mode]} -o cell >> build.txt` + ` 2>&1; b=$?; }; echo "${MARK} built $b $t0 $t1"; cat build.txt;` - + ` if [ $b = 0 ]; then ${run} > /dev/null 2>&1; t2=$(${CLOCK});` - + ` /usr/bin/time -l ${run} > out.txt 2> time.txt; r=$?; t3=$(${CLOCK});` - + ` echo "${MARK} ran $r $t2 $t3"; cat out.txt; echo "${MARK} time";` - + ` cat time.txt; fi; cd; rm -rf $d`; + + ` if [ $b = 0 ]; then cat > tm.c <<'TM'\n${TM}\nTM\n ${CC} tm.c -o tm` + + ` && ./tm warm.txt ${run} > /dev/null 2>&1; n=1; awk '$2 < ${SHORT}` + + ` { exit 1 }' warm.txt || n=3; i=0; while [ $i -lt $n ]; do` + + ` i=$((i+1)); rm -f ran.txt; ./tm ran.txt ${run} > out.txt 2> err.txt;` + + ` echo "${MARK} ran $(cat ran.txt 2> /dev/null || echo 127 0 0)";` + + ` grep -q '^0 ' ran.txt 2> /dev/null || break;` + + ` done; cat out.txt;` + + ` echo "${MARK} err"; cat err.txt; fi; cd; rm -rf $d`; } function cell_note(out: string): string { - const line = out.trim().split("\n").filter((l) => !l.startsWith(MARK) - && !/ real | maximum resident|^\s*\d+\s+\w/.test(l)).pop() ?? ""; + const line = out.trim().split("\n").filter((l) => !l.startsWith(MARK)) + .pop() ?? ""; return line.slice(0, 100); } @@ -248,28 +293,25 @@ async function cell_run(c: Cell, node: number, pack: Buffer): Promise { const built = new RegExp("^" + MARK + " built (\\d+) ([\\d.]+) ([\\d.]+)$", "m") .exec(got.out); - const ran = new RegExp("^" + MARK + " ran (\\d+) ([\\d.]+) ([\\d.]+)$", "m") - .exec(got.out); + const runs = [...got.out.matchAll(new RegExp("^" + MARK + + " ran (\\d+) ([\\d.]+) (\\d+)$", "gm"))] + .sort((x, y) => Number(x[2]) - Number(y[2])); + const ran = runs.find((r) => r[1] !== "0") ?? runs[runs.length >> 1]; if (built === null) { throw new Error("node", { cause: got.err }); } c.comp = Number(built[3]) - Number(built[2]); - if (built[1] !== "0" || ran === null || ran[1] !== "0") { + if (built[1] !== "0" || ran === undefined || ran[1] !== "0") { c.note = c.bench + " " + MODES[c.mode] + ": " + (built[1] !== "0" ? "build: " : "exit " + String(ran?.[1] ?? "?") + ": ") + cell_note(got.out); return; } - const tail = got.out.split(new RegExp("^" + MARK + " ran .*\n", "m"))[1]; - const [body, time] = (tail ?? "").split(MARK + " time\n"); - const rss = /(\d+)\s+maximum resident set size/.exec(time ?? ""); - c.out = body.split("\n").map((l) => l.trim()).filter((l) => l !== "") - .pop() ?? ""; - c.secs = Number(ran[3]) - Number(ran[2]); - c.mem = rss === null ? null : Number(rss[1]) / (1 << 20); - if (c.mem === null) { - c.note = c.bench + " " + MODES[c.mode] + ": unreadable time output"; - } + const tail = got.out.split(new RegExp("^" + MARK + " ran .*\n", "m")).pop(); + c.out = (tail ?? "").split(MARK + " err\n")[0].split("\n") + .map((l) => l.trim()).filter((l) => l !== "").pop() ?? ""; + c.secs = Number(ran[2]); + c.mem = Number(ran[3]) / (1 << 20); } // Chk @@ -302,41 +344,61 @@ async function chk_run(c: Chk, node: number): Promise { if (import.meta.main) { const benches = fs.readdirSync(RUNTIME).filter((f) => !f.startsWith("_")) .sort(); - const reps = PIN ? 3 : 1; - let cells: Cell[] = benches.flatMap((bench) => MODES.flatMap((_, mode) => - Array.from({ length: reps }, () => ({ bench, mode, secs: null, - mem: null, comp: null, out: "", note: "" })))); - const chks: Chk[] = fs.readdirSync(CHECKER).filter((f) => !f.startsWith("_")) - .sort().map((bench) => ({ bench, secs: null, note: "" })); + const fresh = (): Cell[] => benches.flatMap((bench) => MODES.map((_, mode) => + ({ bench, mode, secs: null, mem: null, comp: null, out: "", note: "" }))); + const chk_fresh = (): Chk[] => fs.readdirSync(CHECKER) + .filter((f) => !f.startsWith("_")).sort() + .map((bench) => ({ bench, secs: null, note: "" })); + let cells = fresh(); + let chks = chk_fresh(); + const all: Cell[] = []; + const chk_all: Chk[] = []; const [pins, cpins] = PIN ? [new Map(), new Map()] : pin_read(); const notes = (): string[] => [...cells, ...chks].map((c) => c.note) .filter((n) => n !== ""); const draw = (): void => view_draw(cells, chks, pins, cpins, notes()); - draw(); const nodes = await lib.node_lock(); const pack = lib.pack(RUNTIME); - await lib.node_pool(nodes, [...cells.map((c) => async (node: number) => { - await cell_run(c, node, pack); - draw(); - }), ...chks.map((c) => async (node: number) => { - await chk_run(c, node); + for (let run = 0; run < (PIN ? PIN_RUNS : 1); run += 1) { + if (run > 0) { + await Bun.sleep(PIN_REST * 1000); + cells = fresh(); + chks = chk_fresh(); + } draw(); - })]); + await lib.node_pool(nodes, [...cells.map((c) => async (node: number) => { + await cell_run(c, node, pack); + draw(); + }), ...chks.map((c) => async (node: number) => { + await chk_run(c, node); + draw(); + })]); + all.push(...cells); + chk_all.push(...chks); + } if (PIN) { + console.log(benches.flatMap((bench) => MODES.map((m, mode) => bench + " " + + m + ": " + all.filter((c) => c.bench === bench && c.mode === mode) + .map((c) => c.secs?.toFixed(3) ?? "-").join(" "))).join("\n")); const mid = (xs: (number | null)[]): number | null => { const ys = xs.filter((x) => x !== null).sort((x, y) => x - y); return ys.length === xs.length ? ys[Math.floor(ys.length / 2)] : null; }; cells = benches.flatMap((bench) => MODES.map((_, mode) => { - const cs = cells.filter((c) => c.bench === bench && c.mode === mode); + const cs = all.filter((c) => c.bench === bench && c.mode === mode); return { bench, mode, secs: mid(cs.map((c) => c.secs)), mem: mid(cs.map((c) => c.mem)), comp: mid(cs.map((c) => c.comp)), out: cs[0].out, note: cs.map((c) => c.note).find((n) => n !== "") ?? (cs.every((c) => c.out === cs[0].out) ? "" : bench + " " + MODES[mode] + ": the runs printed different outputs") }; })); + chks = chks.map(({ bench }) => { + const cs = chk_all.filter((c) => c.bench === bench); + return { bench, secs: mid(cs.map((c) => c.secs)), + note: cs.map((c) => c.note).find((n) => n !== "") ?? "" }; + }); if (notes().length > 0) { console.log(notes().join("\n")); lib.verdict(0, 1);