From 4e516c28395aa2578120786e6f3140c33f23edd4 Mon Sep 17 00:00:00 2001 From: Nicolas Abril Date: Thu, 8 Oct 2026 17:58:09 +0200 Subject: [PATCH 1/3] A perf cell is timed by tm, one posix_spawn and wait4, and a cell under 1 s grades the median of three runs; the apple_m4 pins are regraved at 386b7fc1 (histogram at d04cf05e) under this method Co-Authored-By: Claude Opus 5.5 (1M context) --- bench/checker/_pin_/apple_m4.txt | 12 +++--- bench/runtime/_pin_/apple_m4.txt | 40 ++++++++--------- gates/perf.ts | 73 ++++++++++++++++++++++---------- 3 files changed, 77 insertions(+), 48 deletions(-) diff --git a/bench/checker/_pin_/apple_m4.txt b/bench/checker/_pin_/apple_m4.txt index 2fc8e2886..863a9ef37 100644 --- a/bench/checker/_pin_/apple_m4.txt +++ b/bench/checker/_pin_/apple_m4.txt @@ -1,8 +1,8 @@ -# 2026-09-09 26de367b bun gates/perf.ts --pin +# 2026-10-08 386b7fc1 bun gates/perf.ts --pin, each cell timed by tm, the median of three runs under 1 s | bench | CHECKER | |---------------|----------| -| compute_1600 | 0.541s | -| defs_12800 | 0.302s | -| generics_3200 | 0.353s | -| proofs_3200 | 0.811s | -| trees_400 | 0.579s | +| compute_1600 | 0.558s | +| defs_12800 | 0.318s | +| generics_3200 | 0.368s | +| proofs_3200 | 0.819s | +| trees_400 | 0.600s | diff --git a/bench/runtime/_pin_/apple_m4.txt b/bench/runtime/_pin_/apple_m4.txt index 569fea73f..76e4ae543 100644 --- a/bench/runtime/_pin_/apple_m4.txt +++ b/bench/runtime/_pin_/apple_m4.txt @@ -1,20 +1,20 @@ -# 2026-09-11 b20509fd bun gates/perf.ts (the COMPILER column: medians of three runs, by hand) -| bench | COMPILER | SEQ-CPU | PAR-CPU | PAR-GPU | OUTPUT | -|--------------|----------|------------------|------------------|------------------|------------| -| bfs | 0.578s | 3.508s 1.9M | 0.507s 2.6M | 0.956s 13.9M | 651176970 | -| editdist | 0.562s | 1.926s 2.0M | 0.397s 3.0M | 0.481s 13.8M | 2229810577 | -| gameoflife | 0.553s | 6.927s 2.0M | 1.154s 2.6M | 0.100s 13.9M | 2016151040 | -| hashmap | 0.661s | 3.293s 2.3M | 0.517s 9.0M | 1.548s 14.0M | 1307803744 | -| histogram | 1.033s | 1.717s 67.7M | 0.367s 74.8M | 1.481s 80.0M | 122845198 | -| kmeans | 1.094s | 3.969s 23.5M | 0.835s 21.0M | 0.700s 15.8M | 1616398086 | -| lexer | 0.646s | 2.323s 1.9M | 0.429s 2.9M | 3.891s 14.1M | 2401049475 | -| mandelbrot | 0.637s | 4.451s 2.2M | 0.685s 2.8M | 0.068s 13.6M | 3101455856 | -| merkle | 0.597s | 4.171s 130M | 0.777s 131M | 0.081s 13.9M | 3104235417 | -| nbody | 0.671s | 5.899s 2.1M | 0.850s 2.6M | 0.065s 14.2M | 3516450380 | -| queens | 0.602s | 4.768s 2.0M | 0.762s 2.5M | 2.683s 13.9M | 2063750025 | -| raytrace | 0.652s | 4.353s 2.0M | 0.734s 2.6M | 0.395s 14.0M | 1924309504 | -| symreg | 0.686s | 2.750s 2.0M | 0.473s 3.0M | 0.814s 14.3M | 2383953211 | -| terrain | 0.622s | 2.150s 2.0M | 0.364s 3.3M | 0.291s 13.9M | 2572468224 | -| tree-bitonic | 0.666s | 5.847s 146M | 1.246s 159M | 1.987s 14.3M | 3787129428 | -| tree-matmul | 1.473s | 1.855s 2.8M | 0.377s 15.3M | 0.999s 16.8M | 3797651056 | -| tree-radix | 0.668s | 3.785s 651M | 0.759s 615M | 1.028s 14.5M | 1998173798 | +# 2026-10-08 386b7fc1 (histogram: d04cf05e) bun gates/perf.ts --pin, each cell timed by tm, the median of three runs under 1 s +| bench | COMPILER | SEQ-CPU | PAR-CPU | PAR-GPU | OUTPUT | +|---------------|----------|------------------|------------------|------------------|------------| +| bfs | 0.556s | 3.484s 1.9M | 0.500s 2.6M | 0.932s 13.5M | 651176970 | +| editdist | 0.543s | 1.915s 2.0M | 0.389s 3.0M | 0.476s 13.5M | 2229810577 | +| gameoflife | 0.537s | 7.166s 2.0M | 1.165s 2.6M | 0.079s 13.4M | 2016151040 | +| hashmap | 0.673s | 3.418s 2.3M | 0.524s 9.0M | 1.546s 13.6M | 1307803744 | +| histogram | 1.040s | 1.456s 67.7M | 0.314s 74.8M | 1.479s 80.1M | 122845198 | +| kmeans | 1.096s | 3.722s 23.5M | 0.828s 22.0M | 0.679s 14.3M | 1616398086 | +| lexer | 0.650s | 2.400s 1.9M | 0.440s 2.9M | 3.906s 13.7M | 2401049475 | +| mandelbrot | 0.641s | 4.437s 2.2M | 0.676s 2.8M | 0.060s 13.6M | 3101455856 | +| merkle | 0.596s | 4.167s 130M | 0.771s 131M | 0.075s 13.8M | 3104235417 | +| nbody | 0.681s | 5.894s 2.1M | 0.843s 2.6M | 0.058s 13.5M | 3516450380 | +| queens | 0.599s | 5.156s 2.0M | 0.816s 2.5M | 2.481s 13.6M | 2063750025 | +| raytrace | 0.658s | 4.134s 2.0M | 0.705s 2.5M | 0.342s 13.6M | 1924309504 | +| symreg | 0.696s | 2.531s 2.0M | 0.442s 3.0M | 0.798s 13.7M | 2383953211 | +| terrain | 0.626s | 1.991s 2.0M | 0.343s 3.3M | 0.319s 13.7M | 2572468224 | +| tree-bitonic | 0.668s | 5.389s 146M | 1.162s 159M | 1.887s 13.8M | 3787129428 | +| tree-matmul | 1.486s | 2.044s 2.7M | 0.406s 15.3M | 1.017s 15.8M | 3797651056 | +| tree-radix | 0.672s | 3.688s 651M | 0.748s 607M | 1.023s 13.7M | 1998173798 | diff --git a/gates/perf.ts b/gates/perf.ts index 5fae62e68..23a2160aa 100644 --- a/gates/perf.ts +++ b/gates/perf.ts @@ -10,9 +10,11 @@ // -O3 of bend -o: no -lm, no -fmodules, no -fno-slp-vectorize, so the // gate grades the binary bend builds) and runs it with the flags the // pins were measured with (PAR on the power of two under the core -// count, GPU over the bench's span), one warm run and one timed -// by a microsecond clock around /usr/bin/time -l, whose own 10 ms tick -// cannot grade a 50 ms GPU cell (its RSS is the space). A cell passes at +// count, GPU over the bench's span), one warm run and one timed, or three +// and their median when the warm run took under SHORT (a 60 ms GPU cell +// read 59.7-73.6 ms from one run after its builds). tm times each run and +// gives its RSS (the space): one posix_spawn and wait4, where the perl +// clocks around /usr/bin/time -l cost 6 ms of that cell. A cell passes at // 1.15x or under; --pin runs every // cell three times and writes the medians as the new pins (a space in // tenths of a megabyte, so a 2.5 MB program is not graded against "2M"), @@ -79,6 +81,34 @@ const MARK = "@@B4"; const CLOCK = "perl -MTime::HiRes=time -e 'print time'"; +const SHORT = 1; + +// tm FILE CMD ARGS... runs CMD and writes its exit status, wall seconds and +// maximum RSS in bytes to FILE. +const TM = `#include +#include +#include +#include +#include +extern char** environ; +int main(int argc, char** argv) { + struct timespec a, b; + struct rusage ru; + pid_t pid; + int st = 0; + clock_gettime(CLOCK_MONOTONIC, &a); + if (argc < 3 || posix_spawn(&pid, argv[2], NULL, NULL, argv + 2, environ) + || wait4(pid, &st, 0, &ru) < 0) { + return 127; + } + clock_gettime(CLOCK_MONOTONIC, &b); + FILE* f = fopen(argv[1], "w"); + fprintf(f, "%d %.6f %ld\\n", WIFEXITED(st) ? WEXITSTATUS(st) + : 128 + WTERMSIG(st), (double)(b.tv_sec - a.tv_sec) + + (b.tv_nsec - a.tv_nsec) / 1e9, (long)ru.ru_maxrss); + return fclose(f); +}`; + const VIEW: string[] = []; let drawn = 0; @@ -231,15 +261,17 @@ function cell_script(c: Cell): string { + ` t1=$(${CLOCK}); [ $b = 0 ] && { ${lib.BUN} bend2/main.ts ${src}` + ` -o main.c >> build.txt 2>&1 && ${BUILD[c.mode]} -o cell >> build.txt` + ` 2>&1; b=$?; }; echo "${MARK} built $b $t0 $t1"; cat build.txt;` - + ` if [ $b = 0 ]; then ${run} > /dev/null 2>&1; t2=$(${CLOCK});` - + ` /usr/bin/time -l ${run} > out.txt 2> time.txt; r=$?; t3=$(${CLOCK});` - + ` echo "${MARK} ran $r $t2 $t3"; cat out.txt; echo "${MARK} time";` - + ` cat time.txt; fi; cd; rm -rf $d`; + + ` if [ $b = 0 ]; then cat > tm.c <<'TM'\n${TM}\nTM\n ${CC} tm.c -o tm` + + ` && ./tm warm.txt ${run} > /dev/null 2>&1; n=1; awk '$2 < ${SHORT}` + + ` { exit 1 }' warm.txt || n=3; i=0; while [ $i -lt $n ]; do` + + ` i=$((i+1)); ./tm ran.txt ${run} > out.txt 2> err.txt;` + + ` echo "${MARK} ran $(cat ran.txt)"; done; cat out.txt;` + + ` echo "${MARK} err"; cat err.txt; fi; cd; rm -rf $d`; } function cell_note(out: string): string { - const line = out.trim().split("\n").filter((l) => !l.startsWith(MARK) - && !/ real | maximum resident|^\s*\d+\s+\w/.test(l)).pop() ?? ""; + const line = out.trim().split("\n").filter((l) => !l.startsWith(MARK)) + .pop() ?? ""; return line.slice(0, 100); } @@ -248,28 +280,25 @@ async function cell_run(c: Cell, node: number, pack: Buffer): Promise { const built = new RegExp("^" + MARK + " built (\\d+) ([\\d.]+) ([\\d.]+)$", "m") .exec(got.out); - const ran = new RegExp("^" + MARK + " ran (\\d+) ([\\d.]+) ([\\d.]+)$", "m") - .exec(got.out); + const runs = [...got.out.matchAll(new RegExp("^" + MARK + + " ran (\\d+) ([\\d.]+) (\\d+)$", "gm"))] + .sort((x, y) => Number(x[2]) - Number(y[2])); + const ran = runs.find((r) => r[1] !== "0") ?? runs[runs.length >> 1]; if (built === null) { throw new Error("node", { cause: got.err }); } c.comp = Number(built[3]) - Number(built[2]); - if (built[1] !== "0" || ran === null || ran[1] !== "0") { + if (built[1] !== "0" || ran === undefined || ran[1] !== "0") { c.note = c.bench + " " + MODES[c.mode] + ": " + (built[1] !== "0" ? "build: " : "exit " + String(ran?.[1] ?? "?") + ": ") + cell_note(got.out); return; } - const tail = got.out.split(new RegExp("^" + MARK + " ran .*\n", "m"))[1]; - const [body, time] = (tail ?? "").split(MARK + " time\n"); - const rss = /(\d+)\s+maximum resident set size/.exec(time ?? ""); - c.out = body.split("\n").map((l) => l.trim()).filter((l) => l !== "") - .pop() ?? ""; - c.secs = Number(ran[3]) - Number(ran[2]); - c.mem = rss === null ? null : Number(rss[1]) / (1 << 20); - if (c.mem === null) { - c.note = c.bench + " " + MODES[c.mode] + ": unreadable time output"; - } + const tail = got.out.split(new RegExp("^" + MARK + " ran .*\n", "m")).pop(); + c.out = (tail ?? "").split(MARK + " err\n")[0].split("\n") + .map((l) => l.trim()).filter((l) => l !== "").pop() ?? ""; + c.secs = Number(ran[2]); + c.mem = Number(ran[3]) / (1 << 20); } // Chk From 2baec10a0854b335c9889bf66c9a904eb20ddf9b Mon Sep 17 00:00:00 2001 From: Nicolas Abril Date: Thu, 8 Oct 2026 20:44:40 +0200 Subject: [PATCH 2/3] tm checks fopen; a cell clears ran.txt before each run, prints a missing result as exit 127 and stops at the first failing run; SHORT is 2 s; the checker pin stamp says the checker keeps the perl clock Co-Authored-By: Claude Opus 5.5 (1M context) --- bench/checker/_pin_/apple_m4.txt | 2 +- gates/perf.ts | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/bench/checker/_pin_/apple_m4.txt b/bench/checker/_pin_/apple_m4.txt index 863a9ef37..150ef8e72 100644 --- a/bench/checker/_pin_/apple_m4.txt +++ b/bench/checker/_pin_/apple_m4.txt @@ -1,4 +1,4 @@ -# 2026-10-08 386b7fc1 bun gates/perf.ts --pin, each cell timed by tm, the median of three runs under 1 s +# 2026-10-08 386b7fc1 bun gates/perf.ts --pin (the checker keeps the perl clock and the best of three runs) | bench | CHECKER | |---------------|----------| | compute_1600 | 0.558s | diff --git a/gates/perf.ts b/gates/perf.ts index 23a2160aa..dcf01cd50 100644 --- a/gates/perf.ts +++ b/gates/perf.ts @@ -81,7 +81,7 @@ const MARK = "@@B4"; const CLOCK = "perl -MTime::HiRes=time -e 'print time'"; -const SHORT = 1; +const SHORT = 2; // tm FILE CMD ARGS... runs CMD and writes its exit status, wall seconds and // maximum RSS in bytes to FILE. @@ -103,6 +103,9 @@ int main(int argc, char** argv) { } clock_gettime(CLOCK_MONOTONIC, &b); FILE* f = fopen(argv[1], "w"); + if (f == NULL) { + return 127; + } fprintf(f, "%d %.6f %ld\\n", WIFEXITED(st) ? WEXITSTATUS(st) : 128 + WTERMSIG(st), (double)(b.tv_sec - a.tv_sec) + (b.tv_nsec - a.tv_nsec) / 1e9, (long)ru.ru_maxrss); @@ -264,8 +267,10 @@ function cell_script(c: Cell): string { + ` if [ $b = 0 ]; then cat > tm.c <<'TM'\n${TM}\nTM\n ${CC} tm.c -o tm` + ` && ./tm warm.txt ${run} > /dev/null 2>&1; n=1; awk '$2 < ${SHORT}` + ` { exit 1 }' warm.txt || n=3; i=0; while [ $i -lt $n ]; do` - + ` i=$((i+1)); ./tm ran.txt ${run} > out.txt 2> err.txt;` - + ` echo "${MARK} ran $(cat ran.txt)"; done; cat out.txt;` + + ` i=$((i+1)); rm -f ran.txt; ./tm ran.txt ${run} > out.txt 2> err.txt;` + + ` echo "${MARK} ran $(cat ran.txt 2> /dev/null || echo 127 0 0)";` + + ` grep -q '^0 ' ran.txt 2> /dev/null || break;` + + ` done; cat out.txt;` + ` echo "${MARK} err"; cat err.txt; fi; cd; rm -rf $d`; } From aea70118995ad108c604dad9b329d11cd558e35d Mon Sep 17 00:00:00 2001 From: Nicolas Abril Date: Thu, 8 Oct 2026 21:44:16 +0200 Subject: [PATCH 3/3] --pin runs the gate five times, 2 s apart, as a graded run is dispatched and measured, and pins each cell's median; only the run times are regraved, each at the commit it was pinned at, and COMPILER, the spaces and the checker keep their pins Co-Authored-By: Claude Opus 5.5 (1M context) --- bench/checker/_pin_/apple_m4.txt | 12 +++--- bench/runtime/_pin_/apple_m4.txt | 40 +++++++++---------- gates/perf.ts | 66 +++++++++++++++++++++++--------- 3 files changed, 73 insertions(+), 45 deletions(-) diff --git a/bench/checker/_pin_/apple_m4.txt b/bench/checker/_pin_/apple_m4.txt index 150ef8e72..2fc8e2886 100644 --- a/bench/checker/_pin_/apple_m4.txt +++ b/bench/checker/_pin_/apple_m4.txt @@ -1,8 +1,8 @@ -# 2026-10-08 386b7fc1 bun gates/perf.ts --pin (the checker keeps the perl clock and the best of three runs) +# 2026-09-09 26de367b bun gates/perf.ts --pin | bench | CHECKER | |---------------|----------| -| compute_1600 | 0.558s | -| defs_12800 | 0.318s | -| generics_3200 | 0.368s | -| proofs_3200 | 0.819s | -| trees_400 | 0.600s | +| compute_1600 | 0.541s | +| defs_12800 | 0.302s | +| generics_3200 | 0.353s | +| proofs_3200 | 0.811s | +| trees_400 | 0.579s | diff --git a/bench/runtime/_pin_/apple_m4.txt b/bench/runtime/_pin_/apple_m4.txt index 76e4ae543..5a99edded 100644 --- a/bench/runtime/_pin_/apple_m4.txt +++ b/bench/runtime/_pin_/apple_m4.txt @@ -1,20 +1,20 @@ -# 2026-10-08 386b7fc1 (histogram: d04cf05e) bun gates/perf.ts --pin, each cell timed by tm, the median of three runs under 1 s -| bench | COMPILER | SEQ-CPU | PAR-CPU | PAR-GPU | OUTPUT | -|---------------|----------|------------------|------------------|------------------|------------| -| bfs | 0.556s | 3.484s 1.9M | 0.500s 2.6M | 0.932s 13.5M | 651176970 | -| editdist | 0.543s | 1.915s 2.0M | 0.389s 3.0M | 0.476s 13.5M | 2229810577 | -| gameoflife | 0.537s | 7.166s 2.0M | 1.165s 2.6M | 0.079s 13.4M | 2016151040 | -| hashmap | 0.673s | 3.418s 2.3M | 0.524s 9.0M | 1.546s 13.6M | 1307803744 | -| histogram | 1.040s | 1.456s 67.7M | 0.314s 74.8M | 1.479s 80.1M | 122845198 | -| kmeans | 1.096s | 3.722s 23.5M | 0.828s 22.0M | 0.679s 14.3M | 1616398086 | -| lexer | 0.650s | 2.400s 1.9M | 0.440s 2.9M | 3.906s 13.7M | 2401049475 | -| mandelbrot | 0.641s | 4.437s 2.2M | 0.676s 2.8M | 0.060s 13.6M | 3101455856 | -| merkle | 0.596s | 4.167s 130M | 0.771s 131M | 0.075s 13.8M | 3104235417 | -| nbody | 0.681s | 5.894s 2.1M | 0.843s 2.6M | 0.058s 13.5M | 3516450380 | -| queens | 0.599s | 5.156s 2.0M | 0.816s 2.5M | 2.481s 13.6M | 2063750025 | -| raytrace | 0.658s | 4.134s 2.0M | 0.705s 2.5M | 0.342s 13.6M | 1924309504 | -| symreg | 0.696s | 2.531s 2.0M | 0.442s 3.0M | 0.798s 13.7M | 2383953211 | -| terrain | 0.626s | 1.991s 2.0M | 0.343s 3.3M | 0.319s 13.7M | 2572468224 | -| tree-bitonic | 0.668s | 5.389s 146M | 1.162s 159M | 1.887s 13.8M | 3787129428 | -| tree-matmul | 1.486s | 2.044s 2.7M | 0.406s 15.3M | 1.017s 15.8M | 3797651056 | -| tree-radix | 0.672s | 3.688s 651M | 0.748s 607M | 1.023s 13.7M | 1998173798 | +# 2026-10-08 the SEQ-CPU, PAR-CPU and PAR-GPU times are the medians of 5 gate runs (each cell timed by tm, the median of three runs under 2 s) at the commit each was pinned at: 35dbdb75; 386b7fc1 for bitonic, kmeans and nbody SEQ and PAR; d04cf05e for histogram. COMPILER and the spaces as 2026-09-11 b20509fd bun gates/perf.ts (the COMPILER column: medians of three runs, by hand) +| bench | COMPILER | SEQ-CPU | PAR-CPU | PAR-GPU | OUTPUT | +|--------------|----------|------------------|------------------|------------------|------------| +| bfs | 0.578s | 3.501s 1.9M | 0.500s 2.6M | 0.894s 13.9M | 651176970 | +| editdist | 0.562s | 1.916s 2.0M | 0.393s 3.0M | 0.476s 13.8M | 2229810577 | +| gameoflife | 0.553s | 6.919s 2.0M | 1.151s 2.6M | 0.085s 13.9M | 2016151040 | +| hashmap | 0.661s | 3.281s 2.3M | 0.509s 9.0M | 1.543s 14.0M | 1307803744 | +| histogram | 1.033s | 1.466s 67.7M | 0.312s 74.8M | 1.477s 80.0M | 122845198 | +| kmeans | 1.094s | 3.772s 23.5M | 0.826s 21.0M | 0.684s 15.8M | 1616398086 | +| lexer | 0.646s | 2.292s 1.9M | 0.427s 2.9M | 3.923s 14.1M | 2401049475 | +| mandelbrot | 0.637s | 4.444s 2.2M | 0.677s 2.8M | 0.067s 13.6M | 3101455856 | +| merkle | 0.597s | 4.166s 130M | 0.771s 131M | 0.080s 13.9M | 3104235417 | +| nbody | 0.671s | 5.894s 2.1M | 0.842s 2.6M | 0.061s 14.2M | 3516450380 | +| queens | 0.602s | 4.745s 2.0M | 0.770s 2.5M | 2.701s 13.9M | 2063750025 | +| raytrace | 0.652s | 4.346s 2.0M | 0.730s 2.6M | 0.391s 14.0M | 1924309504 | +| symreg | 0.686s | 2.744s 2.0M | 0.466s 3.0M | 0.807s 14.3M | 2383953211 | +| terrain | 0.622s | 2.139s 2.0M | 0.360s 3.3M | 0.313s 13.9M | 2572468224 | +| tree-bitonic | 0.666s | 5.490s 146M | 1.185s 159M | 1.966s 14.3M | 3787129428 | +| tree-matmul | 1.473s | 1.861s 2.8M | 0.371s 15.3M | 1.013s 16.8M | 3797651056 | +| tree-radix | 0.668s | 3.780s 651M | 0.738s 615M | 1.025s 14.5M | 1998173798 | diff --git a/gates/perf.ts b/gates/perf.ts index dcf01cd50..8f3da6047 100644 --- a/gates/perf.ts +++ b/gates/perf.ts @@ -15,11 +15,15 @@ // read 59.7-73.6 ms from one run after its builds). tm times each run and // gives its RSS (the space): one posix_spawn and wait4, where the perl // clocks around /usr/bin/time -l cost 6 ms of that cell. A cell passes at -// 1.15x or under; --pin runs every -// cell three times and writes the medians as the new pins (a space in -// tenths of a megabyte, so a 2.5 MB program is not graded against "2M"), -// refusing to write while any cell is unmeasured; --gate prints only the -// verdict. +// 1.15x or under; --pin runs the +// gate PIN_RUNS times, PIN_REST seconds apart, each run dispatched and +// measured as a graded one, prints every run's times, and writes each +// cell's median as its pin, a space in tenths of a +// megabyte (so a 2.5 MB program is not graded against "2M"), refusing to +// write while any cell is unmeasured. Three copies of each cell in one +// run share that run's moment: mandelbrot's PAR-GPU pinned at 0.060 s, +// then read 0.060-0.070 s at the commit it was pinned at. --gate prints +// only the verdict. import * as child from "node:child_process"; import * as fs from "node:fs"; @@ -83,6 +87,10 @@ const CLOCK = "perl -MTime::HiRes=time -e 'print time'"; const SHORT = 2; +const PIN_RUNS = 5; + +const PIN_REST = 2; + // tm FILE CMD ARGS... runs CMD and writes its exit status, wall seconds and // maximum RSS in bytes to FILE. const TM = `#include @@ -336,41 +344,61 @@ async function chk_run(c: Chk, node: number): Promise { if (import.meta.main) { const benches = fs.readdirSync(RUNTIME).filter((f) => !f.startsWith("_")) .sort(); - const reps = PIN ? 3 : 1; - let cells: Cell[] = benches.flatMap((bench) => MODES.flatMap((_, mode) => - Array.from({ length: reps }, () => ({ bench, mode, secs: null, - mem: null, comp: null, out: "", note: "" })))); - const chks: Chk[] = fs.readdirSync(CHECKER).filter((f) => !f.startsWith("_")) - .sort().map((bench) => ({ bench, secs: null, note: "" })); + const fresh = (): Cell[] => benches.flatMap((bench) => MODES.map((_, mode) => + ({ bench, mode, secs: null, mem: null, comp: null, out: "", note: "" }))); + const chk_fresh = (): Chk[] => fs.readdirSync(CHECKER) + .filter((f) => !f.startsWith("_")).sort() + .map((bench) => ({ bench, secs: null, note: "" })); + let cells = fresh(); + let chks = chk_fresh(); + const all: Cell[] = []; + const chk_all: Chk[] = []; const [pins, cpins] = PIN ? [new Map(), new Map()] : pin_read(); const notes = (): string[] => [...cells, ...chks].map((c) => c.note) .filter((n) => n !== ""); const draw = (): void => view_draw(cells, chks, pins, cpins, notes()); - draw(); const nodes = await lib.node_lock(); const pack = lib.pack(RUNTIME); - await lib.node_pool(nodes, [...cells.map((c) => async (node: number) => { - await cell_run(c, node, pack); - draw(); - }), ...chks.map((c) => async (node: number) => { - await chk_run(c, node); + for (let run = 0; run < (PIN ? PIN_RUNS : 1); run += 1) { + if (run > 0) { + await Bun.sleep(PIN_REST * 1000); + cells = fresh(); + chks = chk_fresh(); + } draw(); - })]); + await lib.node_pool(nodes, [...cells.map((c) => async (node: number) => { + await cell_run(c, node, pack); + draw(); + }), ...chks.map((c) => async (node: number) => { + await chk_run(c, node); + draw(); + })]); + all.push(...cells); + chk_all.push(...chks); + } if (PIN) { + console.log(benches.flatMap((bench) => MODES.map((m, mode) => bench + " " + + m + ": " + all.filter((c) => c.bench === bench && c.mode === mode) + .map((c) => c.secs?.toFixed(3) ?? "-").join(" "))).join("\n")); const mid = (xs: (number | null)[]): number | null => { const ys = xs.filter((x) => x !== null).sort((x, y) => x - y); return ys.length === xs.length ? ys[Math.floor(ys.length / 2)] : null; }; cells = benches.flatMap((bench) => MODES.map((_, mode) => { - const cs = cells.filter((c) => c.bench === bench && c.mode === mode); + const cs = all.filter((c) => c.bench === bench && c.mode === mode); return { bench, mode, secs: mid(cs.map((c) => c.secs)), mem: mid(cs.map((c) => c.mem)), comp: mid(cs.map((c) => c.comp)), out: cs[0].out, note: cs.map((c) => c.note).find((n) => n !== "") ?? (cs.every((c) => c.out === cs[0].out) ? "" : bench + " " + MODES[mode] + ": the runs printed different outputs") }; })); + chks = chks.map(({ bench }) => { + const cs = chk_all.filter((c) => c.bench === bench); + return { bench, secs: mid(cs.map((c) => c.secs)), + note: cs.map((c) => c.note).find((n) => n !== "") ?? "" }; + }); if (notes().length > 0) { console.log(notes().join("\n")); lib.verdict(0, 1);