bench: drive HTTP/3 with h2load built with ngtcp2/nghttp3, not h3x

bench/README.md

@@ -44,7 +44,7 @@ A number from a broken fixture is worse than no number.
 | `THREADS` | 8 | load generator threads |
 | `MIN_UTIL` | 90 | reject a cell whose reactors were not saturated |
 | `BASELINE` | `results/latest.json` | compare against this instead of the previous run |
-| `H3X` | `~/h3x/build/h3x` | the HTTP/3 driver |
+| `H2LOAD_H3` | `~/h3x/bench/h2load` | an `h2load` built with ngtcp2/nghttp3, the HTTP/3 driver |
 
 ## Results are kept
 
@@ -68,7 +68,7 @@ A number with nothing to compare against cannot catch a regression, which is the
 | --- | --- |
 | `h1`, `h1s` | `wrk` |
 | `h2c`, `h2` | `h2load` (nghttp2) |
-| `h3` | [`h3x`](https://github.com/MDA2AV) - the `h2load` in most distributions is built without ngtcp2/nghttp3 and cannot drive HTTP/3 (it reports 0 started) |
+| `h3` | `h2load` built with ngtcp2/nghttp3 (`H2LOAD_H3`) - the one in most distributions is not, and reports 0 started. Not `h3x`: its rate divides by the whole run, which waits ~30 s on any request that never finishes, so a few stuck streams made a 300k req/s server read as 50k |
 | `echo` | `Playground/Clients/Quic` - the QUIC echo servers speak no HTTP, so nothing off the shelf can drive them. The driver is itself a sample, not a bench-only artifact |
 
 `Bench.Clients` drives ioxide's own HTTP clients (`h1`/`h2`/`h3`) against an external server, for

bench/any.sh

@@ -39,7 +39,6 @@ for a in "$@"; do
 done
 set -- ${ARGS+"${ARGS[@]}"}
 SERVER_CPUS=${SERVER_CPUS:-$(. "$(dirname "$0")/lib.sh"; bench_server_cpus)}
-H3X=${H3X:-/home/diogo/h3x/build/h3x}
 
 # One definition of how each protocol is driven, shared with every other script under bench/.
 . "$(dirname "$0")/lib.sh"
@@ -133,6 +132,11 @@ bad_run() {
   [ -n "${n:-}" ] && [ "${n:-0}" != 0 ] && { echo "non-2xx=$n"; return 0; }
   grep -q 'Socket errors' "$out" && { echo "socket errors"; return 0; }
   grep -qE '^\s*[1-9][0-9]* failed' "$out" && { echo "failed requests"; return 0; }
+  grep -qP '^requests: .* [1-9]\d* (failed|errored|timeout)' "$out" && { echo "failed requests"; return 0; }   # h2load
+  grep -qP '^status codes: .* [1-9]\d* [345]xx' "$out" && { echo "non-2xx"; return 0; }                         # h2load
+  # h2load counts a request killed with its connection as neither failed nor errored, only not done.
+  grep -qP '^requests: .* [1-9]\d* started, 0 done' "$out" && { echo "no request completed"; return 0; }        # h2load
+  grep -qP '^req/s\s+:\s+0\.00\s' "$out" && { echo "a connection completed nothing"; return 0; }                # h2load
   return 1
 }
 
@@ -146,7 +150,7 @@ if [ "${1:-}" = "--list" ]; then
     [ "$proto" = none ] && r="no (no load generator here can drive it)"
     [ "$proto" = echo ] && [ ! -x Playground/Clients/Quic/bin/Release/net11.0/Playground.Clients.Quic ] \
         && r="no (Clients/Quic not built)"
-    [ "$proto" = h3 ] && [ ! -x "$H3X" ] && r="no (h3x missing)"
+    [ "$proto" = h3 ] && [ ! -x "$H2LOAD_H3" ] && r="no (no h2load with HTTP/3 - set H2LOAD_H3)"
     [ -z "$(binary "$sample")" ] && r="no (not built)"
     printf '%-26s %-7s %-6s %-11s %-20s %s\n' "$sample" "$proto" "$port" "$path" "$origin" "$r"
   done < <(registry)
@@ -184,7 +188,7 @@ while read -r sample proto port path origin extra; do
   [ "$proto" = none ] && skip="no load generator here can drive it"
   [ "$proto" = echo ] && [ ! -x Playground/Clients/Quic/bin/Release/net11.0/Playground.Clients.Quic ] \
       && skip="Clients/Quic (the driver) not built"
-  [ "$proto" = h3 ] && [ ! -x "$H3X" ] && skip="h3x missing"
+  [ "$proto" = h3 ] && [ ! -x "$H2LOAD_H3" ] && skip="no h2load with HTTP/3 (H2LOAD_H3)"
   [ -z "$(binary "$sample")" ] && skip="not built"
   if [ -n "$skip" ]; then
     printf '   %-24s %12s %10s %10s  %s\n' "$sample" - - - "skipped: $skip"
@@ -241,8 +245,9 @@ while read -r sample proto port path origin extra; do
   cleanup
 
   note=""; rps=${rps:-0}
-  if [ "$rps" = 0 ]; then note="driver produced no number"
-  elif bad=$(bad_run "$WORK/$STAMP.txt"); then note="discarded: $bad"; rps=0
+  if [ "$(awk -v r="$rps" 'BEGIN{print (r == 0)}')" = 1 ]; then note="driver produced no number"; rps=0
+  fi
+  if bad=$(bad_run "$WORK/$STAMP.txt"); then note="discarded: $bad"; rps=0
   fi
 
   # As a percentage of REACTORS cores, not one - two pegged reactors burn 200% of a core.

bench/lib.sh

@@ -94,6 +94,9 @@ bench_assert_single_listener() {
 # Scale multiplies the concurrency and exists for the headroom probe below; a measured run always
 # uses 1.
 
+# HTTP/3 needs an h2load built with ngtcp2/nghttp3 - most distributions' is not, and reports 0 started.
+H2LOAD_H3=${H2LOAD_H3:-$HOME/h3x/bench/h2load}
+
 bench_load() {
     local proto=$1 port=$2 secs=$3 out=$4 path=$5 scale=${6:-1}
     local conns=$((CONNS * scale)) threads=$THREADS streams=$((32 * scale))
@@ -121,10 +124,12 @@ bench_load() {
                 PLAYGROUND_ECHO_SECONDS="$secs" "$drv" >"$out" 2>&1
             grep -oP '^\K[\d.]+(?= req/s)' "$out" | head -1 ;;
         h3)
-            [ -x "$H3X" ] || { echo "" ; return; }
-            "${pin[@]}" "$H3X" -d "$secs" --connections "$conns" -m "$streams" -k \
-                "https://127.0.0.1:$port$path" >"$out" 2>&1
-            grep -oP 'throughput:\s+\K[\d.]+' "$out" | head -1 ;;
+            # Not h3x: its rate divides by the whole run, which waits ~30 s on any request that never
+            # finishes - a dozen stuck streams made a 300k req/s server read as 50k.
+            [ -x "$H2LOAD_H3" ] || { echo "" ; return; }
+            "${pin[@]}" "$H2LOAD_H3" --alpn-list=h3 -D "$secs" -c "$conns" -m "$streams" \
+                -t "$(( threads < conns ? threads : conns ))" "https://127.0.0.1:$port$path" >"$out" 2>&1
+            grep -oP 'finished in [\d.]+s, \K[\d.]+(?= req/s)' "$out" | head -1 ;;
     esac
 }
 

bench/run.sh

@@ -7,7 +7,7 @@
 #
 # Servers run 4 reactors (BENCH_REACTORS to override); the tcp-raw/tcp-pipe pair repeats at 12
 # reactors as the io_uring baseline. Loads: wrk for everything TCP (fixed -c 64 -t 4,
-# time-bounded), h3x for the HTTP/3 server (H3X to point at the binary), Bench.Clients for the
+# time-bounded), h2load built with HTTP/3 for the HTTP/3 server (H2LOAD_H3 to point at it), Bench.Clients for the
 # ring-native clients. Sidecars (redis, postgres, an h2c nginx) start via docker when available;
 # anything missing skips with a note instead of failing the run.
 #
@@ -17,7 +17,6 @@ cd "$(dirname "$0")/.."
 
 R=${BENCH_REACTORS:-4}
 DUR=${BENCH_SECONDS:-8}
-H3X=${H3X:-/home/diogo/h3x/build/h3x}
 
 # The drivers live in one place. This file used to carry its own HTTP/3 invocation that differed
 # from any.sh's, and the same server measured 260k under one and 505k under the other.
@@ -93,12 +92,12 @@ fi
 
 # ── h3 server ───────────────────────────────────────────────────────────────────────────────
 play Http3/Nghttp3Request PLAYGROUND_REACTORS=$R PLAYGROUND_QUIC_PORT=18444 PLAYGROUND_PORT=18090
-if [ -x "$H3X" ]; then
+if [ -x "$H2LOAD_H3" ]; then
   N=$(CONNS=${CONNS:-64} THREADS=${THREADS:-8} REACTORS=$R \
       bench_load h3 18444 "$DUR" "$BIN/h3-server.txt" / | cut -d. -f1)
   note "h3-server" "${R}r" "${N:-fail} req/s"
 else
-  note "h3-server" "${R}r" "SKIP (h3x not found - set H3X=/path/to/h3x)"
+  note "h3-server" "${R}r" "SKIP (no h2load with HTTP/3 - set H2LOAD_H3=/path/to/h2load)"
 fi
 # the h3 server stays up: the h3 client bench below dials it
 

bench/samples.tsv

@@ -11,7 +11,7 @@
 #         h1s   HTTP/1.1 over TLS                  (wrk)
 #         h2c   HTTP/2 prior-knowledge, cleartext  (h2load)
 #         h2    HTTP/2 over TLS, ALPN              (h2load)
-#         h3    HTTP/3 over QUIC                   (h3x)
+#         h3    HTTP/3 over QUIC                   (h2load built with HTTP/3)
 #         echo  raw stream echo, no HTTP           (Playground/Clients/Quic drives it)
 #         driver a sample that GENERATES load rather than serving it
 #         none  nothing here can drive it - the mutual-TLS samples need a client certificate,