diff --git a/bench/README.md b/bench/README.md index d83102a..acf56af 100644 --- a/bench/README.md +++ b/bench/README.md @@ -44,7 +44,7 @@ A number from a broken fixture is worse than no number. | `THREADS` | 8 | load generator threads | | `MIN_UTIL` | 90 | reject a cell whose reactors were not saturated | | `BASELINE` | `results/latest.json` | compare against this instead of the previous run | -| `H3X` | `~/h3x/build/h3x` | the HTTP/3 driver | +| `H2LOAD_H3` | `~/h3x/bench/h2load` | an `h2load` built with ngtcp2/nghttp3, the HTTP/3 driver | ## Results are kept @@ -68,7 +68,7 @@ A number with nothing to compare against cannot catch a regression, which is the | --- | --- | | `h1`, `h1s` | `wrk` | | `h2c`, `h2` | `h2load` (nghttp2) | -| `h3` | [`h3x`](https://github.com/MDA2AV) - the `h2load` in most distributions is built without ngtcp2/nghttp3 and cannot drive HTTP/3 (it reports 0 started) | +| `h3` | `h2load` built with ngtcp2/nghttp3 (`H2LOAD_H3`) - the one in most distributions is not, and reports 0 started. Not `h3x`: its rate divides by the whole run, which waits ~30 s on any request that never finishes, so a few stuck streams made a 300k req/s server read as 50k | | `echo` | `Playground/Clients/Quic` - the QUIC echo servers speak no HTTP, so nothing off the shelf can drive them. The driver is itself a sample, not a bench-only artifact | `Bench.Clients` drives ioxide's own HTTP clients (`h1`/`h2`/`h3`) against an external server, for diff --git a/bench/any.sh b/bench/any.sh index 9c5d877..9c1dadc 100755 --- a/bench/any.sh +++ b/bench/any.sh @@ -39,7 +39,6 @@ for a in "$@"; do done set -- ${ARGS+"${ARGS[@]}"} SERVER_CPUS=${SERVER_CPUS:-$(. "$(dirname "$0")/lib.sh"; bench_server_cpus)} -H3X=${H3X:-/home/diogo/h3x/build/h3x} # One definition of how each protocol is driven, shared with every other script under bench/. . "$(dirname "$0")/lib.sh" @@ -133,6 +132,11 @@ bad_run() { [ -n "${n:-}" ] && [ "${n:-0}" != 0 ] && { echo "non-2xx=$n"; return 0; } grep -q 'Socket errors' "$out" && { echo "socket errors"; return 0; } grep -qE '^\s*[1-9][0-9]* failed' "$out" && { echo "failed requests"; return 0; } + grep -qP '^requests: .* [1-9]\d* (failed|errored|timeout)' "$out" && { echo "failed requests"; return 0; } # h2load + grep -qP '^status codes: .* [1-9]\d* [345]xx' "$out" && { echo "non-2xx"; return 0; } # h2load + # h2load counts a request killed with its connection as neither failed nor errored, only not done. + grep -qP '^requests: .* [1-9]\d* started, 0 done' "$out" && { echo "no request completed"; return 0; } # h2load + grep -qP '^req/s\s+:\s+0\.00\s' "$out" && { echo "a connection completed nothing"; return 0; } # h2load return 1 } @@ -146,7 +150,7 @@ if [ "${1:-}" = "--list" ]; then [ "$proto" = none ] && r="no (no load generator here can drive it)" [ "$proto" = echo ] && [ ! -x Playground/Clients/Quic/bin/Release/net11.0/Playground.Clients.Quic ] \ && r="no (Clients/Quic not built)" - [ "$proto" = h3 ] && [ ! -x "$H3X" ] && r="no (h3x missing)" + [ "$proto" = h3 ] && [ ! -x "$H2LOAD_H3" ] && r="no (no h2load with HTTP/3 - set H2LOAD_H3)" [ -z "$(binary "$sample")" ] && r="no (not built)" printf '%-26s %-7s %-6s %-11s %-20s %s\n' "$sample" "$proto" "$port" "$path" "$origin" "$r" done < <(registry) @@ -184,7 +188,7 @@ while read -r sample proto port path origin extra; do [ "$proto" = none ] && skip="no load generator here can drive it" [ "$proto" = echo ] && [ ! -x Playground/Clients/Quic/bin/Release/net11.0/Playground.Clients.Quic ] \ && skip="Clients/Quic (the driver) not built" - [ "$proto" = h3 ] && [ ! -x "$H3X" ] && skip="h3x missing" + [ "$proto" = h3 ] && [ ! -x "$H2LOAD_H3" ] && skip="no h2load with HTTP/3 (H2LOAD_H3)" [ -z "$(binary "$sample")" ] && skip="not built" if [ -n "$skip" ]; then printf ' %-24s %12s %10s %10s %s\n' "$sample" - - - "skipped: $skip" @@ -241,8 +245,9 @@ while read -r sample proto port path origin extra; do cleanup note=""; rps=${rps:-0} - if [ "$rps" = 0 ]; then note="driver produced no number" - elif bad=$(bad_run "$WORK/$STAMP.txt"); then note="discarded: $bad"; rps=0 + if [ "$(awk -v r="$rps" 'BEGIN{print (r == 0)}')" = 1 ]; then note="driver produced no number"; rps=0 + fi + if bad=$(bad_run "$WORK/$STAMP.txt"); then note="discarded: $bad"; rps=0 fi # As a percentage of REACTORS cores, not one - two pegged reactors burn 200% of a core. diff --git a/bench/lib.sh b/bench/lib.sh index 65f1229..7d5a6c0 100644 --- a/bench/lib.sh +++ b/bench/lib.sh @@ -94,6 +94,9 @@ bench_assert_single_listener() { # Scale multiplies the concurrency and exists for the headroom probe below; a measured run always # uses 1. +# HTTP/3 needs an h2load built with ngtcp2/nghttp3 - most distributions' is not, and reports 0 started. +H2LOAD_H3=${H2LOAD_H3:-$HOME/h3x/bench/h2load} + bench_load() { local proto=$1 port=$2 secs=$3 out=$4 path=$5 scale=${6:-1} local conns=$((CONNS * scale)) threads=$THREADS streams=$((32 * scale)) @@ -121,10 +124,12 @@ bench_load() { PLAYGROUND_ECHO_SECONDS="$secs" "$drv" >"$out" 2>&1 grep -oP '^\K[\d.]+(?= req/s)' "$out" | head -1 ;; h3) - [ -x "$H3X" ] || { echo "" ; return; } - "${pin[@]}" "$H3X" -d "$secs" --connections "$conns" -m "$streams" -k \ - "https://127.0.0.1:$port$path" >"$out" 2>&1 - grep -oP 'throughput:\s+\K[\d.]+' "$out" | head -1 ;; + # Not h3x: its rate divides by the whole run, which waits ~30 s on any request that never + # finishes - a dozen stuck streams made a 300k req/s server read as 50k. + [ -x "$H2LOAD_H3" ] || { echo "" ; return; } + "${pin[@]}" "$H2LOAD_H3" --alpn-list=h3 -D "$secs" -c "$conns" -m "$streams" \ + -t "$(( threads < conns ? threads : conns ))" "https://127.0.0.1:$port$path" >"$out" 2>&1 + grep -oP 'finished in [\d.]+s, \K[\d.]+(?= req/s)' "$out" | head -1 ;; esac } diff --git a/bench/run.sh b/bench/run.sh index 764f778..6343fa0 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -7,7 +7,7 @@ # # Servers run 4 reactors (BENCH_REACTORS to override); the tcp-raw/tcp-pipe pair repeats at 12 # reactors as the io_uring baseline. Loads: wrk for everything TCP (fixed -c 64 -t 4, -# time-bounded), h3x for the HTTP/3 server (H3X to point at the binary), Bench.Clients for the +# time-bounded), h2load built with HTTP/3 for the HTTP/3 server (H2LOAD_H3 to point at it), Bench.Clients for the # ring-native clients. Sidecars (redis, postgres, an h2c nginx) start via docker when available; # anything missing skips with a note instead of failing the run. # @@ -17,7 +17,6 @@ cd "$(dirname "$0")/.." R=${BENCH_REACTORS:-4} DUR=${BENCH_SECONDS:-8} -H3X=${H3X:-/home/diogo/h3x/build/h3x} # The drivers live in one place. This file used to carry its own HTTP/3 invocation that differed # from any.sh's, and the same server measured 260k under one and 505k under the other. @@ -93,12 +92,12 @@ fi # ── h3 server ─────────────────────────────────────────────────────────────────────────────── play Http3/Nghttp3Request PLAYGROUND_REACTORS=$R PLAYGROUND_QUIC_PORT=18444 PLAYGROUND_PORT=18090 -if [ -x "$H3X" ]; then +if [ -x "$H2LOAD_H3" ]; then N=$(CONNS=${CONNS:-64} THREADS=${THREADS:-8} REACTORS=$R \ bench_load h3 18444 "$DUR" "$BIN/h3-server.txt" / | cut -d. -f1) note "h3-server" "${R}r" "${N:-fail} req/s" else - note "h3-server" "${R}r" "SKIP (h3x not found - set H3X=/path/to/h3x)" + note "h3-server" "${R}r" "SKIP (no h2load with HTTP/3 - set H2LOAD_H3=/path/to/h2load)" fi # the h3 server stays up: the h3 client bench below dials it diff --git a/bench/samples.tsv b/bench/samples.tsv index 42b2dc4..591a1ff 100644 --- a/bench/samples.tsv +++ b/bench/samples.tsv @@ -11,7 +11,7 @@ # h1s HTTP/1.1 over TLS (wrk) # h2c HTTP/2 prior-knowledge, cleartext (h2load) # h2 HTTP/2 over TLS, ALPN (h2load) -# h3 HTTP/3 over QUIC (h3x) +# h3 HTTP/3 over QUIC (h2load built with HTTP/3) # echo raw stream echo, no HTTP (Playground/Clients/Quic drives it) # driver a sample that GENERATES load rather than serving it # none nothing here can drive it - the mutual-TLS samples need a client certificate,