From b3c8ee9dbb054c9009c379691bf92a76e8a8931f Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Sat, 26 Sep 2026 14:15:45 +0200 Subject: [PATCH 1/3] test(adaptive): io_uring capped to none must not be viable (celeris#679) Failing-first: on 9f4d89b ioUringViable ignores the probed tier, so CELERIS_MAX_IOURING_TIER=none on a 6.10+ kernel (GitHub's runners are on 6.17) leaves the conns-per-worker up-switch enabled and a high-concurrency hint picks an io_uring start that iouring.New then refuses. The fix follows in the next commit. --- adaptive/start_test.go | 97 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 94 insertions(+), 3 deletions(-) diff --git a/adaptive/start_test.go b/adaptive/start_test.go index 71aaa498..6bbdbaec 100644 --- a/adaptive/start_test.go +++ b/adaptive/start_test.go @@ -3,23 +3,29 @@ package adaptive import ( + "bytes" + "context" + "log/slog" "testing" "github.com/goceleris/celeris/engine" + "github.com/goceleris/celeris/probe" "github.com/goceleris/celeris/resource" ) // viableProfile is a modern kernel with the full io_uring fast tier. func viableProfile() engine.CapabilityProfile { return engine.CapabilityProfile{ - KernelMajor: 6, KernelMinor: 12, + KernelMajor: 6, KernelMinor: 12, IOUringTier: engine.Optional, DeferTaskrun: true, SingleIssuer: true, MultishotRecv: true, ProvidedBuffers: true, } } -// oldProfile is a pre-fast-tier kernel (io_uring not worth running). +// oldProfile is a pre-fast-tier kernel (io_uring not worth running). Its tier +// is the one 5.15 probes as (Base), so it is the kernel/feature test that +// rejects it, not the availability test. func oldProfile() engine.CapabilityProfile { - return engine.CapabilityProfile{KernelMajor: 5, KernelMinor: 15} + return engine.CapabilityProfile{KernelMajor: 5, KernelMinor: 15, IOUringTier: engine.Base} } // withMemlock overrides the memlock probe for the duration of a test. @@ -90,3 +96,88 @@ func TestIOUringViable(t *testing.T) { t.Fatal("memlock-starved should be non-viable") } } + +// TestIOUringViable_TierUnavailable pins celeris#679 item 4. The profile is +// what probe.Probe returns on a 6.12 kernel under CELERIS_MAX_IOURING_TIER=none +// (probe.capIOUringTier with maxTier None clears IOUringTier and every feature +// flag, and leaves KernelMajor/KernelMinor alone), which is also what it +// returns when io_uring_setup is refused there (a default seccomp profile, +// kernel.io_uring_disabled). iouring.New refuses to build an engine on that +// profile, so io_uring is not viable; the kernel-version branch alone used to +// say it was. +func TestIOUringViable_TierUnavailable(t *testing.T) { + withMemlock(t, -1) + capped := engine.CapabilityProfile{KernelMajor: 6, KernelMinor: 12, IOUringTier: engine.None} + if ioUringViable(capped, resource.Config{}) { + t.Error("io_uring tier None on a 6.12 kernel is viable: every promotion would fail to build the engine") + } + // Even a profile that (impossibly) kept the fast-tier flags is not viable + // without an available tier: availability is checked on its own. + flagsButNone := viableProfile() + flagsButNone.IOUringTier = engine.None + if ioUringViable(flagsButNone, resource.Config{}) { + t.Error("io_uring tier None with fast-tier flags is viable") + } + // Control: capping at base leaves a buildable engine, and on a 6.12 + // kernel the bundles-era branch still makes it viable. The availability + // test must not reject it. + base := engine.CapabilityProfile{KernelMajor: 6, KernelMinor: 12, IOUringTier: engine.Base, LinkedSQEs: true} + if !ioUringViable(base, resource.Config{}) { + t.Error("io_uring tier Base on a 6.12 kernel is not viable, but iouring.New builds it") + } +} + +// TestNew_TierCapNoneNeverPromotes is the same defect through the production +// path: probe.Probe with the real CELERIS_MAX_IOURING_TIER=none, then New. It +// discriminates only on a 6.10+ kernel, where the unfixed code called io_uring +// viable; below that the capped profile fails the fast-tier test anyway and +// the test passes on either code (it says which, so a green run on an old +// kernel is not read as proof). GitHub's ubuntu runners are on 6.17. +func TestNew_TierCapNoneNeverPromotes(t *testing.T) { + withMemlock(t, -1) + t.Setenv("CELERIS_ADAPTIVE_START", "") + t.Setenv("CELERIS_MAX_IOURING_TIER", "none") + p := probe.Probe() + if p.IOUringTier != engine.None { + t.Fatalf("precondition: CELERIS_MAX_IOURING_TIER=none must cap the probed tier to none, got %v", p.IOUringTier) + } + discriminating := p.KernelMajor > 6 || (p.KernelMajor == 6 && p.KernelMinor >= 10) + t.Logf("kernel %s: discriminating=%v", p.KernelVersion, discriminating) + + for _, tc := range []struct { + name string + hint resource.WorkloadHint + }{ + {"no-hint", resource.WorkloadUnspecified}, + {"high-concurrency-hint", resource.WorkloadHighConcurrency}, + } { + hint := tc.hint + t.Run(tc.name, func(t *testing.T) { + var logs bytes.Buffer + cfg := resource.Config{ + Addr: "127.0.0.1:0", + Protocol: engine.HTTP1, + Resources: resource.Resources{WorkloadHint: hint}, + Logger: slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelWarn})), + } + e, err := New(cfg, noopHandler{}, nil) + if err != nil { + t.Fatalf("New: %v", err) + } + // Release what New built (its start engine) the way a server does. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _ = e.Listen(ctx) + + if e.startType != engine.Epoll { + t.Errorf("start engine = %v, want epoll: io_uring is capped to none", e.startType) + } + if e.ctrl.connSwitchEnabled { + t.Error("the conns-per-worker UP switch is enabled with io_uring capped to none: every promotion builds an io_uring engine that iouring.New refuses, then backs off with a WARN") + } + if logs.Len() > 0 { + t.Errorf("New logged at WARN or above with io_uring capped to none:\n%s", logs.String()) + } + }) + } +} From db3cc3bfa47ec7664f6b80612f9d784345c386a5 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Sat, 26 Sep 2026 14:23:59 +0200 Subject: [PATCH 2/3] fix(adaptive): treat io_uring capped to none as not viable; correct stale io_uring env docs (celeris#679) ioUringViable ignored the probed io_uring tier. CELERIS_MAX_IOURING_TIER=none clears the tier and every feature flag but not the kernel version, so on a 6.10+ kernel the "bundles era" branch alone called io_uring viable. The same holds when io_uring_setup is refused on such a kernel (a default container seccomp profile, kernel.io_uring_disabled), because the probe then reports tier None too. With io_uring deemed viable: - the conns-per-worker up-switch stayed enabled, and each promotion built an io_uring engine that iouring.New refuses ("io_uring not available on this system") and backed off with a WARN; - a high-concurrency hint chose an io_uring start that fell back to epoll with a WARN. ioUringViable now checks IOUringTier.Available() first, which is the predicate iouring.New uses, so the two cannot disagree. The test-only commit before this one failed on GitHub's 6.17 runner, in CI run 36241380107 (Adaptive job: 103 PASS, 4 FAIL, 0 SKIP). It failed TestIOUringViable_TierUnavailable and both subtests of TestNew_TierCapNoneNeverPromotes, with the up-switch enabled and New logging "io_uring start engine unavailable, falling back to epoll start". Comment corrections, from #679: 1. engine/iouring/doc.go CELERIS_IOURING_PBUF_COUNT: a value that is not a power of 2 is rounded up to one and clamped to [1024, 32768]. It is not clamped to [16, 32768], and it does not fail ring registration. The envPbufCount comment in worker.go carried the same claim. 2. doc.go and probe.go (resolveSendZCPolicy) CELERIS_IOURING_SEND_ZC: they no longer name the closed #465. Whether "auto" should keep enabling SEND_ZC is open and owned by celeris#585, so nothing claims it is decided. 3. The adaptive.Engine comment said CELERIS_ADAPTIVE_START "disables the runtime switch". It only chooses the start engine, and determinism comes from Config.Engine. 4. doc.go and README: CELERIS_MAX_IOURING_TIER is not used by CI. The README now also says what `none` does to Adaptive. --- README.md | 2 +- adaptive/engine.go | 32 ++++++++++++++++++++++++-------- engine/iouring/doc.go | 19 +++++++++++-------- engine/iouring/probe.go | 4 ++-- engine/iouring/worker.go | 3 ++- 5 files changed, 40 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 6162aa4d..57874839 100644 --- a/README.md +++ b/README.md @@ -336,7 +336,7 @@ The engines read these at startup. None is needed for normal operation; to run a | Variable | Engine | Values (default in bold) | Effect | |----------|--------|--------------------------|--------| | `CELERIS_ADAPTIVE_START` | Adaptive | `epoll`, `iouring`, **`auto`** | Chooses the engine Adaptive **starts** on. It does not turn off runtime switching. Unrecognized values mean `auto`. | -| `CELERIS_MAX_IOURING_TIER` | io_uring | `optional`, `high`, `base`, `none` (**unset: detected tier**) | Caps the io_uring feature tier below what the kernel supports; for exercising fallback paths. Any other value, typos included, counts as `none`, and at `none` the io_uring engine reports io_uring as unavailable. | +| `CELERIS_MAX_IOURING_TIER` | io_uring | `optional`, `high`, `base`, `none` (**unset: detected tier**) | Caps the io_uring feature tier below what the kernel supports; for exercising fallback paths. Any other value, typos included, counts as `none`, and at `none` the io_uring engine reports io_uring as unavailable and Adaptive neither starts on io_uring nor switches to it. | | `CELERIS_IOURING_SEND_ZC` | io_uring | `on`/`1`/`true`, `off`/`0`/`false`, **`auto`** | Zero-copy send. `auto` enables it where the startup probe finds SEND_ZC working; `on` cannot enable it where the probe failed. Unrecognized values mean `auto` and log a warning. | | `CELERIS_IOURING_MULTISHOT_RECV` | io_uring | `1` (**unset: off**) | Multishot receive into a provided-buffer ring (high tier, 5.19+). Any value other than `1` leaves it off. | | `CELERIS_IOURING_PBUF_COUNT` | io_uring | positive integer (**1024**) | Provided-buffer-ring entries per worker; used only with multishot receive. Rounded up to a power of two and clamped to 1024–32768. `0` or an invalid value keeps the default. | diff --git a/adaptive/engine.go b/adaptive/engine.go index 97d12a0b..445dbdbe 100644 --- a/adaptive/engine.go +++ b/adaptive/engine.go @@ -46,11 +46,12 @@ var ( // most complex path in this package and has historically been the source of // rare, hard-to-reproduce issues; the SwitchRejectedCount / // EngineMetrics.AdaptiveSwitches counters exist so a throughput anomaly can be -// correlated with switching activity. Operators who need fully deterministic -// behaviour can pin the start engine via CELERIS_ADAPTIVE_START=epoll|iouring -// (see chooseStartEngine), which disables the runtime switch. For benchmarking, -// run the adaptive columns multiple times: a rare switch transient can skew a -// single pass. +// correlated with switching activity. CELERIS_ADAPTIVE_START=epoll|iouring +// chooses only the engine it STARTS on (see chooseStartEngine, the variable's +// only reader); the controller still switches afterwards. Operators who need +// fully deterministic behaviour should pin a single engine with Config.Engine +// (Epoll or IOUring) instead. For benchmarking, run the adaptive columns +// multiple times: a rare switch transient can skew a single pass. type Engine struct { primary engine.Engine // epoll (nil until built when it is the lazy standby) secondary engine.Engine // io_uring (nil until built when it is the lazy standby) @@ -121,9 +122,21 @@ type Engine struct { } // ioUringViable reports whether io_uring is worth running at all on this host: -// the kernel must expose the fast tier AND RLIMIT_MEMLOCK must be able to fund -// the requested worker count. These are the two t0-knowable disqualifiers from -// the epoll-vs-io_uring sweep: +// the probed tier must be available at all, the kernel must expose the fast +// tier AND RLIMIT_MEMLOCK must be able to fund the requested worker count. +// +// - Availability: iouring.New refuses to build an engine when the probed +// IOUringTier is None ("io_uring not available on this system"), and the +// probe reports None when CELERIS_MAX_IOURING_TIER caps it there (or when +// io_uring is missing or blocked). The kernel-version test below cannot see +// that: the cap clears the feature flags but not KernelMajor/KernelMinor, so +// on a 6.10+ kernel the "bundles era" branch alone used to call io_uring +// viable, and every promotion then failed to build it and backed off with +// a WARN (celeris#679). Checked first, with iouring.New's own predicate, so +// the two cannot disagree. +// +// The other two are the t0-knowable disqualifiers from the epoll-vs-io_uring +// sweep: // // - Kernel/feature: io_uring loses to epoll on old kernels (missing the // fast-path setup flags); require the "bundles" era (>6.10) OR the 6.1+ @@ -134,6 +147,9 @@ type Engine struct { // does not memlock buffer rings, so it keeps all workers. In that case // io_uring is never the right engine. func ioUringViable(p engine.CapabilityProfile, cfg resource.Config) bool { + if !p.IOUringTier.Available() { + return false + } bundlesEra := p.KernelMajor > 6 || (p.KernelMajor == 6 && p.KernelMinor >= 10) fastTier := p.DeferTaskrun && p.SingleIssuer && p.MultishotRecv && p.ProvidedBuffers if !bundlesEra && !fastTier { diff --git a/engine/iouring/doc.go b/engine/iouring/doc.go index 76333f5c..a76365a3 100644 --- a/engine/iouring/doc.go +++ b/engine/iouring/doc.go @@ -8,18 +8,21 @@ // // - CELERIS_IOURING_SEND_ZC: Controls io_uring zero-copy send (IORING_OP_SEND_ZC). // Values: "on" ("1", "true") forces zero-copy send on if the functional probe passed; -// "off" ("0", "false") forces plain SEND; "auto" or unset preserves current default behavior -// (enabled on kernels where the functional probe passes). Final default decision pending -// measured cluster A/B benchmarks (celeris#465). +// "off" ("0", "false") forces plain SEND; "auto" or unset enables it wherever the startup +// functional probe passes, and "on" cannot enable it where the probe failed. Any other +// value is treated as "auto" and logs a warning. Whether "auto" should keep enabling it is +// an open measurement, owned by celeris#585 (SEND_ZC on/off A/B on the real fabric). // // - CELERIS_IOURING_MULTISHOT_RECV: Opts into multishot receive with provided buffer rings // (IORING_REGISTER_PBUF_RING + IORING_RECV_MULTISHOT). Set to "1" to enable. Disabled by default. // -// - CELERIS_IOURING_PBUF_COUNT: Overrides the auto-scaled provided-buffer-ring size per worker. -// Must be a power of 2 (e.g. 1024, 2048, 4096); values are clamped to [16, 32768]. -// Non-power-of-2 values cause ring registration failure and automatic fallback to -// single-shot per-connection buffers. +// - CELERIS_IOURING_PBUF_COUNT: Overrides the auto-scaled provided-buffer-ring size per worker +// (used only with multishot receive). A positive value is rounded up to the next power of +// 2 and clamped to [1024, 32768] (bufRingCountMin, bufRingCountMax); 0, a negative value or +// a non-integer keeps the auto-scaled size. See resolveBufRingCount. // // - CELERIS_MAX_IOURING_TIER: Caps the detected io_uring tier at startup ("none", "base", "high", -// "optional"). Used primarily by CI to exercise lower-tier fallback paths on modern kernels. +// "optional"), to exercise lower-tier fallback paths on modern kernels. Any other value +// counts as "none". At "none" this engine reports io_uring as unavailable, and the adaptive +// engine treats io_uring as not viable, so it neither starts on it nor switches to it. package iouring diff --git a/engine/iouring/probe.go b/engine/iouring/probe.go index 50a85547..dae4ef1f 100644 --- a/engine/iouring/probe.go +++ b/engine/iouring/probe.go @@ -291,8 +291,8 @@ func parseSendZCResult(initialRes int32, initialFlags uint32, notifArrived bool, // Values: // - "on", "1", "true": force enabled if functional probe passed. // - "off", "0", "false": force disabled. -// - "auto", "" (default): preserves current default behavior (enabled when functional probe -// passed). Final default decision pending cluster A/B fabric benchmark (celeris#465). +// - "auto", "" (default): enabled when the functional probe passed. Whether that stays the +// default is an open measurement owned by celeris#585 (SEND_ZC on/off A/B on the fabric). // - any other value: returns recognized=false and falls back to auto behavior. func resolveSendZCPolicy(functional bool, envVal string) (enabled, recognized bool) { if !functional { diff --git a/engine/iouring/worker.go b/engine/iouring/worker.go index bd7c1451..c14795bf 100644 --- a/engine/iouring/worker.go +++ b/engine/iouring/worker.go @@ -81,7 +81,8 @@ const bufRingCountMin = 1024 const bufRingCountMax = 1 << 15 // 32768 entries × 8 KiB = 256 MiB worst case (kernel PBUF_RING cap) // CELERIS_IOURING_PBUF_COUNT overrides the auto-scaled provided-buffer-ring -// size. Must be a power of 2 and at least bufRingCountMin. Use this when +// size. A value that is not a power of 2 is rounded up to one, and the result +// is clamped to [bufRingCountMin, bufRingCountMax]. Use this when // the default scaling formula under-provisions your workload — typically // the case for very-high-concurrency benchmarks (16k+ connections) where // each worker may have more in-flight multishot recvs than the formula From 450d538f166755678574334060a252bec4c3ee45 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Sat, 26 Sep 2026 18:14:34 +0200 Subject: [PATCH 3/3] docs(probe, iouring): no CI claim for the tier cap; exact SEND_ZC warning condition (celeris#679) - probe.Probe's godoc still said the tier cap "allows CI to exercise every tier's code path". No workflow sets CELERIS_MAX_IOURING_TIER. It now says what the cap does and what it leaves alone: any other non-empty value counts as none, and the kernel version stays as detected (the fact behind #679's bug). - engine/iouring/doc.go introduced its knobs as being "for CI matrix testing". No workflow sets any of them; tests do, with t.Setenv. - An unrecognized CELERIS_IOURING_SEND_ZC value is logged only when the probed profile has SEND_ZC (optional tier) and the functional probe passed: engine.go reads the variable inside `if profile.SendZC`, and resolveSendZCPolicy returns (false, true) before looking at it when the probe failed. doc.go, the README row and resolveSendZCPolicy's comment said, or implied, that it is always logged. --- README.md | 2 +- engine/iouring/doc.go | 8 +++++--- engine/iouring/probe.go | 6 +++++- probe/probe.go | 7 ++++--- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 57874839..c7a851e9 100644 --- a/README.md +++ b/README.md @@ -337,7 +337,7 @@ The engines read these at startup. None is needed for normal operation; to run a |----------|--------|--------------------------|--------| | `CELERIS_ADAPTIVE_START` | Adaptive | `epoll`, `iouring`, **`auto`** | Chooses the engine Adaptive **starts** on. It does not turn off runtime switching. Unrecognized values mean `auto`. | | `CELERIS_MAX_IOURING_TIER` | io_uring | `optional`, `high`, `base`, `none` (**unset: detected tier**) | Caps the io_uring feature tier below what the kernel supports; for exercising fallback paths. Any other value, typos included, counts as `none`, and at `none` the io_uring engine reports io_uring as unavailable and Adaptive neither starts on io_uring nor switches to it. | -| `CELERIS_IOURING_SEND_ZC` | io_uring | `on`/`1`/`true`, `off`/`0`/`false`, **`auto`** | Zero-copy send. `auto` enables it where the startup probe finds SEND_ZC working; `on` cannot enable it where the probe failed. Unrecognized values mean `auto` and log a warning. | +| `CELERIS_IOURING_SEND_ZC` | io_uring | `on`/`1`/`true`, `off`/`0`/`false`, **`auto`** | Zero-copy send. `auto` enables it where the startup probe finds SEND_ZC working; `on` cannot enable it where the probe failed. Unrecognized values mean `auto`; one is logged as a warning only where the probe finds SEND_ZC working (elsewhere the variable has no effect). | | `CELERIS_IOURING_MULTISHOT_RECV` | io_uring | `1` (**unset: off**) | Multishot receive into a provided-buffer ring (high tier, 5.19+). Any value other than `1` leaves it off. | | `CELERIS_IOURING_PBUF_COUNT` | io_uring | positive integer (**1024**) | Provided-buffer-ring entries per worker; used only with multishot receive. Rounded up to a power of two and clamped to 1024–32768. `0` or an invalid value keeps the default. | | `CELERIS_IOURING_FIXED_FILES` | io_uring | **do not set** | Development only. Fixed-file support is incomplete ([#541](https://github.com/goceleris/celeris/issues/541)); enabling it makes connections read from unrelated descriptors. | diff --git a/engine/iouring/doc.go b/engine/iouring/doc.go index a76365a3..6c53f525 100644 --- a/engine/iouring/doc.go +++ b/engine/iouring/doc.go @@ -4,14 +4,16 @@ // // # Environment Knobs // -// The engine recognizes several environment variables for operator control and CI matrix testing: +// The engine recognizes several environment variables, for operator control and for tests: // // - CELERIS_IOURING_SEND_ZC: Controls io_uring zero-copy send (IORING_OP_SEND_ZC). // Values: "on" ("1", "true") forces zero-copy send on if the functional probe passed; // "off" ("0", "false") forces plain SEND; "auto" or unset enables it wherever the startup // functional probe passes, and "on" cannot enable it where the probe failed. Any other -// value is treated as "auto" and logs a warning. Whether "auto" should keep enabling it is -// an open measurement, owned by celeris#585 (SEND_ZC on/off A/B on the real fabric). +// value is treated as "auto". It is logged as a warning only when the probed profile has +// SEND_ZC (the optional tier) and the functional probe passed, because only then is the +// value examined. Whether "auto" should keep enabling it is an open measurement, owned by +// celeris#585 (SEND_ZC on/off A/B on the real fabric). // // - CELERIS_IOURING_MULTISHOT_RECV: Opts into multishot receive with provided buffer rings // (IORING_REGISTER_PBUF_RING + IORING_RECV_MULTISHOT). Set to "1" to enable. Disabled by default. diff --git a/engine/iouring/probe.go b/engine/iouring/probe.go index dae4ef1f..3fcc32a4 100644 --- a/engine/iouring/probe.go +++ b/engine/iouring/probe.go @@ -293,7 +293,11 @@ func parseSendZCResult(initialRes int32, initialFlags uint32, notifArrived bool, // - "off", "0", "false": force disabled. // - "auto", "" (default): enabled when the functional probe passed. Whether that stays the // default is an open measurement owned by celeris#585 (SEND_ZC on/off A/B on the fabric). -// - any other value: returns recognized=false and falls back to auto behavior. +// - any other value: falls back to auto behavior, and returns recognized=false. +// +// When the functional probe failed, the value is not examined: the result is +// (false, true) whatever it is, so an unrecognized value is reported only when +// the probe passed. func resolveSendZCPolicy(functional bool, envVal string) (enabled, recognized bool) { if !functional { return false, true diff --git a/probe/probe.go b/probe/probe.go index a4b784ad..48041a85 100644 --- a/probe/probe.go +++ b/probe/probe.go @@ -8,9 +8,10 @@ import ( ) // Probe detects system capabilities using the platform-default syscall prober. -// If CELERIS_MAX_IOURING_TIER is set (none/base/high/optional), the detected -// io_uring tier and associated features are capped at that level. This allows -// CI to exercise every tier's code path on modern kernels. +// If CELERIS_MAX_IOURING_TIER is set (none/base/high/optional; any other +// non-empty value counts as none), the detected io_uring tier and associated +// features are capped at that level, so a lower tier's code paths can be +// exercised on a modern kernel. The kernel version is left as detected. func Probe() engine.CapabilityProfile { profile := ProbeWith(defaultProber()) if maxTier := os.Getenv("CELERIS_MAX_IOURING_TIER"); maxTier != "" {