diff --git a/crates/backend/src/backend.rs b/crates/backend/src/backend.rs index 6586912..7046c4f 100644 --- a/crates/backend/src/backend.rs +++ b/crates/backend/src/backend.rs @@ -38,12 +38,7 @@ pub struct PoolFailOpen<'a> { pub thresholds: FailOpenThresholds, } -/// Constructs a [`Backend`] for a server address — the seam that lets the -/// route builder run over mock backends without opening sockets. -/// -/// SYNC and I/O-free: connections are lazy, so building a route graph over a -/// dead server succeeds — the server just starts life failing (and TKOs). -/// Address shape is validated by `rusty-mcrouter-config`. +/// constructs a backend without opening a connection pub trait BackendFactory { type Backend: Backend; @@ -55,8 +50,8 @@ pub trait BackendFactory { ) -> Self::Backend; } -/// Production route-generation factory, deduplicating destinations through -/// the worker's registry and sharing TKO trackers across workers. +/// production route-generation factory, deduplicating destinations through +/// the worker's registry and sharing TKO trackers across workers pub struct DestinationFactory<'a> { destinations: &'a DestinationRegistry, tko_map: Arc, @@ -74,6 +69,7 @@ impl<'a> DestinationFactory<'a> { fn pool_gate(&self, fail_open: Option>) -> Option> { let config = fail_open?; + if let Some(gate) = self.pool_gates.borrow().get(config.name) { return Some(Arc::clone(gate)); } @@ -81,9 +77,11 @@ impl<'a> DestinationFactory<'a> { let gate = self .tko_map .pool_tracker_for(config.name, config.thresholds); + self.pool_gates .borrow_mut() .insert(Arc::clone(gate.name()), Arc::clone(&gate)); + Some(gate) } } @@ -101,6 +99,7 @@ impl BackendFactory for DestinationFactory<'_> { addr: Arc::from(server.access_point()), reply_timeout: cfg.reply_timeout, }; + self.destinations .get_or_create_destination(key, cfg, self.pool_gate(fail_open)) } diff --git a/crates/backend/src/classify.rs b/crates/backend/src/classify.rs index 32a5ed9..c87f7ff 100644 --- a/crates/backend/src/classify.rs +++ b/crates/backend/src/classify.rs @@ -12,12 +12,7 @@ pub enum ResultCode { BadRequest, /// SERVER_ERROR - box answered that its broken RemoteError, - /// mid-stream connection drop with this request in-flight - the request - /// was sent but MAY OR MAY NOT have executed (contrast RemoteError, where - /// the server definitively answered). mcrouter folds this into - /// REMOTE_ERROR (a carbon-vocabulary limitation); we keep it distinct so - /// failover config and stats can tell "server rejected" from "maybe - /// executed" - the policy predicates below still treat both like mcrouter + /// connection dropped with the request in flight - it may have executed ConnectionDropped, /// never reached the server or the reply stream is busted LocalError, @@ -31,8 +26,6 @@ pub enum ResultCode { Tko, } -pub const RESULT_CODE_COUNT: usize = 9; - pub fn code_of(result: &Result) -> ResultCode { match result { Ok(reply) => reply_code(reply), @@ -64,18 +57,20 @@ impl SendError { } impl ResultCode { - pub const ALL: [ResultCode; RESULT_CODE_COUNT] = [ - ResultCode::Success, - ResultCode::BadRequest, - ResultCode::RemoteError, - ResultCode::ConnectionDropped, - ResultCode::LocalError, - ResultCode::ConnectError, - ResultCode::ConnectTimeout, - ResultCode::Timeout, - ResultCode::Tko, + pub const ALL: [Self; 9] = [ + Self::Success, + Self::BadRequest, + Self::RemoteError, + Self::ConnectionDropped, + Self::LocalError, + Self::ConnectError, + Self::ConnectTimeout, + Self::Timeout, + Self::Tko, ]; + pub const COUNT: usize = Self::ALL.len(); + pub fn prometheus_label(self) -> &'static str { match self { ResultCode::Success => "success", @@ -161,12 +156,6 @@ mod tests { use super::*; use crate::error::{LocalError, ProtocolError}; - const ALL: [ResultCode; RESULT_CODE_COUNT] = ResultCode::ALL; - - /// The full SendError -> ResultCode classification table. Every - /// constructible error shape appears exactly once; if a variant is added - /// this test won't cover it, but the exhaustive match in `code()` will - /// refuse to compile until it is classified. #[test] fn send_error_classification_table() { let table: &[(SendError, ResultCode)] = &[ @@ -264,25 +253,24 @@ mod tests { /// reason slot round-trips through an AtomicU8). #[test] fn from_u8_round_trips() { - for code in ALL { + for code in ResultCode::ALL { assert_eq!(ResultCode::from_u8(code as u8), code); } // out-of-range collapses to Success (never a stale TKO reason) assert_eq!(ResultCode::from_u8(200), ResultCode::Success); } - /// RESULT_CODE_COUNT indexes a stats array; every discriminant must fit. #[test] fn result_code_count_covers_all_discriminants() { - for code in ALL { - assert!((code as usize) < RESULT_CODE_COUNT, "for {code:?}"); + for code in ResultCode::ALL { + assert!((code as usize) < ResultCode::COUNT, "for {code:?}"); } } /// The mcrouter McResUtil.h policy sets, as tables. #[test] fn tko_and_failover_predicate_sets() { - for code in ALL { + for code in ResultCode::ALL { // soft == { Timeout } (McResUtil.h:96) assert_eq!( code.is_soft_tko_error(), diff --git a/crates/backend/src/connection/actor.rs b/crates/backend/src/connection/actor.rs index 5f5fe81..d92b78c 100644 --- a/crates/backend/src/connection/actor.rs +++ b/crates/backend/src/connection/actor.rs @@ -202,10 +202,8 @@ impl Connection { Ok(n) => { self.shard_metrics.bytes_read.add(n as u64); - if let Err(e) = self.with_depth_gauges(Self::deliver_replies) { - self.with_depth_gauges(|c| {c.fail_inflight(SendError::Protocol(e)); - }); + self.with_depth_gauges(|c| c.fail_inflight(SendError::Protocol(e))); return PipelineExit::Down(DownReason::Protocol); } diff --git a/crates/backend/src/destination/metrics.rs b/crates/backend/src/destination/metrics.rs index 11d20db..05c33cc 100644 --- a/crates/backend/src/destination/metrics.rs +++ b/crates/backend/src/destination/metrics.rs @@ -5,14 +5,11 @@ use std::{ use rusty_mcrouter_observability_primitives::{Counter, Gauge}; -use crate::{ - classify::{ResultCode, RESULT_CODE_COUNT}, - tko::TkoTracker, -}; +use crate::{classify::ResultCode, tko::TkoTracker}; pub struct DestinationMetrics { pub tracker: Arc, - pub requests: [Counter; RESULT_CODE_COUNT], + pub requests: [Counter; ResultCode::COUNT], pub probes_sent: Gauge, pub connects: Counter, pub idle_closes: Counter, diff --git a/crates/backend/src/error.rs b/crates/backend/src/error.rs index f0f44c6..4d10d87 100644 --- a/crates/backend/src/error.rs +++ b/crates/backend/src/error.rs @@ -20,13 +20,6 @@ pub enum SendError { Request(#[from] RequestError), /// the reply stream is not trustworthy - kill it - /// - /// NOTE: classifies as ResultCode::LocalError, NOT RemoteError, even - /// though the server sent the garbage. The request surface says "no - /// trustworthy verdict on YOUR request" (still failover-eligible); the - /// server-is-broken verdict travels separately via the connection - /// teardown -> ConnEvent::Down -> hard TKO. Reclassifying to RemoteError - /// would double-count the failure. Do not "fix" this. #[error("protocol error: {0}")] Protocol(#[from] ProtocolError), @@ -43,11 +36,7 @@ pub enum LocalError { #[error("encode: {0}")] Encode(#[from] MetaRequestEncodeError), - /// bounded pending queue is full - fail fast and direct request to another - /// backend if eligible. DELIBERATE: classifies as LocalError, which is in - /// the default failover set - a saturated-but-healthy primary sheds - /// overflow to failover children (faithful to mcrouter). If this fires in - /// production, look at capacity, not this enum. + /// bounded pending queue is full; fail fast and allow failover #[error("pending queue full")] QueueFull, diff --git a/crates/backend/src/metrics.rs b/crates/backend/src/metrics.rs index 5c0f883..d63b1e7 100644 --- a/crates/backend/src/metrics.rs +++ b/crates/backend/src/metrics.rs @@ -4,7 +4,7 @@ use rusty_mcrouter_observability_primitives::prometheus::{MetricsSource, Metrics use rusty_mcrouter_observability_primitives::{Counter, Gauge}; use rusty_mcrouter_protocol::RequestKind; -use crate::classify::{ResultCode, RESULT_CODE_COUNT}; +use crate::classify::ResultCode; use crate::destination::DestinationMetricsRegistry; use crate::tko::TkoTrackerMap; @@ -14,7 +14,7 @@ use crate::tko::TkoTrackerMap; // thread's shards share one cache line pub struct BackendMetricsShard { // monotonic counters - pub requests: [[Counter; RESULT_CODE_COUNT]; RequestKind::COUNT], + pub requests: [[Counter; ResultCode::COUNT]; RequestKind::COUNT], pub latency_us_sum: Counter, pub connections_opened: Counter, pub connections_closed: Counter, @@ -151,11 +151,6 @@ impl MetricsSource for TkoTrackerMap { &[("kind", "hard")], global.hard_tkos.load(), ); - out.gauge( - "rusty_mcrouter_suspect_servers", - &[], - self.sus_servers().len() as i64, - ); for gate in self.pool_snapshot() { let pool = &[("pool", &**gate.name())]; @@ -320,8 +315,6 @@ mod tests { ); } - /// RequestKind::COUNT indexes the counter array; every discriminant must - /// fit. (same pattern as classify.rs's RESULT_CODE_COUNT test) #[test] fn request_kind_discriminants_fit_the_array() { for kind in RequestKind::ALL { @@ -335,14 +328,14 @@ mod tests { fn every_command_result_cell_is_distinct() { let metrics = BackendMetricsShard::new(); for kind in 0..RequestKind::COUNT { - for code in 0..RESULT_CODE_COUNT { + for code in 0..ResultCode::COUNT { let cell = &metrics.requests[kind][code]; assert_eq!(cell.load(), 0); cell.inc(); } } let total: u64 = metrics.requests.iter().flatten().map(Counter::load).sum(); - assert_eq!(total, (RequestKind::COUNT * RESULT_CODE_COUNT) as u64); + assert_eq!(total, (RequestKind::COUNT * ResultCode::COUNT) as u64); } #[test] diff --git a/crates/backend/src/tko/map.rs b/crates/backend/src/tko/map.rs index a515172..bee91d3 100644 --- a/crates/backend/src/tko/map.rs +++ b/crates/backend/src/tko/map.rs @@ -8,8 +8,9 @@ use crate::tko::{ }; pub struct TkoTrackerMap { + // server address (host:port) to tracker trackers: Mutex, Weak>>, - // by name, like mcrouter (TkoTracker.cpp:281-298): PoolIds renumber on reload + // pool name to tracker pool_trackers: Mutex, Weak>>, metrics: Arc, sink: TkoEventSink, @@ -48,8 +49,8 @@ impl TkoTrackerMap { tracker } - /// A live gate is returned as-is, keeping its state and its original - /// thresholds (mcrouter semantics). + /// reuse a live tracker for this pool_name, thresholds are only applied + /// when creating a new tracker pub fn pool_tracker_for( self: &Arc, pool_name: &str, @@ -68,24 +69,6 @@ impl TkoTrackerMap { tracker } - pub fn sus_servers(&self) -> Vec<(Arc, bool, u64)> { - let trackers: Vec<_> = self - .trackers - .lock() - .unwrap() - .iter() - .filter_map(|(key, weak)| weak.upgrade().map(|t| (Arc::clone(key), t))) - .collect(); - - trackers - .into_iter() - .filter_map(|(key, t)| { - let failures = t.consecutive_failures(); - (failures > 0).then(|| (key, t.is_tko(), failures)) - }) - .collect() - } - pub fn pool_snapshot(&self) -> Vec> { let mut pools = self.pool_trackers.lock().unwrap(); @@ -202,29 +185,10 @@ mod tests { assert_eq!(map.pool_trackers.lock().unwrap().len(), 1); } - #[test] - fn sus_servers_reports_only_failing_trackers() { - let allocator = DestTokenAllocator::new(); - let map = TkoTrackerMap::new(noop_sink()); - let bad = map.tracker_for("bad:1", 3); - let _good = map.tracker_for("good:1", 3); - assert!(!bad.record_soft_failure(allocator.allocate(), ResultCode::Timeout)); - - let sus = map.sus_servers(); - assert_eq!(sus.len(), 1); - assert_eq!(&*sus[0].0, "bad:1"); - assert!(!sus[0].1, "one failure of three is suspect, not TKO"); - assert_eq!(sus[0].2, 1); - } - // ── contention suite: std::thread, no tokio ───────────────────────── use crate::tko::TkoEvent; - use std::sync::{ - atomic::{AtomicU64, AtomicUsize, Ordering}, - mpsc, Barrier, - }; - use std::time::Duration; + use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; /// THE invariant the whole encoding exists for: across N threads /// hammering soft failures with distinct tokens, every TKO episode has @@ -403,59 +367,4 @@ mod tests { assert!(tracker.record_success(tokens[winner_idx.load(Ordering::SeqCst)])); assert_eq!(map.global_metrics().total(), 0); } - - #[test] - fn sus_servers_does_not_deadlock_with_final_owner_drops() { - let allocator = Arc::new(DestTokenAllocator::new()); - const WORKERS: usize = 4; - const ROUNDS: usize = 5_000; - - let map = TkoTrackerMap::new(noop_sink()); - let active = Arc::new(AtomicUsize::new(WORKERS)); - let start = Arc::new(Barrier::new(WORKERS + 1)); - let (done_tx, done_rx) = mpsc::channel(); - let mut threads = Vec::with_capacity(WORKERS + 1); - - for worker in 0..WORKERS { - let allocator = Arc::clone(&allocator); - let map = Arc::clone(&map); - let active = Arc::clone(&active); - let start = Arc::clone(&start); - let done_tx = done_tx.clone(); - threads.push(std::thread::spawn(move || { - start.wait(); - for round in 0..ROUNDS { - let tracker = map.tracker_for(&format!("churn:{worker}:{round}"), 3); - tracker.record_soft_failure(allocator.allocate(), ResultCode::Timeout); - std::thread::yield_now(); - drop(tracker); - } - active.fetch_sub(1, Ordering::SeqCst); - done_tx.send(()).unwrap(); - })); - } - - let scan_map = Arc::clone(&map); - let scan_active = Arc::clone(&active); - let scan_start = Arc::clone(&start); - let scan_done = done_tx.clone(); - threads.push(std::thread::spawn(move || { - scan_start.wait(); - while scan_active.load(Ordering::SeqCst) != 0 { - let _ = scan_map.sus_servers(); - std::thread::yield_now(); - } - scan_done.send(()).unwrap(); - })); - drop(done_tx); - - for _ in 0..WORKERS + 1 { - done_rx - .recv_timeout(Duration::from_secs(10)) - .expect("tracker churn and suspect scans must not deadlock"); - } - for thread in threads { - thread.join().unwrap(); - } - } } diff --git a/crates/backend/src/tko/pool.rs b/crates/backend/src/tko/pool.rs index a6b643c..b7a9177 100644 --- a/crates/backend/src/tko/pool.rs +++ b/crates/backend/src/tko/pool.rs @@ -5,28 +5,19 @@ use std::sync::{ use rusty_mcrouter_observability_primitives::Counter; -/// Fail-open hysteresis thresholds: the gate opens when `enter` destinations -/// are TKO'd (refusing further marks) and closes again once recoveries drain -/// the count to `exit`. Named fields because two bare u64s in a tuple invite -/// a silent swap — and a swapped enter/exit inverts the hysteresis. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct FailOpenThresholds { - /// "upper": TKO'd-destination count at which the pool fails open. + /// enter fail-open when another TKO mark would exceed this count pub enter: u64, - /// "lower": count at which recoveries close the gate again. + /// exit fail-open on a recovery at this TKO count pub exit: u64, } -/// Outcome of asking the gate for a TKO slot. An enum rather than the old -/// (bool, bool): `Admitted` with `just_entered` was representable nonsense, -/// and two adjacent bools at a call site say nothing about which is which. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum GateDecision { - /// Slot reserved: the counter was incremented, the caller may mark. + /// TKO count incremented Admitted, - /// Fail-open refused the mark; nothing was counted. `just_entered` is - /// true only for the call that crossed `enter`, so EnterFailOpen can be - /// emitted exactly once. + /// no increment; `just_entered` means this call enabled fail-open Refused { just_entered: bool }, } diff --git a/crates/backend/src/tko/tracker.rs b/crates/backend/src/tko/tracker.rs index 444b1a9..ca50c42 100644 --- a/crates/backend/src/tko/tracker.rs +++ b/crates/backend/src/tko/tracker.rs @@ -86,11 +86,8 @@ impl TkoTracker { } } - /// Known limitation, shared with mcrouter (TkoTracker.cpp:69-70, :88-89): - /// marks and unmarks go through whichever gate is attached at that moment, - /// so a server that is TKO'd while a reload moves it to another gated pool - /// is unmarked against the new gate, and the old gate keeps the mark. - /// See docs/design/0002-config-reload.md §known limitations. + /// TODO: replacing the pool while TKO'd leaves the mark in the old pool, + /// but recovery decrements the new pool pub(crate) fn set_pool_tracker(&self, pool: Arc) { *self.pool.lock().unwrap() = Some(pool) } diff --git a/crates/control/src/runtime.rs b/crates/control/src/runtime.rs index 91cb03b..66f192f 100644 --- a/crates/control/src/runtime.rs +++ b/crates/control/src/runtime.rs @@ -350,29 +350,21 @@ mod tests { #[tokio::test] async fn metrics_failure_stops_control() { - let tko_map = TkoTrackerMap::new(noop_sink()); - // Invalid tracker construction panics while holding the map lock. - // Rendering this real metric state then exercises the supervised panic path. - assert!(std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - tko_map.tracker_for("invalid:1", 0); - })) - .is_err()); - let (handle, runtime, _events) = build_runtime(tko_map).await; + let (handle, mut runtime, _events) = build_runtime(TkoTrackerMap::new(noop_sink())).await; let bound = runtime.bound_addr(); - let task = tokio::spawn(runtime.run()); let idle = TcpStream::connect(bound).await.unwrap(); - let mut stream = TcpStream::connect(bound).await.unwrap(); - stream - .write_all(b"GET /metrics HTTP/1.1\r\nHost: localhost\r\n\r\n") - .await - .unwrap(); + let response = scrape(bound).await; + assert!(response.starts_with("HTTP/1.1 200 OK\r\n"), "{response}"); + + runtime.metrics_task.abort_all(); + let task = tokio::spawn(runtime.run()); let error = tokio::time::timeout(Duration::from_secs(5), task) .await .expect("control did not stop after metrics failed") .unwrap() .unwrap_err(); - assert!(format!("{error:#}").contains("metrics connection task failed")); + assert!(format!("{error:#}").contains("metrics task failed")); assert!(handle.workers_ready().await.is_err()); assert_metrics_stopped(bound, idle).await; } diff --git a/docs/design/0001-observability-catalog.md b/docs/design/0001-observability-catalog.md index f43d9f1..a97b20e 100644 --- a/docs/design/0001-observability-catalog.md +++ b/docs/design/0001-observability-catalog.md @@ -57,7 +57,7 @@ blanket rules applied throughout: |----------|----------|-------| | num_client_connections | port | gauge `rusty_mcrouter_client_connections` | | num_servers{,_new,_up,_down,_closed} | defer | destination-up and TKO metrics ship; aggregate server-state families do not | -| num_suspect_servers | port | gauge, straight from `sus_servers` scan | +| num_suspect_servers | n/a | omit the consecutive-failure snapshot count; use per-destination error rates for early warning and destination-up / TKO gauges for current TKO state | | num_connections_opened / _closed | port | counters; we already count connects + idle_closes per destination | | num_connect_retries, num_connect_success_after_retrying | port | counters; connect-retry path exists | | num_ssl_* (10), num_tls_to_plain_* (5), num_ktls_* (5) | n/a | no TLS. revisit wholesale if TLS ships | diff --git a/docs/design/0001-observability.md b/docs/design/0001-observability.md index 8fb8958..1903128 100644 --- a/docs/design/0001-observability.md +++ b/docs/design/0001-observability.md @@ -444,7 +444,7 @@ binary: creates shards, hands them to both sides // the measured leg and the storage shape; this is not per-destination. pub struct BackendMetricsShard { // monotonic counters - pub requests: [[Counter; RESULT_CODE_COUNT]; RequestKind::COUNT], + pub requests: [[Counter; ResultCode::COUNT]; RequestKind::COUNT], pub latency_us_sum: Counter, pub connections_opened: Counter, pub connections_closed: Counter, // incl. idle closes @@ -498,7 +498,7 @@ struct BackendSource { shards: Vec> } impl BackendSource { fn encode(&self, out: &mut String) { - let mut requests = [0u64; RESULT_CODE_COUNT]; + let mut requests = [0u64; ResultCode::COUNT]; for shard in &self.shards { for (acc, c) in requests.iter_mut().zip(&shard.requests) { *acc += c.load(); @@ -542,7 +542,6 @@ implemented metric inventory (names and labels are API): | metric | type | labels | |--------|------|--------| | `rusty_mcrouter_tko` | gauge | `kind` (soft/hard) — GlobalTkoMetrics | -| `rusty_mcrouter_suspect_servers` | gauge | — sus_servers scan | | `rusty_mcrouter_pool_fail_open` | gauge | `pool` | | `rusty_mcrouter_pool_destinations_tko` | gauge | `pool` | | `rusty_mcrouter_fail_open_entered_total` / `_exited_total` | counter | `pool` |