Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 7 additions & 8 deletions crates/backend/src/backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,7 @@ pub struct PoolFailOpen<'a> {
pub thresholds: FailOpenThresholds,
}

/// Constructs a [`Backend`] for a server address — the seam that lets the
/// route builder run over mock backends without opening sockets.
///
/// SYNC and I/O-free: connections are lazy, so building a route graph over a
/// dead server succeeds — the server just starts life failing (and TKOs).
/// Address shape is validated by `rusty-mcrouter-config`.
/// constructs a backend without opening a connection
pub trait BackendFactory {
type Backend: Backend;

Expand All @@ -55,8 +50,8 @@ pub trait BackendFactory {
) -> Self::Backend;
}

/// Production route-generation factory, deduplicating destinations through
/// the worker's registry and sharing TKO trackers across workers.
/// production route-generation factory, deduplicating destinations through
/// the worker's registry and sharing TKO trackers across workers
pub struct DestinationFactory<'a> {
destinations: &'a DestinationRegistry,
tko_map: Arc<TkoTrackerMap>,
Expand All @@ -74,16 +69,19 @@ impl<'a> DestinationFactory<'a> {

fn pool_gate(&self, fail_open: Option<PoolFailOpen<'_>>) -> Option<Arc<PoolTkoTracker>> {
let config = fail_open?;

if let Some(gate) = self.pool_gates.borrow().get(config.name) {
return Some(Arc::clone(gate));
}

let gate = self
.tko_map
.pool_tracker_for(config.name, config.thresholds);

self.pool_gates
.borrow_mut()
.insert(Arc::clone(gate.name()), Arc::clone(&gate));

Some(gate)
}
}
Expand All @@ -101,6 +99,7 @@ impl BackendFactory for DestinationFactory<'_> {
addr: Arc::from(server.access_point()),
reply_timeout: cfg.reply_timeout,
};

self.destinations
.get_or_create_destination(key, cfg, self.pool_gate(fail_open))
}
Expand Down
46 changes: 17 additions & 29 deletions crates/backend/src/classify.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,7 @@ pub enum ResultCode {
BadRequest,
/// SERVER_ERROR - box answered that its broken
RemoteError,
/// mid-stream connection drop with this request in-flight - the request
/// was sent but MAY OR MAY NOT have executed (contrast RemoteError, where
/// the server definitively answered). mcrouter folds this into
/// REMOTE_ERROR (a carbon-vocabulary limitation); we keep it distinct so
/// failover config and stats can tell "server rejected" from "maybe
/// executed" - the policy predicates below still treat both like mcrouter
/// connection dropped with the request in flight - it may have executed
ConnectionDropped,
/// never reached the server or the reply stream is busted
LocalError,
Expand All @@ -31,8 +26,6 @@ pub enum ResultCode {
Tko,
}

pub const RESULT_CODE_COUNT: usize = 9;

pub fn code_of(result: &Result<Reply, SendError>) -> ResultCode {
match result {
Ok(reply) => reply_code(reply),
Expand Down Expand Up @@ -64,18 +57,20 @@ impl SendError {
}

impl ResultCode {
pub const ALL: [ResultCode; RESULT_CODE_COUNT] = [
ResultCode::Success,
ResultCode::BadRequest,
ResultCode::RemoteError,
ResultCode::ConnectionDropped,
ResultCode::LocalError,
ResultCode::ConnectError,
ResultCode::ConnectTimeout,
ResultCode::Timeout,
ResultCode::Tko,
pub const ALL: [Self; 9] = [
Self::Success,
Self::BadRequest,
Self::RemoteError,
Self::ConnectionDropped,
Self::LocalError,
Self::ConnectError,
Self::ConnectTimeout,
Self::Timeout,
Self::Tko,
];

pub const COUNT: usize = Self::ALL.len();

pub fn prometheus_label(self) -> &'static str {
match self {
ResultCode::Success => "success",
Expand Down Expand Up @@ -161,12 +156,6 @@ mod tests {
use super::*;
use crate::error::{LocalError, ProtocolError};

const ALL: [ResultCode; RESULT_CODE_COUNT] = ResultCode::ALL;

/// The full SendError -> ResultCode classification table. Every
/// constructible error shape appears exactly once; if a variant is added
/// this test won't cover it, but the exhaustive match in `code()` will
/// refuse to compile until it is classified.
#[test]
fn send_error_classification_table() {
let table: &[(SendError, ResultCode)] = &[
Expand Down Expand Up @@ -264,25 +253,24 @@ mod tests {
/// reason slot round-trips through an AtomicU8).
#[test]
fn from_u8_round_trips() {
for code in ALL {
for code in ResultCode::ALL {
assert_eq!(ResultCode::from_u8(code as u8), code);
}
// out-of-range collapses to Success (never a stale TKO reason)
assert_eq!(ResultCode::from_u8(200), ResultCode::Success);
}

/// RESULT_CODE_COUNT indexes a stats array; every discriminant must fit.
#[test]
fn result_code_count_covers_all_discriminants() {
for code in ALL {
assert!((code as usize) < RESULT_CODE_COUNT, "for {code:?}");
for code in ResultCode::ALL {
assert!((code as usize) < ResultCode::COUNT, "for {code:?}");
}
}

/// The mcrouter McResUtil.h policy sets, as tables.
#[test]
fn tko_and_failover_predicate_sets() {
for code in ALL {
for code in ResultCode::ALL {
// soft == { Timeout } (McResUtil.h:96)
assert_eq!(
code.is_soft_tko_error(),
Expand Down
4 changes: 1 addition & 3 deletions crates/backend/src/connection/actor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -202,10 +202,8 @@ impl Connection {
Ok(n) => {
self.shard_metrics.bytes_read.add(n as u64);


if let Err(e) = self.with_depth_gauges(Self::deliver_replies) {
self.with_depth_gauges(|c| {c.fail_inflight(SendError::Protocol(e));
});
self.with_depth_gauges(|c| c.fail_inflight(SendError::Protocol(e)));

return PipelineExit::Down(DownReason::Protocol);
}
Expand Down
7 changes: 2 additions & 5 deletions crates/backend/src/destination/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,11 @@ use std::{

use rusty_mcrouter_observability_primitives::{Counter, Gauge};

use crate::{
classify::{ResultCode, RESULT_CODE_COUNT},
tko::TkoTracker,
};
use crate::{classify::ResultCode, tko::TkoTracker};

pub struct DestinationMetrics {
pub tracker: Arc<TkoTracker>,
pub requests: [Counter; RESULT_CODE_COUNT],
pub requests: [Counter; ResultCode::COUNT],
pub probes_sent: Gauge,
pub connects: Counter,
pub idle_closes: Counter,
Expand Down
13 changes: 1 addition & 12 deletions crates/backend/src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,6 @@ pub enum SendError {
Request(#[from] RequestError),

/// the reply stream is not trustworthy - kill it
///
/// NOTE: classifies as ResultCode::LocalError, NOT RemoteError, even
/// though the server sent the garbage. The request surface says "no
/// trustworthy verdict on YOUR request" (still failover-eligible); the
/// server-is-broken verdict travels separately via the connection
/// teardown -> ConnEvent::Down -> hard TKO. Reclassifying to RemoteError
/// would double-count the failure. Do not "fix" this.
#[error("protocol error: {0}")]
Protocol(#[from] ProtocolError),

Expand All @@ -43,11 +36,7 @@ pub enum LocalError {
#[error("encode: {0}")]
Encode(#[from] MetaRequestEncodeError),

/// bounded pending queue is full - fail fast and direct request to another
/// backend if eligible. DELIBERATE: classifies as LocalError, which is in
/// the default failover set - a saturated-but-healthy primary sheds
/// overflow to failover children (faithful to mcrouter). If this fires in
/// production, look at capacity, not this enum.
/// bounded pending queue is full; fail fast and allow failover
#[error("pending queue full")]
QueueFull,

Expand Down
15 changes: 4 additions & 11 deletions crates/backend/src/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use rusty_mcrouter_observability_primitives::prometheus::{MetricsSource, Metrics
use rusty_mcrouter_observability_primitives::{Counter, Gauge};
use rusty_mcrouter_protocol::RequestKind;

use crate::classify::{ResultCode, RESULT_CODE_COUNT};
use crate::classify::ResultCode;
use crate::destination::DestinationMetricsRegistry;
use crate::tko::TkoTrackerMap;

Expand All @@ -14,7 +14,7 @@ use crate::tko::TkoTrackerMap;
// thread's shards share one cache line
pub struct BackendMetricsShard {
// monotonic counters
pub requests: [[Counter; RESULT_CODE_COUNT]; RequestKind::COUNT],
pub requests: [[Counter; ResultCode::COUNT]; RequestKind::COUNT],
pub latency_us_sum: Counter,
pub connections_opened: Counter,
pub connections_closed: Counter,
Expand Down Expand Up @@ -151,11 +151,6 @@ impl MetricsSource for TkoTrackerMap {
&[("kind", "hard")],
global.hard_tkos.load(),
);
out.gauge(
"rusty_mcrouter_suspect_servers",
&[],
self.sus_servers().len() as i64,
);

for gate in self.pool_snapshot() {
let pool = &[("pool", &**gate.name())];
Expand Down Expand Up @@ -320,8 +315,6 @@ mod tests {
);
}

/// RequestKind::COUNT indexes the counter array; every discriminant must
/// fit. (same pattern as classify.rs's RESULT_CODE_COUNT test)
#[test]
fn request_kind_discriminants_fit_the_array() {
for kind in RequestKind::ALL {
Expand All @@ -335,14 +328,14 @@ mod tests {
fn every_command_result_cell_is_distinct() {
let metrics = BackendMetricsShard::new();
for kind in 0..RequestKind::COUNT {
for code in 0..RESULT_CODE_COUNT {
for code in 0..ResultCode::COUNT {
let cell = &metrics.requests[kind][code];
assert_eq!(cell.load(), 0);
cell.inc();
}
}
let total: u64 = metrics.requests.iter().flatten().map(Counter::load).sum();
assert_eq!(total, (RequestKind::COUNT * RESULT_CODE_COUNT) as u64);
assert_eq!(total, (RequestKind::COUNT * ResultCode::COUNT) as u64);
}

#[test]
Expand Down
101 changes: 5 additions & 96 deletions crates/backend/src/tko/map.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ use crate::tko::{
};

pub struct TkoTrackerMap {
// server address (host:port) to tracker
trackers: Mutex<HashMap<Arc<str>, Weak<TkoTracker>>>,
// by name, like mcrouter (TkoTracker.cpp:281-298): PoolIds renumber on reload
// pool name to tracker
pool_trackers: Mutex<HashMap<Arc<str>, Weak<PoolTkoTracker>>>,
metrics: Arc<GlobalTkoMetrics>,
sink: TkoEventSink,
Expand Down Expand Up @@ -48,8 +49,8 @@ impl TkoTrackerMap {
tracker
}

/// A live gate is returned as-is, keeping its state and its original
/// thresholds (mcrouter semantics).
/// reuse a live tracker for this pool_name, thresholds are only applied
/// when creating a new tracker
pub fn pool_tracker_for(
self: &Arc<Self>,
pool_name: &str,
Expand All @@ -68,24 +69,6 @@ impl TkoTrackerMap {
tracker
}

pub fn sus_servers(&self) -> Vec<(Arc<str>, bool, u64)> {
let trackers: Vec<_> = self
.trackers
.lock()
.unwrap()
.iter()
.filter_map(|(key, weak)| weak.upgrade().map(|t| (Arc::clone(key), t)))
.collect();

trackers
.into_iter()
.filter_map(|(key, t)| {
let failures = t.consecutive_failures();
(failures > 0).then(|| (key, t.is_tko(), failures))
})
.collect()
}

pub fn pool_snapshot(&self) -> Vec<Arc<PoolTkoTracker>> {
let mut pools = self.pool_trackers.lock().unwrap();

Expand Down Expand Up @@ -202,29 +185,10 @@ mod tests {
assert_eq!(map.pool_trackers.lock().unwrap().len(), 1);
}

#[test]
fn sus_servers_reports_only_failing_trackers() {
let allocator = DestTokenAllocator::new();
let map = TkoTrackerMap::new(noop_sink());
let bad = map.tracker_for("bad:1", 3);
let _good = map.tracker_for("good:1", 3);
assert!(!bad.record_soft_failure(allocator.allocate(), ResultCode::Timeout));

let sus = map.sus_servers();
assert_eq!(sus.len(), 1);
assert_eq!(&*sus[0].0, "bad:1");
assert!(!sus[0].1, "one failure of three is suspect, not TKO");
assert_eq!(sus[0].2, 1);
}

// ── contention suite: std::thread, no tokio ─────────────────────────

use crate::tko::TkoEvent;
use std::sync::{
atomic::{AtomicU64, AtomicUsize, Ordering},
mpsc, Barrier,
};
use std::time::Duration;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};

/// THE invariant the whole encoding exists for: across N threads
/// hammering soft failures with distinct tokens, every TKO episode has
Expand Down Expand Up @@ -403,59 +367,4 @@ mod tests {
assert!(tracker.record_success(tokens[winner_idx.load(Ordering::SeqCst)]));
assert_eq!(map.global_metrics().total(), 0);
}

#[test]
fn sus_servers_does_not_deadlock_with_final_owner_drops() {
let allocator = Arc::new(DestTokenAllocator::new());
const WORKERS: usize = 4;
const ROUNDS: usize = 5_000;

let map = TkoTrackerMap::new(noop_sink());
let active = Arc::new(AtomicUsize::new(WORKERS));
let start = Arc::new(Barrier::new(WORKERS + 1));
let (done_tx, done_rx) = mpsc::channel();
let mut threads = Vec::with_capacity(WORKERS + 1);

for worker in 0..WORKERS {
let allocator = Arc::clone(&allocator);
let map = Arc::clone(&map);
let active = Arc::clone(&active);
let start = Arc::clone(&start);
let done_tx = done_tx.clone();
threads.push(std::thread::spawn(move || {
start.wait();
for round in 0..ROUNDS {
let tracker = map.tracker_for(&format!("churn:{worker}:{round}"), 3);
tracker.record_soft_failure(allocator.allocate(), ResultCode::Timeout);
std::thread::yield_now();
drop(tracker);
}
active.fetch_sub(1, Ordering::SeqCst);
done_tx.send(()).unwrap();
}));
}

let scan_map = Arc::clone(&map);
let scan_active = Arc::clone(&active);
let scan_start = Arc::clone(&start);
let scan_done = done_tx.clone();
threads.push(std::thread::spawn(move || {
scan_start.wait();
while scan_active.load(Ordering::SeqCst) != 0 {
let _ = scan_map.sus_servers();
std::thread::yield_now();
}
scan_done.send(()).unwrap();
}));
drop(done_tx);

for _ in 0..WORKERS + 1 {
done_rx
.recv_timeout(Duration::from_secs(10))
.expect("tracker churn and suspect scans must not deadlock");
}
for thread in threads {
thread.join().unwrap();
}
}
}
Loading
Loading