diff --git a/docs/getting-started.md b/docs/getting-started.md index 7f2ab69f9..b25a36c93 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -239,6 +239,13 @@ pgwire = true http = true resp = true ilp = false # Example: disable TLS for ILP ingest + +[tuning.startup] +# Boot bounds for the readiness gates (defaults shown). Raise them when a +# backlogged restart needs minutes of replay; a bound set under [server] is +# rejected at load with the [tuning.startup] path named. +raft_ready_timeout_ms = 300000 # metadata group may stall this long (5 min) +data_group_recovery_timeout_ms = 600000 # data groups must replay within (10 min) ``` Every listener binds during startup, before the server accepts any diff --git a/nodedb-types/src/config/tuning/config.rs b/nodedb-types/src/config/tuning/config.rs index 65bc5795f..ba7e77613 100644 --- a/nodedb-types/src/config/tuning/config.rs +++ b/nodedb-types/src/config/tuning/config.rs @@ -12,6 +12,7 @@ use super::memory::MemoryTuning; use super::network::{BridgeTuning, ClusterTransportTuning, NetworkTuning, WalTuning}; use super::scheduler::SchedulerTuning; use super::shutdown::ShutdownTuning; +use super::startup::StartupTuning; /// Top-level tuning configuration. /// @@ -48,6 +49,8 @@ pub struct TuningConfig { #[serde(default)] pub shutdown: ShutdownTuning, #[serde(default)] + pub startup: StartupTuning, + #[serde(default)] pub bitemporal: BitemporalTuning, #[serde(default)] pub maintenance: MaintenanceTuning, @@ -73,6 +76,8 @@ mod tests { let parsed: TuningConfig = toml::from_str(&toml_str).expect("deserialize"); assert_eq!(parsed.data_plane.idle_poll_timeout_ms, 100); + assert_eq!(parsed.startup.raft_ready_timeout_ms, 300_000); + assert_eq!(parsed.startup.data_group_recovery_timeout_ms, 600_000); assert_eq!(parsed.query.sort_run_size, 100_000); assert_eq!(parsed.vector.flat_index_threshold, 10_000); assert_eq!(parsed.sparse.bm25_k1, 1.2); diff --git a/nodedb-types/src/config/tuning/mod.rs b/nodedb-types/src/config/tuning/mod.rs index d521e851f..e16fc263d 100644 --- a/nodedb-types/src/config/tuning/mod.rs +++ b/nodedb-types/src/config/tuning/mod.rs @@ -9,6 +9,7 @@ mod memory; mod network; mod scheduler; mod shutdown; +mod startup; pub use bitemporal::BitemporalTuning; pub use config::TuningConfig; @@ -23,3 +24,4 @@ pub use memory::MemoryTuning; pub use network::{BridgeTuning, ClusterTransportTuning, NetworkTuning, WalTuning}; pub use scheduler::SchedulerTuning; pub use shutdown::ShutdownTuning; +pub use startup::StartupTuning; diff --git a/nodedb-types/src/config/tuning/startup.rs b/nodedb-types/src/config/tuning/startup.rs new file mode 100644 index 000000000..e223d1f9f --- /dev/null +++ b/nodedb-types/src/config/tuning/startup.rs @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Startup tuning — boot-time bounds applied by the readiness gates. + +use std::time::Duration; + +use serde::{Deserialize, Serialize}; + +fn default_raft_ready_timeout_ms() -> u64 { + // 5 minutes: a node with a large metadata apply backlog (tens of + // thousands of entries from a burst of cross-shard writes) needs minutes + // of replay before the metadata group applies its first entry. A tighter + // bound turned a slow boot into a restart loop. + 300_000 +} + +fn default_data_group_recovery_timeout_ms() -> u64 { + // 10 minutes: the value production carried before the bound became a + // hard-coded constant. + 600_000 +} + +/// Boot-time bounds for the readiness gates in `bootstrap::cluster_ready`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StartupTuning { + /// How long the metadata raft group may go without applying an entry + /// before the readiness gate fails startup. Every applied-index advance + /// resets the clock, so a large replay finishes; only a stuck group fails. + /// Default: 300_000 (5 minutes). + #[serde(default = "default_raft_ready_timeout_ms")] + pub raft_ready_timeout_ms: u64, + + /// How long the locally hosted data raft groups have to replay their + /// retained logs before startup fails. Default: 600_000 (10 minutes). + #[serde(default = "default_data_group_recovery_timeout_ms")] + pub data_group_recovery_timeout_ms: u64, +} + +impl Default for StartupTuning { + fn default() -> Self { + Self { + raft_ready_timeout_ms: default_raft_ready_timeout_ms(), + data_group_recovery_timeout_ms: default_data_group_recovery_timeout_ms(), + } + } +} + +impl StartupTuning { + /// Metadata-group readiness stall bound as a `Duration`. + pub fn raft_ready_timeout(&self) -> Duration { + Duration::from_millis(self.raft_ready_timeout_ms) + } + + /// Data-group recovery bound as a `Duration`. + pub fn data_group_recovery_timeout(&self) -> Duration { + Duration::from_millis(self.data_group_recovery_timeout_ms) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults_are_five_and_ten_minutes() { + let t = StartupTuning::default(); + assert_eq!(t.raft_ready_timeout(), Duration::from_secs(300)); + assert_eq!(t.data_group_recovery_timeout(), Duration::from_secs(600)); + } + + #[test] + fn both_bounds_take_overrides() { + let parsed: StartupTuning = + toml::from_str("raft_ready_timeout_ms = 1000\ndata_group_recovery_timeout_ms = 2000\n") + .unwrap(); + assert_eq!(parsed.raft_ready_timeout(), Duration::from_millis(1000)); + assert_eq!( + parsed.data_group_recovery_timeout(), + Duration::from_millis(2000) + ); + } +} diff --git a/nodedb/src/bootstrap/cluster_ready.rs b/nodedb/src/bootstrap/cluster_ready.rs index 6462821e6..c7ccb81ff 100644 --- a/nodedb/src/bootstrap/cluster_ready.rs +++ b/nodedb/src/bootstrap/cluster_ready.rs @@ -28,11 +28,16 @@ pub struct ClusterReadyGates { /// /// In single-node mode `raft_ready_rx` is `None` and the raft-ready wait is /// skipped. Gate fires are always performed regardless of cluster mode. +/// +/// `raft_ready_timeout` and `data_group_recovery_timeout` are the boot bounds +/// from `[tuning.startup]`. pub async fn await_cluster_ready( shared: &Arc, raft_ready_rx: Option>, data_plane_replay_done: Vec>, gates: ClusterReadyGates, + raft_ready_timeout: Duration, + data_group_recovery_timeout: Duration, ) -> anyhow::Result<()> { let ClusterReadyGates { raft_gate, @@ -55,7 +60,7 @@ pub async fn await_cluster_ready( shared, &mut ready_rx, &raft_gate, - RAFT_READY_STALL_TIMEOUT, + raft_ready_timeout, RAFT_READY_POLL_INTERVAL, ) .await?; @@ -143,7 +148,12 @@ pub async fn await_cluster_ready( // elections, so without this wait the gateway can open while a data // group's engines are still empty and an acknowledged write reads back as // if it never happened. Fail closed, like the replay wait above. - if let Err(e) = crate::bootstrap::data_group_recovery::await_data_group_recovery(shared).await { + if let Err(e) = crate::bootstrap::data_group_recovery::await_data_group_recovery( + shared, + data_group_recovery_timeout, + ) + .await + { data_groups_gate.fail(format!("data raft group recovery failed: {e}")); return Err(e); } @@ -197,19 +207,6 @@ pub async fn await_cluster_ready( Ok(()) } -/// How long the metadata group may make NO replay progress before the boot -/// fails. Reset on every applied-index advance, so a large replay never trips -/// it — only a genuinely stuck group does. -/// How long the metadata group may go without *any* applied entry before the -/// readiness gate fails startup. -/// -/// Raised from 30 s after the 2026-09-20 incident: with a large apply backlog -/// (tens of thousands of entries from a burst of cross-shard writes) the group -/// needs minutes, and aborting the start turned a slow boot into a restart -/// loop. The gate still fails a group that never applies anything. -/// Follow-up: make this configurable. -const RAFT_READY_STALL_TIMEOUT: Duration = Duration::from_secs(300); - /// How often the stall check samples the applied index while waiting. const RAFT_READY_POLL_INTERVAL: Duration = Duration::from_secs(1); @@ -218,7 +215,8 @@ const RAFT_READY_POLL_INTERVAL: Duration = Duration::from_secs(1); /// Bounds the wait on lack of PROGRESS, not on total elapsed time. A node /// replaying a large log keeps advancing `applied_index` and must be allowed /// to finish; a group that is genuinely stuck advances nothing and fails -/// after [`RAFT_READY_STALL_TIMEOUT`]. +/// after `stall_timeout` (`[tuning.startup] raft_ready_timeout_ms`, five +/// minutes by default). async fn wait_for_raft_ready( shared: &Arc, ready_rx: &mut tokio::sync::watch::Receiver, diff --git a/nodedb/src/bootstrap/data_group_recovery.rs b/nodedb/src/bootstrap/data_group_recovery.rs index d0da54919..16f6aef60 100644 --- a/nodedb/src/bootstrap/data_group_recovery.rs +++ b/nodedb/src/bootstrap/data_group_recovery.rs @@ -31,11 +31,6 @@ use crate::control::state::SharedState; /// seconds, so a coarse poll costs nothing and avoids a busy loop. const POLL_INTERVAL: Duration = Duration::from_millis(50); -/// Upper bound on the whole wait. Generous relative to a randomized election -/// timeout plus replay of a retained log, but finite: a group that cannot elect -/// or cannot apply is a failure, not a reason to hang forever. -pub const DATA_GROUP_RECOVERY_TIMEOUT: Duration = Duration::from_secs(600); - /// True when `group_id` names a data group whose log carries user writes that /// must be replayed into the Data Plane before queries are served. /// @@ -170,14 +165,22 @@ fn pending_groups(statuses: Vec) -> Vec) -> anyhow::Result<()> { +pub async fn await_data_group_recovery( + shared: &Arc, + timeout: Duration, +) -> anyhow::Result<()> { let Some(status_fn) = shared.raft_status_fn.get() else { return Ok(()); }; let status_fn = Arc::clone(status_fn); - let deadline = Instant::now() + DATA_GROUP_RECOVERY_TIMEOUT; + let deadline = Instant::now() + timeout; loop { let pending = pending_groups(status_fn()); @@ -193,7 +196,7 @@ pub async fn await_data_group_recovery(shared: &Arc) -> anyhow::Res .collect::>() .join("; "); return Err(anyhow::anyhow!( - "data raft group recovery timeout after {DATA_GROUP_RECOVERY_TIMEOUT:?}: {detail}" + "data raft group recovery timeout after {timeout:?}: {detail}" )); } diff --git a/nodedb/src/config/server/config.rs b/nodedb/src/config/server/config.rs index aac4351e9..fd7e9f477 100644 --- a/nodedb/src/config/server/config.rs +++ b/nodedb/src/config/server/config.rs @@ -120,6 +120,30 @@ pub struct ServerConfig { pub scheduler: SchedulerConfig, } +/// Rejects startup bounds written where they used to live, naming the moved-to +/// path. `deny_unknown_fields` catches the key anyway, but its message lists +/// every valid field instead of the replacement. +fn reject_moved_startup_bounds(content: &str) -> crate::Result<()> { + let Ok(doc) = toml::from_str::(content) else { + // A malformed document fails in the real parse below; nothing to + // inspect here. + return Ok(()); + }; + let Some(server) = doc.get("server").and_then(|v| v.as_table()) else { + return Ok(()); + }; + for key in ["raft_ready_timeout_ms", "data_group_recovery_timeout_ms"] { + if server.contains_key(key) { + return Err(crate::Error::Config { + detail: format!( + "`[server] {key}` moved to `[tuning.startup] {key}`; set the bound there" + ), + }); + } + } + Ok(()) +} + impl ServerConfig { /// Load configuration from a TOML file, falling back to defaults. pub fn from_file(path: &std::path::Path) -> crate::Result { @@ -131,6 +155,7 @@ impl ServerConfig { // that field — this is textual substitution before parsing, not a // second, competing expansion. let content = super::env_expand::expand_env(path, &content)?; + reject_moved_startup_bounds(&content)?; let parsed: Self = toml::from_str(&content).map_err(|e| crate::Error::Config { detail: format!("invalid TOML config: {e}"), })?; @@ -387,6 +412,49 @@ mod tests { assert!(msg.contains("positive integer"), "{msg}"); } + /// The pre-tuning location of a startup bound is rejected with the new + /// path named, not serde's full valid-field list. + #[test] + fn from_file_rejects_a_startup_bound_at_its_old_server_path() { + let path = write_temp_config( + "nodedb-moved-startup-bound.toml", + "[server]\nraft_ready_timeout_ms = 300000\n", + ); + let err = ServerConfig::from_file(&path).unwrap_err(); + std::fs::remove_file(&path).ok(); + let msg = err.to_string(); + assert!(msg.contains("tuning.startup"), "{msg}"); + assert!(msg.contains("raft_ready_timeout_ms"), "{msg}"); + } + + /// The moved key is rejected under either legacy name. + #[test] + fn from_file_rejects_the_legacy_data_group_recovery_bound() { + let path = write_temp_config( + "nodedb-moved-recovery-bound.toml", + "[server]\ndata_group_recovery_timeout_ms = 600000\n", + ); + let err = ServerConfig::from_file(&path).unwrap_err(); + std::fs::remove_file(&path).ok(); + let msg = err.to_string(); + assert!(msg.contains("tuning.startup"), "{msg}"); + assert!(msg.contains("data_group_recovery_timeout_ms"), "{msg}"); + } + + /// Both boot bounds live under `[tuning.startup]`; the other bound keeps + /// its default when only one is set. + #[test] + fn from_file_reads_the_startup_bounds_from_tuning() { + let path = write_temp_config( + "nodedb-startup-tuning.toml", + "[tuning.startup]\ndata_group_recovery_timeout_ms = 1234000\n", + ); + let cfg = ServerConfig::from_file(&path).expect("load config"); + std::fs::remove_file(&path).ok(); + assert_eq!(cfg.tuning.startup.data_group_recovery_timeout_ms, 1_234_000); + assert_eq!(cfg.tuning.startup.raft_ready_timeout_ms, 300_000); + } + #[test] fn from_file_rejects_scope_expiry_below_the_floor() { let path = write_temp_config( diff --git a/nodedb/src/main.rs b/nodedb/src/main.rs index eefc2c946..2309807b1 100644 --- a/nodedb/src/main.rs +++ b/nodedb/src/main.rs @@ -260,6 +260,7 @@ async fn server_main() -> anyhow::Result<()> { }; // Wait for raft readiness, run catalog sanity check, warm peer cache, fire gates. + // The two boot bounds come from [tuning.startup]. nodedb::bootstrap::cluster_ready::await_cluster_ready( &shared, raft_ready_rx, @@ -274,6 +275,8 @@ async fn server_main() -> anyhow::Result<()> { health_loop_gate, gateway_enable_gate, }, + config.tuning.startup.raft_ready_timeout(), + config.tuning.startup.data_group_recovery_timeout(), ) .await?;