From b463409106075255287df07bdb163e1a4f714625 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 11:04:27 -0400 Subject: [PATCH 01/38] fix(forwarder): pin static receivers so server allow-list snapshots cannot revoke them --- services/forwarder/src/p2p/allowlist.rs | 149 +++++++++++++++++++++--- services/forwarder/src/p2p/mod.rs | 14 +-- 2 files changed, 140 insertions(+), 23 deletions(-) diff --git a/services/forwarder/src/p2p/allowlist.rs b/services/forwarder/src/p2p/allowlist.rs index 37b21651..3431922a 100644 --- a/services/forwarder/src/p2p/allowlist.rs +++ b/services/forwarder/src/p2p/allowlist.rs @@ -67,6 +67,9 @@ pub const DEFAULT_ALLOWLIST_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); struct State { /// Endpoint ids currently permitted to connect. allowed: HashSet, + /// Operator-pinned (static config) receivers. Always allowed; never + /// revoked or persisted by server snapshots. + pinned: HashSet, /// Open admitted connections, keyed by remote endpoint id then a per-connection /// registration id so multiple connections from one peer are tracked /// independently. @@ -130,23 +133,26 @@ impl AllowList { self.lock().allowed.contains(endpoint_id) } - /// Unions `additional` endpoint ids into the allowed set in memory only. - /// - /// Unlike [`AllowList::apply_update`], this neither persists to the cache - /// nor revokes anything: it is additive. It exists so statically configured - /// receivers can be allowed *on top of* the cached/last-known (or - /// server-fetched) set at startup, rather than replacing it. - pub fn add_allowed(&self, additional: impl IntoIterator) { + /// Pins operator-configured receivers: always allowed, never revoked by + /// [`AllowList::apply_update`], never persisted to the server-snapshot + /// cache. Intended to be called once at startup; a later call with a + /// smaller set does not un-pin previously pinned nodes until the next + /// `apply_update`. + pub fn set_pinned(&self, pinned: impl IntoIterator) { let mut state = self.lock(); - state.allowed.extend(additional); + state.pinned = pinned.into_iter().collect(); + let pinned = state.pinned.clone(); + state.allowed.extend(pinned); } - /// Atomically replaces the allowed set with `allowed`, persisting it and + /// Atomically replaces the allowed set with `allowed` ∪ the pinned set, + /// persisting the server snapshot (`allowed` only, never pinned) and /// force-closing every open connection whose endpoint id was removed. /// - /// Returns the revoked endpoint ids. Persistence happens before the in-memory - /// swap: if the cache cannot be written the in-memory state is left at its - /// last-known value and the error is returned. + /// Returns the revoked endpoint ids; pinned endpoints are never revoked. + /// Persistence happens before the in-memory swap: if the cache cannot be + /// written the in-memory state is left at its last-known value and the + /// error is returned. /// /// # Errors /// @@ -166,9 +172,11 @@ impl AllowList { persist(path, &new_allowed)?; } + let effective: HashSet = + new_allowed.union(&state.pinned).copied().collect(); let revoked: Vec = - state.allowed.difference(&new_allowed).copied().collect(); - state.allowed = new_allowed; + state.allowed.difference(&effective).copied().collect(); + state.allowed = effective; let mut connections_to_close = Vec::new(); for endpoint_id in &revoked { if let Some(connections) = state.connections.remove(endpoint_id) { @@ -927,6 +935,119 @@ mod tests { Ok(()) } + #[tokio::test] + async fn apply_update_preserves_pinned_receivers() -> TestResult { + let pinned = EndpointBuilder::test([70; 32]).bind().await?; + let server_only = EndpointBuilder::test([71; 32]).bind().await?; + + let allow = AllowList::new(vec![]); + allow.set_pinned([pinned.endpoint_id()]); + + // Server snapshot that does NOT contain the pinned receiver. + let revoked = allow.apply_update([server_only.endpoint_id()])?; + + assert!( + allow.contains(&pinned.endpoint_id()), + "pinned receiver must survive server snapshot" + ); + assert!(allow.contains(&server_only.endpoint_id())); + assert!( + !revoked.contains(&pinned.endpoint_id()), + "pinned receiver must never be revoked" + ); + + pinned.close().await; + server_only.close().await; + Ok(()) + } + + #[tokio::test] + async fn apply_update_does_not_persist_pinned_to_cache() -> TestResult { + let dir = tempfile::tempdir()?; + let cache_path = dir.path().join("allowlist"); + + let pinned = EndpointBuilder::test([72; 32]).bind().await?; + let server_only = EndpointBuilder::test([73; 32]).bind().await?; + + let allow = AllowList::load(&cache_path)?; + allow.set_pinned([pinned.endpoint_id()]); + allow.apply_update([server_only.endpoint_id()])?; + + let cached = parse_endpoint_ids(&std::fs::read_to_string(&cache_path)?)?; + assert_eq!( + cached, + HashSet::from([server_only.endpoint_id()]), + "cache must hold the server snapshot only, never pinned receivers" + ); + + pinned.close().await; + server_only.close().await; + Ok(()) + } + + #[tokio::test] + async fn apply_update_does_not_force_close_pinned_connection() -> TestResult { + let receiver = EndpointBuilder::test([74; 32]).bind().await?; + let forwarder = EndpointBuilder::test([75; 32]).bind().await?; + let forwarder_addr = forwarder.endpoint_addr().await; + + let allow = AllowList::new(vec![]); + allow.set_pinned([receiver.endpoint_id()]); + let server = tokio::spawn(admission_server(forwarder.clone(), allow.clone())); + + // Establish and confirm an admitted, registered subscription. + let (connection, reply) = + tokio::time::timeout(Duration::from_secs(5), subscribe(&receiver, forwarder_addr)) + .await??; + assert_eq!(&reply, SUBSCRIBE_OK); + + // An empty server snapshot must not revoke or force-close the pinned peer. + let revoked = allow.apply_update([])?; + assert!(revoked.is_empty(), "pinned peer must not be revoked"); + + // The connection stays functional: a fresh subscription probe on the + // same connection still succeeds. + let (mut send, mut recv) = connection.open_bi().await?; + send.write_all(SUBSCRIBE).await?; + send.finish()?; + let mut buf = [0u8; SUBSCRIBE_OK.len()]; + tokio::time::timeout(Duration::from_secs(5), recv.read_exact(&mut buf)).await??; + assert_eq!(&buf, SUBSCRIBE_OK); + + server.abort(); + receiver.close().await; + forwarder.close().await; + Ok(()) + } + + #[tokio::test] + async fn pinned_receivers_removed_from_config_are_not_resurrected_on_reboot() -> TestResult { + let dir = tempfile::tempdir()?; + let cache_path = dir.path().join("allowlist"); + + let pinned = EndpointBuilder::test([76; 32]).bind().await?; + let server_only = EndpointBuilder::test([77; 32]).bind().await?; + + // First boot: the receiver is pinned via static config and a server + // snapshot is persisted to the cache. + let first_boot = AllowList::load(&cache_path)?; + first_boot.set_pinned([pinned.endpoint_id()]); + first_boot.apply_update([server_only.endpoint_id()])?; + + // Reboot with the receiver removed from static config (no set_pinned): + // the cache must not resurrect it. + let second_boot = AllowList::load(&cache_path)?; + assert!( + !second_boot.contains(&pinned.endpoint_id()), + "removing a receiver from static config must take effect on reboot" + ); + assert!(second_boot.contains(&server_only.endpoint_id())); + + pinned.close().await; + server_only.close().await; + Ok(()) + } + #[derive(Clone)] struct TestCatalogServerState { bearer_token: &'static str, diff --git a/services/forwarder/src/p2p/mod.rs b/services/forwarder/src/p2p/mod.rs index 62617824..05b239bd 100644 --- a/services/forwarder/src/p2p/mod.rs +++ b/services/forwarder/src/p2p/mod.rs @@ -326,16 +326,12 @@ fn build_allow_list(config: &P2pConfig) -> Result { let static_receivers = parse_endpoint_ids(&config.static_allowed_receivers)?; let allow_list = match &config.allowlist_cache_path { - Some(path) => { - // Static receivers are additive: union them on top of the cached - // last-known set rather than replacing it (which would revoke every - // cached receiver). `add_allowed` does not persist or revoke. - let allow_list = AllowList::load(path)?; - allow_list.add_allowed(static_receivers); - allow_list - } - None => AllowList::new(static_receivers), + Some(path) => AllowList::load(path)?, + None => AllowList::new(vec![]), }; + // Static receivers are pinned: always allowed on top of the cached + // last-known set, never revoked or persisted by later server snapshots. + allow_list.set_pinned(static_receivers); Ok(allow_list) } From 96b43433e6f525c3d901a0ea58b530d938cc9bd2 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 11:11:46 -0400 Subject: [PATCH 02/38] docs(forwarder): clarify allow-list pinning contracts; add static-receiver sync regression guard --- services/forwarder/src/p2p/allowlist.rs | 22 ++++++++++++---------- services/forwarder/src/p2p/mod.rs | 12 ++++++++++++ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/services/forwarder/src/p2p/allowlist.rs b/services/forwarder/src/p2p/allowlist.rs index 3431922a..51a73ffd 100644 --- a/services/forwarder/src/p2p/allowlist.rs +++ b/services/forwarder/src/p2p/allowlist.rs @@ -7,11 +7,13 @@ //! 1. **Authoritative in-memory set.** [`AllowList::try_register_connection`] //! admits and tracks a connection under one lock, before any control or data //! stream work happens. -//! 2. **Last-known persistence.** The allowed set is cached on disk so a -//! refresh failure (e.g. an offline server) falls back to the previously -//! persisted list rather than failing open or failing closed-empty. Updates -//! persist *before* the in-memory swap, so a write failure leaves the -//! last-known set in force ([`AllowList::apply_update`]). +//! 2. **Last-known persistence.** The server-snapshot portion of the allowed +//! set is cached on disk (pinned/static receivers are re-derived from +//! config at startup, never persisted) so a refresh failure (e.g. an +//! offline server) falls back to the previously persisted list rather than +//! failing open or failing closed-empty. Updates persist *before* the +//! in-memory swap, so a write failure leaves the last-known set in force +//! ([`AllowList::apply_update`]). //! 3. **Revocation / force-close.** Admitted connections are tracked by remote //! endpoint id ([`AllowList::try_register_connection`]); when an update removes a //! peer, its open connections are force-closed immediately. @@ -135,14 +137,14 @@ impl AllowList { /// Pins operator-configured receivers: always allowed, never revoked by /// [`AllowList::apply_update`], never persisted to the server-snapshot - /// cache. Intended to be called once at startup; a later call with a - /// smaller set does not un-pin previously pinned nodes until the next - /// `apply_update`. + /// cache. Intended to be called once at startup; the pinned set itself is + /// replaced immediately, but endpoints removed from it remain allowed + /// until the next `apply_update`. pub fn set_pinned(&self, pinned: impl IntoIterator) { let mut state = self.lock(); + let state = &mut *state; state.pinned = pinned.into_iter().collect(); - let pinned = state.pinned.clone(); - state.allowed.extend(pinned); + state.allowed.extend(state.pinned.iter().copied()); } /// Atomically replaces the allowed set with `allowed` ∪ the pinned set, diff --git a/services/forwarder/src/p2p/mod.rs b/services/forwarder/src/p2p/mod.rs index 05b239bd..f98e65f2 100644 --- a/services/forwarder/src/p2p/mod.rs +++ b/services/forwarder/src/p2p/mod.rs @@ -739,6 +739,18 @@ mod tests { "statically configured receiver must be allowed alongside the cached set" ); + // Regression guard: the first server sync (an empty snapshot here) must + // never revoke the statically configured receiver. + let revoked = allow_list.apply_update([])?; + assert!( + allow_list.contains(&static_receiver.endpoint_id()), + "statically configured receiver must survive a server allow-list sync" + ); + assert!( + !revoked.contains(&static_receiver.endpoint_id()), + "statically configured receiver must never be revoked by a server sync" + ); + cached.close().await; static_receiver.close().await; Ok(()) From 4cbb7a24464a635e348d3d9daed9ef25bbb2191f Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 11:21:16 -0400 Subject: [PATCH 03/38] fix(forwarder): enforce negotiated capabilities and add control.allow_reader_control gate for P2P reader control --- services/forwarder/README.md | 2 + services/forwarder/src/config.rs | 18 ++ services/forwarder/src/main.rs | 1 + services/forwarder/src/p2p/control.rs | 247 ++++++++++++++++++- services/forwarder/src/p2p/endpoint.rs | 1 + services/forwarder/src/p2p/reader_control.rs | 56 ++++- services/forwarder/src/status_http.rs | 7 +- 7 files changed, 321 insertions(+), 11 deletions(-) diff --git a/services/forwarder/README.md b/services/forwarder/README.md index e128bede..df445cfc 100644 --- a/services/forwarder/README.md +++ b/services/forwarder/README.md @@ -84,6 +84,7 @@ or `server_url`. |-------|------|----------|---------|-------------| | `allow_power_actions` | `bool` | No | `false` | Enables local control API actions that restart or shut down the host. | | `allow_remote_config` | `bool` | No | `true` | Allows receivers to read/write forwarder config over the P2P control plane. | +| `allow_reader_control` | `bool` | No | `true` | Allows receivers to execute reader-control verbs (status, download, clear, epoch) over the P2P control plane. | ### `[update]` @@ -147,6 +148,7 @@ bind = "127.0.0.1:8080" [control] allow_power_actions = false allow_remote_config = true +allow_reader_control = true [update] mode = "check-and-download" diff --git a/services/forwarder/src/config.rs b/services/forwarder/src/config.rs index 290746c1..964d207c 100644 --- a/services/forwarder/src/config.rs +++ b/services/forwarder/src/config.rs @@ -76,6 +76,9 @@ pub struct ControlConfig { /// Defaults to `true` per the product decision to allow remote config by /// default. pub allow_remote_config: bool, + /// Gates reader status/control verbs over P2P. Defaults to `true` to match + /// the remote-config product default. + pub allow_reader_control: bool, } #[derive(Debug, Clone)] @@ -167,6 +170,7 @@ pub struct RawStatusHttpConfig { pub struct RawControlConfig { pub allow_power_actions: Option, pub allow_remote_config: Option, + pub allow_reader_control: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -312,10 +316,12 @@ pub fn load_config_from_str( Some(c) => ControlConfig { allow_power_actions: c.allow_power_actions.unwrap_or(false), allow_remote_config: c.allow_remote_config.unwrap_or(true), + allow_reader_control: c.allow_reader_control.unwrap_or(true), }, None => ControlConfig { allow_power_actions: false, allow_remote_config: true, + allow_reader_control: true, }, }; @@ -950,6 +956,7 @@ max_concurrent_bidi_streams = 2 let (toml, _dir) = minimal_toml(""); let cfg = load_config_from_str(&toml, Path::new("/tmp/test.toml")).unwrap(); assert!(cfg.control.allow_remote_config); + assert!(cfg.control.allow_reader_control); // Existing default for power actions is unchanged. assert!(!cfg.control.allow_power_actions); } @@ -959,6 +966,7 @@ max_concurrent_bidi_streams = 2 let (toml, _dir) = minimal_toml("[control]\nallow_power_actions = true"); let cfg = load_config_from_str(&toml, Path::new("/tmp/test.toml")).unwrap(); assert!(cfg.control.allow_remote_config); + assert!(cfg.control.allow_reader_control); assert!(cfg.control.allow_power_actions); } @@ -967,10 +975,20 @@ max_concurrent_bidi_streams = 2 let (toml, _dir) = minimal_toml("[control]\nallow_remote_config = false"); let cfg = load_config_from_str(&toml, Path::new("/tmp/test.toml")).unwrap(); assert!(!cfg.control.allow_remote_config); + assert!(cfg.control.allow_reader_control); // Power actions still default to false when unspecified. assert!(!cfg.control.allow_power_actions); } + #[test] + fn control_section_honors_explicit_allow_reader_control_false() { + let (toml, _dir) = minimal_toml("[control]\nallow_reader_control = false"); + let cfg = load_config_from_str(&toml, Path::new("/tmp/test.toml")).unwrap(); + assert!(!cfg.control.allow_reader_control); + assert!(cfg.control.allow_remote_config); + assert!(!cfg.control.allow_power_actions); + } + #[test] fn eink_section_absent_parses_ok() { let (toml, _dir) = minimal_toml(""); diff --git a/services/forwarder/src/main.rs b/services/forwarder/src/main.rs index d7c680e8..30c44565 100644 --- a/services/forwarder/src/main.rs +++ b/services/forwarder/src/main.rs @@ -306,6 +306,7 @@ async fn main() { restart_signal.clone(), )); let reader_control_handler = Arc::new(forwarder::p2p::ForwarderReaderControlHandler::new( + cfg.control.allow_reader_control, status_server.reader_control_service(), Arc::clone(&journal), )); diff --git a/services/forwarder/src/p2p/control.rs b/services/forwarder/src/p2p/control.rs index 569a194e..935874d2 100644 --- a/services/forwarder/src/p2p/control.rs +++ b/services/forwarder/src/p2p/control.rs @@ -30,7 +30,7 @@ use rt_p2p_protocol::{ MAX_FRAME_BYTES, Ping, Pong, ProtocolError, ProtocolErrorCode, ReaderControlRequest, ReaderControlResponse, ReaderInfo, ReaderStatus, RestartRequest, RestartResponse, StreamCatalog, SyncClock, UpsStatus, WireProtocolError, control_c2f, control_f2c, encode_frame, - negotiate, + has_capability, negotiate, }; use tokio::sync::mpsc; use tokio::time::MissedTickBehavior; @@ -426,6 +426,7 @@ pub(crate) async fn negotiate_control_stream( pub(crate) async fn run_control_stream_loop( send: SendStream, recv: RecvStream, + capabilities: Vec, heartbeat: HeartbeatConfig, outbound_events: Option, reader_control: Arc, @@ -434,6 +435,7 @@ pub(crate) async fn run_control_stream_loop( run_control_loop( send, recv, + capabilities, heartbeat, reader_control, outbound_events, @@ -480,7 +482,7 @@ pub(crate) async fn serve_control_stream_with_typed_control( outbound_events: Option, remote_config: Arc, ) -> Result<(), BoxError> { - let (send, recv, _capabilities) = negotiate_control_stream( + let (send, recv, capabilities) = negotiate_control_stream( send, recv, catalog, @@ -494,6 +496,7 @@ pub(crate) async fn serve_control_stream_with_typed_control( run_control_loop( send, recv, + capabilities, heartbeat, reader_control, outbound_events, @@ -590,6 +593,7 @@ async fn negotiate_and_serve_catalog_stream( async fn run_control_loop( mut send: SendStream, mut recv: RecvStream, + capabilities: Vec, config: HeartbeatConfig, reader_control: Arc, mut outbound_events: Option, @@ -620,6 +624,8 @@ async fn run_control_loop( // response channel served by its own select arm below. let (config_response_tx, mut config_response_rx) = mpsc::channel::(16); let mut control_tasks = tokio::task::JoinSet::new(); + let reader_control_negotiated = has_capability(&capabilities, CAP_READER_CONTROL); + let remote_config_negotiated = has_capability(&capabilities, CAP_REMOTE_CONFIG); let result = loop { tokio::select! { @@ -655,6 +661,15 @@ async fn run_control_loop( } } Some(control_c2f::Msg::ReaderControlRequest(request)) => { + if !reader_control_negotiated { + let response = reader_control_error_response( + &request, + "reader control was not negotiated on this connection", + ); + let _ = control_response_tx.send(response).await; + continue; + } + let stream_id = request.stream_id.clone(); let request_id = request.request_id.clone(); let reader_control = Arc::clone(&reader_control); @@ -667,6 +682,19 @@ async fn run_control_loop( }); } Some(control_c2f::Msg::ConfigGetRequest(request)) => { + if !remote_config_negotiated { + let response = ConfigGetResponse { + request_id: request.request_id, + config_json: String::new(), + restart_needed: false, + }; + let frame = ControlF2C { + msg: Some(control_f2c::Msg::ConfigGetResponse(response)), + }; + let _ = config_response_tx.send(frame).await; + continue; + } + let remote_config = Arc::clone(&remote_config); let config_response_tx = config_response_tx.clone(); control_tasks.spawn(async move { @@ -678,6 +706,21 @@ async fn run_control_loop( }); } Some(control_c2f::Msg::ConfigSetRequest(request)) => { + if !remote_config_negotiated { + let response = ConfigSetResponse { + request_id: request.request_id, + ok: false, + restart_needed: false, + error: "remote config was not negotiated on this connection" + .to_owned(), + }; + let frame = ControlF2C { + msg: Some(control_f2c::Msg::ConfigSetResponse(response)), + }; + let _ = config_response_tx.send(frame).await; + continue; + } + let remote_config = Arc::clone(&remote_config); let config_response_tx = config_response_tx.clone(); control_tasks.spawn(async move { @@ -689,6 +732,20 @@ async fn run_control_loop( }); } Some(control_c2f::Msg::RestartRequest(request)) => { + if !remote_config_negotiated { + let response = RestartResponse { + request_id: request.request_id, + accepted: false, + error: "remote config was not negotiated on this connection" + .to_owned(), + }; + let frame = ControlF2C { + msg: Some(control_f2c::Msg::RestartResponse(response)), + }; + let _ = config_response_tx.send(frame).await; + continue; + } + let remote_config = Arc::clone(&remote_config); let config_response_tx = config_response_tx.clone(); control_tasks.spawn(async move { @@ -747,6 +804,19 @@ async fn run_control_loop( result } +fn reader_control_error_response( + request: &ReaderControlRequest, + message: &str, +) -> ReaderControlResponse { + ReaderControlResponse { + stream_id: request.stream_id.clone(), + request_id: request.request_id.clone(), + success: false, + message: message.to_owned(), + reader_info_json: None, + } +} + async fn recv_control_event(events: &mut Option) -> Option { match events { Some(events) => events.recv().await, @@ -919,6 +989,30 @@ mod tests { } } + #[derive(Debug)] + struct SpyControlHandler { + calls: AtomicUsize, + } + + impl ReaderControlHandler for SpyControlHandler { + fn supports_reader_control(&self) -> bool { + true + } + + fn handle(&self, request: ReaderControlRequest) -> ReaderControlFuture<'_> { + self.calls.fetch_add(1, Ordering::SeqCst); + Box::pin(async move { + rt_p2p_protocol::ReaderControlResponse { + stream_id: request.stream_id, + request_id: request.request_id, + success: true, + message: "spy invoked".to_owned(), + reader_info_json: None, + } + }) + } + } + #[derive(Debug)] struct SlowControlHandler { started_tx: Mutex>>, @@ -985,6 +1079,7 @@ mod tests { config_json: String, restart_needed: bool, last_set: Mutex>, + set_calls: AtomicUsize, restart_calls: AtomicUsize, } @@ -995,6 +1090,7 @@ mod tests { config_json: r#"{"schema_version":1}"#.to_owned(), restart_needed: false, last_set: Mutex::new(None), + set_calls: AtomicUsize::new(0), restart_calls: AtomicUsize::new(0), } } @@ -1032,6 +1128,7 @@ mod tests { error: REMOTE_CONFIG_DISABLED.to_owned(), }; } + self.set_calls.fetch_add(1, Ordering::SeqCst); *self.last_set.lock().unwrap() = Some(request.config_json.clone()); ConfigSetResponse { request_id: request.request_id, @@ -1069,6 +1166,14 @@ mod tests { hello } + /// Builds a client `Hello` that advertises reader-control support so the + /// negotiated capability set can include [`CAP_READER_CONTROL`]. + fn hello_with_reader_control() -> Hello { + let mut hello = forwarder_hello(); + hello.capabilities.push(CAP_READER_CONTROL.to_owned()); + hello + } + async fn spawn_forwarder_with_remote_config( seed: [u8; 32], remote_config: Arc, @@ -1158,6 +1263,75 @@ mod tests { Ok(()) } + #[tokio::test] + async fn reader_control_request_rejected_when_capability_not_negotiated() -> TestResult { + let handler = Arc::new(SpyControlHandler { + calls: AtomicUsize::new(0), + }); + let (forwarder, forwarder_addr, handle) = spawn_forwarder_with_control( + [54; 32], + StaticCatalog::new(sample_catalog()), + LONG_HANDSHAKE, + quiet_heartbeat(), + Arc::clone(&handler) as _, + None, + Arc::new(NoopRemoteConfigHandler), + ) + .await?; + + let receiver = EndpointBuilder::test([55; 32]).bind().await?; + let (connection, mut send, mut recv) = tokio::time::timeout( + LONG_HANDSHAKE, + open_control(&receiver, forwarder_addr, forwarder_hello()), + ) + .await??; + + let _hello_ok = read_frame::(&mut recv).await?; + let _catalog = read_frame::(&mut recv).await?; + + let stream_id = vec![6u8; 16]; + write_frame( + &mut send, + &ControlC2F { + msg: Some(control_c2f::Msg::ReaderControlRequest( + ReaderControlRequest { + stream_id: stream_id.clone(), + command: "clear_records".to_owned(), + request_id: "req-no-cap".to_owned(), + mode: None, + timeout: None, + enabled: None, + epoch_name: None, + }, + )), + }, + ) + .await?; + + let frame = + tokio::time::timeout(LONG_HANDSHAKE, read_frame::(&mut recv)).await??; + match frame.msg { + Some(control_f2c::Msg::ReaderControlResponse(response)) => { + assert_eq!(response.stream_id, stream_id); + assert_eq!(response.request_id, "req-no-cap"); + assert!(!response.success); + assert!( + response.message.contains("not negotiated"), + "message should mention negotiation, got {:?}", + response.message + ); + } + other => return Err(format!("expected ReaderControlResponse, got {other:?}").into()), + } + assert_eq!(handler.calls.load(Ordering::SeqCst), 0); + + connection.close(0u32.into(), b"done"); + handle.abort(); + receiver.close().await; + forwarder.close().await; + Ok(()) + } + #[tokio::test] async fn remote_config_capability_absent_when_disabled() -> TestResult { let (forwarder, forwarder_addr, handle) = spawn_forwarder_with_remote_config( @@ -1291,6 +1465,57 @@ mod tests { Ok(()) } + #[tokio::test] + async fn config_set_rejected_when_capability_not_negotiated() -> TestResult { + let handler = Arc::new(FakeRemoteConfigHandler::new(true)); + let (forwarder, forwarder_addr, handle) = + spawn_forwarder_with_remote_config([56; 32], Arc::clone(&handler) as _).await?; + + let receiver = EndpointBuilder::test([59; 32]).bind().await?; + let (connection, mut send, mut recv) = tokio::time::timeout( + LONG_HANDSHAKE, + open_control(&receiver, forwarder_addr, forwarder_hello()), + ) + .await??; + + let _hello_ok = read_frame::(&mut recv).await?; + let _catalog = read_frame::(&mut recv).await?; + + write_frame( + &mut send, + &ControlC2F { + msg: Some(control_c2f::Msg::ConfigSetRequest(ConfigSetRequest { + request_id: "set-no-cap".to_owned(), + config_json: r#"{"schema_version":1}"#.to_owned(), + })), + }, + ) + .await?; + + let frame = + tokio::time::timeout(LONG_HANDSHAKE, read_frame::(&mut recv)).await??; + match frame.msg { + Some(control_f2c::Msg::ConfigSetResponse(response)) => { + assert_eq!(response.request_id, "set-no-cap"); + assert!(!response.ok); + assert!( + response.error.contains("not negotiated"), + "error should mention negotiation, got {:?}", + response.error + ); + } + other => return Err(format!("expected ConfigSetResponse, got {other:?}").into()), + } + assert_eq!(handler.set_calls.load(Ordering::SeqCst), 0); + assert!(handler.last_set.lock().unwrap().is_none()); + + connection.close(0u32.into(), b"done"); + handle.abort(); + receiver.close().await; + forwarder.close().await; + Ok(()) + } + #[tokio::test] async fn config_set_rejected_when_disabled() -> TestResult { let (forwarder, forwarder_addr, handle) = spawn_forwarder_with_remote_config( @@ -1326,7 +1551,11 @@ mod tests { Some(control_f2c::Msg::ConfigSetResponse(response)) => { assert_eq!(response.request_id, "set-2"); assert!(!response.ok, "set must be rejected when disabled"); - assert_eq!(response.error, REMOTE_CONFIG_DISABLED); + assert!( + response.error.contains("not negotiated"), + "error should mention negotiation, got {:?}", + response.error + ); } other => return Err(format!("expected ConfigSetResponse, got {other:?}").into()), } @@ -1417,7 +1646,11 @@ mod tests { Some(control_f2c::Msg::RestartResponse(response)) => { assert_eq!(response.request_id, "restart-2"); assert!(!response.accepted, "restart must be rejected when disabled"); - assert_eq!(response.error, REMOTE_CONFIG_DISABLED); + assert!( + response.error.contains("not negotiated"), + "error should mention negotiation, got {:?}", + response.error + ); } other => return Err(format!("expected RestartResponse, got {other:?}").into()), } @@ -1549,7 +1782,7 @@ mod tests { let receiver = EndpointBuilder::test([47; 32]).bind().await?; let (connection, mut send, mut recv) = tokio::time::timeout( LONG_HANDSHAKE, - open_control(&receiver, forwarder_addr, forwarder_hello()), + open_control(&receiver, forwarder_addr, hello_with_reader_control()), ) .await??; @@ -1616,7 +1849,7 @@ mod tests { let receiver = EndpointBuilder::test([49; 32]).bind().await?; let (connection, mut send, mut recv) = tokio::time::timeout( LONG_HANDSHAKE, - open_control(&receiver, forwarder_addr, forwarder_hello()), + open_control(&receiver, forwarder_addr, hello_with_reader_control()), ) .await??; @@ -1715,7 +1948,7 @@ mod tests { let receiver = EndpointBuilder::test([51; 32]).bind().await?; let (connection, mut send, mut recv) = tokio::time::timeout( LONG_HANDSHAKE, - open_control(&receiver, forwarder_addr, forwarder_hello()), + open_control(&receiver, forwarder_addr, hello_with_reader_control()), ) .await??; diff --git a/services/forwarder/src/p2p/endpoint.rs b/services/forwarder/src/p2p/endpoint.rs index eab4232a..002a1657 100644 --- a/services/forwarder/src/p2p/endpoint.rs +++ b/services/forwarder/src/p2p/endpoint.rs @@ -307,6 +307,7 @@ async fn handle_connection( let control_result = run_control_stream_loop( control_send, control_recv, + capabilities, config.heartbeat, outbound_events, config.reader_control, diff --git a/services/forwarder/src/p2p/reader_control.rs b/services/forwarder/src/p2p/reader_control.rs index 22c4de15..5bed8a16 100644 --- a/services/forwarder/src/p2p/reader_control.rs +++ b/services/forwarder/src/p2p/reader_control.rs @@ -9,6 +9,7 @@ use tokio::sync::Mutex; #[derive(Clone)] pub struct ForwarderReaderControlHandler { + allow_reader_control: bool, service: ReaderControlService, journal: Arc>, } @@ -23,16 +24,21 @@ impl std::fmt::Debug for ForwarderReaderControlHandler { impl ForwarderReaderControlHandler { #[must_use] pub fn new( + allow_reader_control: bool, service: ReaderControlService, journal: Arc>, ) -> Self { - Self { service, journal } + Self { + allow_reader_control, + service, + journal, + } } } impl ReaderControlHandler for ForwarderReaderControlHandler { fn supports_reader_control(&self) -> bool { - true + self.allow_reader_control } fn handle(&self, request: ReaderControlRequest) -> ReaderControlFuture<'_> { @@ -249,6 +255,52 @@ mod tests { } } + fn handler_with_reader_control_flag( + allow_reader_control: bool, + ) -> (ForwarderReaderControlHandler, tempfile::TempDir) { + let subsystem = Arc::new(Mutex::new(crate::status_http::SubsystemStatus::ready())); + let control_clients = Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())); + let download_trackers = Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())); + let reconnect_notifies = Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())); + let (ui_tx, _) = tokio::sync::broadcast::channel(16); + let (status_event_tx, _) = tokio::sync::broadcast::channel(16); + let logger = Arc::new(rt_ui_log::UiLogger::with_buffer( + ui_tx.clone(), + |entry| crate::ui_events::ForwarderUiEvent::LogEntry { entry }, + 16, + )); + let service = ReaderControlService::new( + subsystem, + control_clients, + download_trackers, + reconnect_notifies, + ui_tx, + status_event_tx, + logger, + ); + let tempdir = tempfile::tempdir().expect("tempdir"); + let journal = + crate::storage::journal::Journal::open(&tempdir.path().join("journal.sqlite3")) + .expect("journal"); + ( + ForwarderReaderControlHandler::new( + allow_reader_control, + service, + Arc::new(Mutex::new(journal)), + ), + tempdir, + ) + } + + #[test] + fn supports_reader_control_returns_constructor_flag() { + let (handler, _dir) = handler_with_reader_control_flag(false); + assert!(!handler.supports_reader_control()); + + let (handler, _dir) = handler_with_reader_control_flag(true); + assert!(handler.supports_reader_control()); + } + #[test] fn set_epoch_name_maps_name_and_blank_to_action() { let request = ReaderControlRequest { diff --git a/services/forwarder/src/status_http.rs b/services/forwarder/src/status_http.rs index cc8abd07..2ebd7c8b 100644 --- a/services/forwarder/src/status_http.rs +++ b/services/forwarder/src/status_http.rs @@ -1485,12 +1485,15 @@ pub async fn apply_section_update( return apply_control_action_from_config(&action, config_state, logger).await; } update_config_file(config_state, subsystem, ui_tx, |raw| { - // Preserve any existing allow_remote_config setting; this handler - // only mutates allow_power_actions. + // Preserve existing P2P control settings; this handler only + // mutates allow_power_actions. let allow_remote_config = raw.control.as_ref().and_then(|c| c.allow_remote_config); + let allow_reader_control = + raw.control.as_ref().and_then(|c| c.allow_reader_control); raw.control = Some(crate::config::RawControlConfig { allow_power_actions, allow_remote_config, + allow_reader_control, }); Ok(()) }) From 93bb10c2d0a72193a05724a6e85f976115652911 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 11:29:45 -0400 Subject: [PATCH 04/38] test(forwarder): cover ConfigGet capability gate; document gate-vs-handler ordering --- services/forwarder/src/p2p/control.rs | 54 +++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/services/forwarder/src/p2p/control.rs b/services/forwarder/src/p2p/control.rs index 935874d2..0cf8dda7 100644 --- a/services/forwarder/src/p2p/control.rs +++ b/services/forwarder/src/p2p/control.rs @@ -695,6 +695,10 @@ async fn run_control_loop( continue; } + // When control.allow_remote_config is false, CAP_REMOTE_CONFIG is + // never advertised, so this negotiated-capability gate fires before + // the handler's disabled check. The handler check remains defense in + // depth. let remote_config = Arc::clone(&remote_config); let config_response_tx = config_response_tx.clone(); control_tasks.spawn(async move { @@ -1079,6 +1083,7 @@ mod tests { config_json: String, restart_needed: bool, last_set: Mutex>, + get_calls: AtomicUsize, set_calls: AtomicUsize, restart_calls: AtomicUsize, } @@ -1090,6 +1095,7 @@ mod tests { config_json: r#"{"schema_version":1}"#.to_owned(), restart_needed: false, last_set: Mutex::new(None), + get_calls: AtomicUsize::new(0), set_calls: AtomicUsize::new(0), restart_calls: AtomicUsize::new(0), } @@ -1110,6 +1116,7 @@ mod tests { restart_needed: false, }; } + self.get_calls.fetch_add(1, Ordering::SeqCst); ConfigGetResponse { request_id: request.request_id, config_json: self.config_json.clone(), @@ -1465,6 +1472,53 @@ mod tests { Ok(()) } + #[tokio::test] + async fn config_get_rejected_when_capability_not_negotiated() -> TestResult { + let handler = Arc::new(FakeRemoteConfigHandler::new(true)); + let (forwarder, forwarder_addr, handle) = + spawn_forwarder_with_remote_config([74; 32], Arc::clone(&handler) as _).await?; + + let receiver = EndpointBuilder::test([75; 32]).bind().await?; + let (connection, mut send, mut recv) = tokio::time::timeout( + LONG_HANDSHAKE, + open_control(&receiver, forwarder_addr, forwarder_hello()), + ) + .await??; + + let _hello_ok = read_frame::(&mut recv).await?; + let _catalog = read_frame::(&mut recv).await?; + + write_frame( + &mut send, + &ControlC2F { + msg: Some(control_c2f::Msg::ConfigGetRequest(ConfigGetRequest { + request_id: "get-no-cap".to_owned(), + })), + }, + ) + .await?; + + let frame = + tokio::time::timeout(LONG_HANDSHAKE, read_frame::(&mut recv)).await??; + match frame.msg { + Some(control_f2c::Msg::ConfigGetResponse(response)) => { + assert_eq!(response.request_id, "get-no-cap"); + assert!( + response.config_json.is_empty(), + "config_json must be empty when remote-config capability was not negotiated" + ); + } + other => return Err(format!("expected ConfigGetResponse, got {other:?}").into()), + } + assert_eq!(handler.get_calls.load(Ordering::SeqCst), 0); + + connection.close(0u32.into(), b"done"); + handle.abort(); + receiver.close().await; + forwarder.close().await; + Ok(()) + } + #[tokio::test] async fn config_set_rejected_when_capability_not_negotiated() -> TestResult { let handler = Arc::new(FakeRemoteConfigHandler::new(true)); From e6a44ff7a9d9cf3065d18efbf4e072250f930640 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 11:52:31 -0400 Subject: [PATCH 05/38] fix(forwarder): write capability-gate denials inline to avoid control-loop self-deadlock --- services/forwarder/src/p2p/control.rs | 179 ++++++++++++++++++++++++-- 1 file changed, 165 insertions(+), 14 deletions(-) diff --git a/services/forwarder/src/p2p/control.rs b/services/forwarder/src/p2p/control.rs index 0cf8dda7..8eaf3d2d 100644 --- a/services/forwarder/src/p2p/control.rs +++ b/services/forwarder/src/p2p/control.rs @@ -252,6 +252,10 @@ pub struct NoopRemoteConfigHandler; /// Error returned for any remote-config verb when the feature is disabled. pub(crate) const REMOTE_CONFIG_DISABLED: &str = "remote config disabled"; +/// Error returned for any remote-config verb when the peer did not negotiate +/// [`CAP_REMOTE_CONFIG`] on this connection. +const REMOTE_CONFIG_NOT_NEGOTIATED: &str = "remote config was not negotiated on this connection"; + impl RemoteConfigHandler for NoopRemoteConfigHandler { fn allow_remote_config(&self) -> bool { false @@ -421,8 +425,9 @@ pub(crate) async fn negotiate_control_stream( /// Runs the post-negotiation heartbeat/control loop on an already-negotiated /// control stream until the peer disconnects cleanly (`Ok`) or is declared dead -/// (`Err`). Uses the default no-op reader-control handler and forwards status -/// updates from the optional `outbound_events` channel to the peer. +/// (`Err`). Dispatches reader-control and remote-config requests to the +/// supplied handlers and forwards status updates from the optional +/// `outbound_events` channel to the peer. pub(crate) async fn run_control_stream_loop( send: SendStream, recv: RecvStream, @@ -662,11 +667,20 @@ async fn run_control_loop( } Some(control_c2f::Msg::ReaderControlRequest(request)) => { if !reader_control_negotiated { + // Denials are written inline: the response channels are + // drained by other arms of this same select loop, so + // sending on them from here could fill the bounded channel + // and deadlock the loop under a request flood. let response = reader_control_error_response( - &request, + request, "reader control was not negotiated on this connection", ); - let _ = control_response_tx.send(response).await; + let frame = ControlF2C { + msg: Some(control_f2c::Msg::ReaderControlResponse(response)), + }; + if write_frame(&mut send, &frame).await.is_err() { + break Ok(()); + } continue; } @@ -691,7 +705,9 @@ async fn run_control_loop( let frame = ControlF2C { msg: Some(control_f2c::Msg::ConfigGetResponse(response)), }; - let _ = config_response_tx.send(frame).await; + if write_frame(&mut send, &frame).await.is_err() { + break Ok(()); + } continue; } @@ -715,13 +731,14 @@ async fn run_control_loop( request_id: request.request_id, ok: false, restart_needed: false, - error: "remote config was not negotiated on this connection" - .to_owned(), + error: REMOTE_CONFIG_NOT_NEGOTIATED.to_owned(), }; let frame = ControlF2C { msg: Some(control_f2c::Msg::ConfigSetResponse(response)), }; - let _ = config_response_tx.send(frame).await; + if write_frame(&mut send, &frame).await.is_err() { + break Ok(()); + } continue; } @@ -740,13 +757,14 @@ async fn run_control_loop( let response = RestartResponse { request_id: request.request_id, accepted: false, - error: "remote config was not negotiated on this connection" - .to_owned(), + error: REMOTE_CONFIG_NOT_NEGOTIATED.to_owned(), }; let frame = ControlF2C { msg: Some(control_f2c::Msg::RestartResponse(response)), }; - let _ = config_response_tx.send(frame).await; + if write_frame(&mut send, &frame).await.is_err() { + break Ok(()); + } continue; } @@ -809,12 +827,12 @@ async fn run_control_loop( } fn reader_control_error_response( - request: &ReaderControlRequest, + request: ReaderControlRequest, message: &str, ) -> ReaderControlResponse { ReaderControlResponse { - stream_id: request.stream_id.clone(), - request_id: request.request_id.clone(), + stream_id: request.stream_id, + request_id: request.request_id, success: false, message: message.to_owned(), reader_info_json: None, @@ -1570,6 +1588,139 @@ mod tests { Ok(()) } + #[tokio::test] + async fn restart_rejected_when_capability_not_negotiated() -> TestResult { + let handler = Arc::new(FakeRemoteConfigHandler::new(true)); + let (forwarder, forwarder_addr, handle) = + spawn_forwarder_with_remote_config([78; 32], Arc::clone(&handler) as _).await?; + + let receiver = EndpointBuilder::test([79; 32]).bind().await?; + let (connection, mut send, mut recv) = tokio::time::timeout( + LONG_HANDSHAKE, + open_control(&receiver, forwarder_addr, forwarder_hello()), + ) + .await??; + + let _hello_ok = read_frame::(&mut recv).await?; + let _catalog = read_frame::(&mut recv).await?; + + write_frame( + &mut send, + &ControlC2F { + msg: Some(control_c2f::Msg::RestartRequest(RestartRequest { + request_id: "restart-no-cap".to_owned(), + })), + }, + ) + .await?; + + let frame = + tokio::time::timeout(LONG_HANDSHAKE, read_frame::(&mut recv)).await??; + match frame.msg { + Some(control_f2c::Msg::RestartResponse(response)) => { + assert_eq!(response.request_id, "restart-no-cap"); + assert!(!response.accepted); + assert!( + response.error.contains("not negotiated"), + "error should mention negotiation, got {:?}", + response.error + ); + } + other => return Err(format!("expected RestartResponse, got {other:?}").into()), + } + assert_eq!(handler.restart_calls.load(Ordering::SeqCst), 0); + + connection.close(0u32.into(), b"done"); + handle.abort(); + receiver.close().await; + forwarder.close().await; + Ok(()) + } + + /// A non-conforming peer that floods gated requests without reading the + /// denials must not wedge the control loop: denials are written straight to + /// the stream by the select arm, so no bounded internal channel (cap 16, + /// drained only by another arm of the same loop) can fill up and deadlock. + #[tokio::test(flavor = "multi_thread")] + async fn gate_denial_flood_does_not_deadlock_control_loop() -> TestResult { + // Far above the internal response-channel capacity of 16. The old + // channel-based denial path only wedged once channel occupancy hit the + // cap, which under the select loop's random branch ordering is a random + // walk; a large burst makes hitting it overwhelmingly likely. + const FLOOD: usize = 2000; + + let handler = Arc::new(FakeRemoteConfigHandler::new(true)); + let (forwarder, forwarder_addr, handle) = + spawn_forwarder_with_remote_config([76; 32], Arc::clone(&handler) as _).await?; + + let receiver = EndpointBuilder::test([77; 32]).bind().await?; + let (connection, mut send, mut recv) = tokio::time::timeout( + LONG_HANDSHAKE, + open_control(&receiver, forwarder_addr, forwarder_hello()), + ) + .await??; + + let _hello_ok = read_frame::(&mut recv).await?; + let _catalog = read_frame::(&mut recv).await?; + + // Send the whole burst before reading anything back, so far more + // denials than the old channel capacity are in flight at once. + for i in 0..FLOOD { + write_frame( + &mut send, + &ControlC2F { + msg: Some(control_c2f::Msg::ConfigSetRequest(ConfigSetRequest { + request_id: format!("flood-{i}"), + config_json: r#"{"schema_version":1}"#.to_owned(), + })), + }, + ) + .await?; + } + + // Every request must be denied, in order. + for i in 0..FLOOD { + let frame = tokio::time::timeout(LONG_HANDSHAKE, read_frame::(&mut recv)) + .await + .map_err(|_| format!("control loop wedged: no denial for request {i}/{FLOOD}"))??; + match frame.msg { + Some(control_f2c::Msg::ConfigSetResponse(response)) => { + assert_eq!(response.request_id, format!("flood-{i}")); + assert!(!response.ok); + assert!( + response.error.contains("not negotiated"), + "error should mention negotiation, got {:?}", + response.error + ); + } + other => return Err(format!("expected ConfigSetResponse, got {other:?}").into()), + } + } + assert_eq!(handler.set_calls.load(Ordering::SeqCst), 0); + + // The loop must still be live after the flood: a Ping gets a Pong. + write_frame( + &mut send, + &ControlC2F { + msg: Some(control_c2f::Msg::Ping(Ping { nonce: 42 })), + }, + ) + .await?; + let frame = tokio::time::timeout(LONG_HANDSHAKE, read_frame::(&mut recv)) + .await + .map_err(|_| "control loop wedged: no Pong after gate-denial flood".to_owned())??; + match frame.msg { + Some(control_f2c::Msg::Pong(pong)) => assert_eq!(pong.nonce, 42), + other => return Err(format!("expected Pong, got {other:?}").into()), + } + + connection.close(0u32.into(), b"done"); + handle.abort(); + receiver.close().await; + forwarder.close().await; + Ok(()) + } + #[tokio::test] async fn config_set_rejected_when_disabled() -> TestResult { let (forwarder, forwarder_addr, handle) = spawn_forwarder_with_remote_config( From 82e03983d77f63fb7d204e35ced7294643be8527 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 12:02:13 -0400 Subject: [PATCH 06/38] fix(forwarder): block P2P remote-config writes to protected auth/p2p/control sections --- services/forwarder/README.md | 2 + services/forwarder/src/p2p/remote_config.rs | 225 +++++++++++++++++++- services/forwarder/src/status_http.rs | 68 +++++- 3 files changed, 287 insertions(+), 8 deletions(-) diff --git a/services/forwarder/README.md b/services/forwarder/README.md index df445cfc..abbeb47d 100644 --- a/services/forwarder/README.md +++ b/services/forwarder/README.md @@ -86,6 +86,8 @@ or `server_url`. | `allow_remote_config` | `bool` | No | `true` | Allows receivers to read/write forwarder config over the P2P control plane. | | `allow_reader_control` | `bool` | No | `true` | Allows receivers to execute reader-control verbs (status, download, clear, epoch) over the P2P control plane. | +Remote (P2P) config writes may not modify `[auth]`, `[p2p]`, or `[control]`; those sections are only editable locally. + ### `[update]` | Field | Type | Required | Default | Description | diff --git a/services/forwarder/src/p2p/remote_config.rs b/services/forwarder/src/p2p/remote_config.rs index b40d6ce5..56cf8c9e 100644 --- a/services/forwarder/src/p2p/remote_config.rs +++ b/services/forwarder/src/p2p/remote_config.rs @@ -7,9 +7,10 @@ //! //! - get -> [`crate::status_http::config_json_string`] (same body as //! `GET /api/v1/config`), -//! - set -> [`crate::status_http::write_config_json`] (parses the full config -//! document, validates via the canonical loader, writes the TOML file -//! atomically, and marks a restart as needed), +//! - set -> [`crate::status_http::write_config_json_restricted`] (rejects +//! privileged section changes, parses the full config document, validates via +//! the canonical loader, writes the TOML file atomically, and marks a restart +//! as needed), //! - restart -> the same `restart_signal` [`Notify`] the HTTP restart endpoint //! triggers. //! @@ -28,7 +29,9 @@ use rt_p2p_protocol::{ }; use tokio::sync::{Mutex, Notify, broadcast}; -use crate::status_http::{ConfigState, SubsystemStatus, config_json_string, write_config_json}; +use crate::status_http::{ + ConfigState, SubsystemStatus, config_json_string, write_config_json_restricted, +}; use crate::ui_events::ForwarderUiEvent; use super::control::{ @@ -122,7 +125,7 @@ impl RemoteConfigHandler for ForwarderRemoteConfigHandler { error: REMOTE_CONFIG_DISABLED.to_owned(), }; } - match write_config_json( + match write_config_json_restricted( &request.config_json, &self.config_state, &self.subsystem, @@ -302,11 +305,21 @@ mod tests { let (config_path, _dir) = temp_config(""); let h = handler(true, config_path.clone(), Arc::new(Notify::new())); - // schema_version 2 is rejected by the canonical loader. + // schema_version 2 is rejected by the canonical loader; keep protected + // sections unchanged so this remains a validation test. + let mut value: serde_json::Value = serde_json::from_str( + &h.get_config(ConfigGetRequest { + request_id: "g".into(), + }) + .await + .config_json, + ) + .unwrap(); + value["schema_version"] = serde_json::json!(2); let response = h .set_config(ConfigSetRequest { request_id: "s2".to_owned(), - config_json: r#"{"schema_version":2}"#.to_owned(), + config_json: serde_json::to_string(&value).unwrap(), }) .await; @@ -336,6 +349,204 @@ mod tests { assert_eq!(response.error, REMOTE_CONFIG_DISABLED); } + #[tokio::test] + async fn set_rejects_p2p_section_changes() { + let (config_path, _dir) = temp_config("[p2p]\nenabled = false\n"); + let h = handler(true, config_path.clone(), Arc::new(Notify::new())); + let mut value: serde_json::Value = serde_json::from_str( + &h.get_config(ConfigGetRequest { + request_id: "g".into(), + }) + .await + .config_json, + ) + .unwrap(); + + value["p2p"]["static_allowed_receivers"] = serde_json::json!(["ff".repeat(32)]); + let response = h + .set_config(ConfigSetRequest { + request_id: "s".into(), + config_json: serde_json::to_string(&value).unwrap(), + }) + .await; + + assert!(!response.ok); + assert!( + response.error.contains("p2p"), + "error must name the protected section: {}", + response.error + ); + let cfg = crate::config::load_config_from_path(&config_path).unwrap(); + assert!(cfg.p2p.static_allowed_receivers.is_empty()); + } + + #[tokio::test] + async fn set_rejects_auth_section_changes() { + let (config_path, dir) = temp_config(""); + let alternate_token_path = dir.path().join("alternate-token"); + std::fs::write(&alternate_token_path, "alternate-token\n").expect("write alternate token"); + let h = handler(true, config_path.clone(), Arc::new(Notify::new())); + let mut value: serde_json::Value = serde_json::from_str( + &h.get_config(ConfigGetRequest { + request_id: "g".into(), + }) + .await + .config_json, + ) + .unwrap(); + + value["auth"]["token_file"] = serde_json::json!(alternate_token_path.display().to_string()); + let response = h + .set_config(ConfigSetRequest { + request_id: "s".into(), + config_json: serde_json::to_string(&value).unwrap(), + }) + .await; + + assert!(!response.ok); + assert!( + response.error.contains("auth"), + "error must name the protected section: {}", + response.error + ); + let cfg = crate::config::load_config_from_path(&config_path).unwrap(); + assert_eq!(cfg.token, "test-token"); + } + + #[tokio::test] + async fn set_rejects_control_section_changes() { + let (config_path, _dir) = temp_config("[control]\nallow_remote_config = true\n"); + let h = handler(true, config_path.clone(), Arc::new(Notify::new())); + let mut value: serde_json::Value = serde_json::from_str( + &h.get_config(ConfigGetRequest { + request_id: "g".into(), + }) + .await + .config_json, + ) + .unwrap(); + + value["control"]["allow_remote_config"] = serde_json::json!(false); + let response = h + .set_config(ConfigSetRequest { + request_id: "s".into(), + config_json: serde_json::to_string(&value).unwrap(), + }) + .await; + + assert!(!response.ok); + assert!( + response.error.contains("control"), + "error must name the protected section: {}", + response.error + ); + let cfg = crate::config::load_config_from_path(&config_path).unwrap(); + assert!(cfg.control.allow_remote_config); + } + + #[tokio::test] + async fn set_allows_operational_section_changes() { + let (config_path, dir) = temp_config(""); + let h = handler(true, config_path.clone(), Arc::new(Notify::new())); + let mut value: serde_json::Value = serde_json::from_str( + &h.get_config(ConfigGetRequest { + request_id: "g".into(), + }) + .await + .config_json, + ) + .unwrap(); + let sqlite_path = dir.path().join("edited.sqlite3"); + + value["display_name"] = serde_json::json!("Remote Edited Forwarder"); + value["journal"] = serde_json::json!({ + "sqlite_path": sqlite_path.display().to_string(), + "prune_watermark_pct": 80, + }); + value["readers"] = serde_json::json!([ + { + "target": "192.168.1.101", + "enabled": false, + "local_fallback_port": 10101, + } + ]); + let response = h + .set_config(ConfigSetRequest { + request_id: "s".into(), + config_json: serde_json::to_string(&value).unwrap(), + }) + .await; + + assert!(response.ok, "set should succeed: {}", response.error); + let cfg = crate::config::load_config_from_path(&config_path).unwrap(); + assert_eq!(cfg.display_name.as_deref(), Some("Remote Edited Forwarder")); + assert_eq!(cfg.journal.sqlite_path, sqlite_path.display().to_string()); + assert_eq!(cfg.journal.prune_watermark_pct, 80); + assert_eq!(cfg.readers.len(), 1); + assert_eq!(cfg.readers[0].target, "192.168.1.101"); + assert!(!cfg.readers[0].enabled); + assert_eq!(cfg.readers[0].local_fallback_port, Some(10101)); + } + + #[tokio::test] + async fn set_allows_missing_control_to_all_null_control() { + let (config_path, _dir) = temp_config(""); + let h = handler(true, config_path, Arc::new(Notify::new())); + let mut value: serde_json::Value = serde_json::from_str( + &h.get_config(ConfigGetRequest { + request_id: "g".into(), + }) + .await + .config_json, + ) + .unwrap(); + + value["control"] = serde_json::json!({ + "allow_power_actions": null, + "allow_remote_config": null, + "allow_reader_control": null, + }); + let response = h + .set_config(ConfigSetRequest { + request_id: "s".into(), + config_json: serde_json::to_string(&value).unwrap(), + }) + .await; + + assert!(response.ok, "set should succeed: {}", response.error); + } + + #[tokio::test] + async fn set_rejects_populated_control_to_missing_control() { + let (config_path, _dir) = temp_config("[control]\nallow_remote_config = true\n"); + let h = handler(true, config_path.clone(), Arc::new(Notify::new())); + let mut value: serde_json::Value = serde_json::from_str( + &h.get_config(ConfigGetRequest { + request_id: "g".into(), + }) + .await + .config_json, + ) + .unwrap(); + + value.as_object_mut().unwrap().remove("control"); + let response = h + .set_config(ConfigSetRequest { + request_id: "s".into(), + config_json: serde_json::to_string(&value).unwrap(), + }) + .await; + + assert!(!response.ok); + assert!( + response.error.contains("control"), + "error must name the protected section: {}", + response.error + ); + let cfg = crate::config::load_config_from_path(&config_path).unwrap(); + assert!(cfg.control.allow_remote_config); + } + #[cfg(unix)] #[tokio::test] async fn restart_notifies_signal_without_killing_process() { diff --git a/services/forwarder/src/status_http.rs b/services/forwarder/src/status_http.rs index 2ebd7c8b..d4d12798 100644 --- a/services/forwarder/src/status_http.rs +++ b/services/forwarder/src/status_http.rs @@ -1762,6 +1762,16 @@ pub async fn config_json_string( Ok((json, restart_needed)) } +/// Sections a P2P peer may never change: identity/credentials (`auth`), +/// transport trust (`p2p`), and the gates themselves (`control`). +/// +/// Boundary rationale: `[auth]`/`[p2p]`/`[control]` changes are trust/identity +/// escalation — permanent and self-granting; `[journal]`, `[[readers]]`, and +/// display fields are the operational surface remote management exists to serve +/// — a hostile allow-listed receiver can disrupt operations through them but +/// cannot expand its own access. +const REMOTE_PROTECTED_SECTIONS: &[&str] = &["auth", "p2p", "control"]; + /// Persist a full config document (the same JSON shape `config_json_string` /// returns) to the TOML config file, then mark a restart as needed. /// @@ -1777,11 +1787,55 @@ pub async fn write_config_json( subsystem: &Arc>, ui_tx: &tokio::sync::broadcast::Sender, ) -> Result<(), String> { - let raw: crate::config::RawConfig = + let _lock = config_state.write_lock.lock().await; + write_config_json_locked(config_json, config_state, subsystem, ui_tx).await +} + +/// Persist a full config document received from P2P remote config, rejecting +/// any attempted change to privileged config sections before writing. +pub async fn write_config_json_restricted( + config_json: &str, + config_state: &ConfigState, + subsystem: &Arc>, + ui_tx: &tokio::sync::broadcast::Sender, +) -> Result<(), String> { + let incoming: crate::config::RawConfig = serde_json::from_str(config_json).map_err(|e| format!("invalid config JSON: {e}"))?; let _lock = config_state.write_lock.lock().await; + let current_toml = + std::fs::read_to_string(&config_state.path).map_err(|e| format!("File read error: {e}"))?; + let current: crate::config::RawConfig = + toml::from_str(¤t_toml).map_err(|e| format!("TOML parse error: {e}"))?; + + let incoming_value = + serde_json::to_value(&incoming).map_err(|e| format!("JSON serialize error: {e}"))?; + let current_value = + serde_json::to_value(¤t).map_err(|e| format!("JSON serialize error: {e}"))?; + + for section in REMOTE_PROTECTED_SECTIONS { + if normalize_section(incoming_value.get(section)) + != normalize_section(current_value.get(section)) + { + return Err(format!( + "remote config may not modify the protected [{section}] section" + )); + } + } + + write_config_json_locked(config_json, config_state, subsystem, ui_tx).await +} + +async fn write_config_json_locked( + config_json: &str, + config_state: &ConfigState, + subsystem: &Arc>, + ui_tx: &tokio::sync::broadcast::Sender, +) -> Result<(), String> { + let raw: crate::config::RawConfig = + serde_json::from_str(config_json).map_err(|e| format!("invalid config JSON: {e}"))?; + let new_toml = toml::to_string_pretty(&raw).map_err(|e| format!("TOML serialize error: {e}"))?; @@ -1796,6 +1850,18 @@ pub async fn write_config_json( Ok(()) } +/// A missing section, `null`, and an all-null object are equivalent RawConfig +/// states. +fn normalize_section(v: Option<&serde_json::Value>) -> serde_json::Value { + match v { + None | Some(serde_json::Value::Null) => serde_json::Value::Null, + Some(serde_json::Value::Object(map)) if map.values().all(serde_json::Value::is_null) => { + serde_json::Value::Null + } + Some(other) => other.clone(), + } +} + fn text_response(status: StatusCode, body: impl Into) -> Response { (status, [(header::CONTENT_TYPE, "text/plain")], body.into()).into_response() } From 7a4dc69328250357aca4fb41c7323933db8b069d Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 12:07:49 -0400 Subject: [PATCH 07/38] test(forwarder): guard remote-config round-trip with populated protected sections --- services/forwarder/src/p2p/remote_config.rs | 40 +++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/services/forwarder/src/p2p/remote_config.rs b/services/forwarder/src/p2p/remote_config.rs index 56cf8c9e..628b5937 100644 --- a/services/forwarder/src/p2p/remote_config.rs +++ b/services/forwarder/src/p2p/remote_config.rs @@ -488,6 +488,46 @@ mod tests { assert_eq!(cfg.readers[0].local_fallback_port, Some(10101)); } + /// Regression guard against serialization drift (e.g. a future + /// `skip_serializing_if` on a protected raw section) causing false + /// rejections: a pure get -> set round-trip of a config with populated + /// `[p2p]` and `[control]` sections must be accepted when only + /// operational fields change. + #[tokio::test] + async fn set_roundtrip_with_populated_protected_sections_is_accepted() { + let (config_path, _dir) = temp_config( + "[p2p]\nenabled = false\n\n[control]\nallow_power_actions = false\nallow_remote_config = true\n", + ); + let h = handler(true, config_path.clone(), Arc::new(Notify::new())); + let mut value: serde_json::Value = serde_json::from_str( + &h.get_config(ConfigGetRequest { + request_id: "g".into(), + }) + .await + .config_json, + ) + .unwrap(); + + // Only an operational change; protected sections round-trip untouched. + value["display_name"] = serde_json::json!("Round Trip"); + let response = h + .set_config(ConfigSetRequest { + request_id: "s".into(), + config_json: serde_json::to_string(&value).unwrap(), + }) + .await; + + assert!( + response.ok, + "round-trip with populated protected sections must be accepted: {}", + response.error + ); + let cfg = crate::config::load_config_from_path(&config_path).unwrap(); + assert_eq!(cfg.display_name.as_deref(), Some("Round Trip")); + assert!(!cfg.p2p.enabled); + assert!(cfg.control.allow_remote_config); + } + #[tokio::test] async fn set_allows_missing_control_to_all_null_control() { let (config_path, _dir) = temp_config(""); From 5a0d687c06b8312ac9b59b692ff623049ce78126 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 12:15:54 -0400 Subject: [PATCH 08/38] refactor(forwarder): remove unrestricted full-document config writer; document write-path contracts --- services/forwarder/src/status_http.rs | 45 +++++++++++++++------------ 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/services/forwarder/src/status_http.rs b/services/forwarder/src/status_http.rs index d4d12798..908b1608 100644 --- a/services/forwarder/src/status_http.rs +++ b/services/forwarder/src/status_http.rs @@ -1769,11 +1769,14 @@ pub async fn config_json_string( /// escalation — permanent and self-granting; `[journal]`, `[[readers]]`, and /// display fields are the operational surface remote management exists to serve /// — a hostile allow-listed receiver can disrupt operations through them but -/// cannot expand its own access. +/// cannot expand its own access. `[status_http]`, `[update]`, and `[ups]` were +/// considered and left writable as operational surface; `status_http.bind` is +/// the borderline case (a receiver could rebind the local admin HTTP), but it +/// grants the receiver itself no access. const REMOTE_PROTECTED_SECTIONS: &[&str] = &["auth", "p2p", "control"]; -/// Persist a full config document (the same JSON shape `config_json_string` -/// returns) to the TOML config file, then mark a restart as needed. +/// Persist a full config document received from P2P remote config, rejecting +/// any attempted change to privileged config sections before writing. /// /// The document is parsed into the same [`crate::config::RawConfig`] the get /// path serializes, re-serialized to TOML, and validated by running the @@ -1781,18 +1784,11 @@ const REMOTE_PROTECTED_SECTIONS: &[&str] = &["auth", "p2p", "control"]; /// to load on restart is rejected without corrupting the on-disk file. Reuses /// the same atomic writer and `restart_needed` signal as the per-section HTTP /// writers. Returns a plain error message on failure. -pub async fn write_config_json( - config_json: &str, - config_state: &ConfigState, - subsystem: &Arc>, - ui_tx: &tokio::sync::broadcast::Sender, -) -> Result<(), String> { - let _lock = config_state.write_lock.lock().await; - write_config_json_locked(config_json, config_state, subsystem, ui_tx).await -} - -/// Persist a full config document received from P2P remote config, rejecting -/// any attempted change to privileged config sections before writing. +/// +/// There is intentionally no unrestricted full-document writer: local +/// (trusted) config edits go through the per-section HTTP handlers +/// ([`apply_section_update`] / [`update_config_file`]), so every full-document +/// write path enforces [`REMOTE_PROTECTED_SECTIONS`]. pub async fn write_config_json_restricted( config_json: &str, config_state: &ConfigState, @@ -1824,18 +1820,20 @@ pub async fn write_config_json_restricted( } } - write_config_json_locked(config_json, config_state, subsystem, ui_tx).await + write_config_json_locked(incoming, config_state, subsystem, ui_tx).await } +/// Shared locked write body. Caller must hold `config_state.write_lock`. +/// +/// Takes the already-parsed [`crate::config::RawConfig`] (the same value the +/// protected-section comparison ran against) so "what was compared is what is +/// written" holds structurally rather than by re-parsing the same string. async fn write_config_json_locked( - config_json: &str, + raw: crate::config::RawConfig, config_state: &ConfigState, subsystem: &Arc>, ui_tx: &tokio::sync::broadcast::Sender, ) -> Result<(), String> { - let raw: crate::config::RawConfig = - serde_json::from_str(config_json).map_err(|e| format!("invalid config JSON: {e}"))?; - let new_toml = toml::to_string_pretty(&raw).map_err(|e| format!("TOML serialize error: {e}"))?; @@ -1852,6 +1850,13 @@ async fn write_config_json_locked( /// A missing section, `null`, and an all-null object are equivalent RawConfig /// states. +/// +/// Normalization is single-level by design: every protected raw section is a +/// flat struct of scalars/`Vec` today. If a nested struct is ever +/// added to a protected section, an all-null nested object will not be +/// flattened — the failure mode is a false *reject* (fail-closed), and the +/// populated-section round-trip test in `p2p/remote_config.rs` will catch +/// serialization drift. fn normalize_section(v: Option<&serde_json::Value>) -> serde_json::Value { match v { None | Some(serde_json::Value::Null) => serde_json::Value::Null, From 77caab90c091bf5e721ac11080fbe92ed89bb682 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 12:27:45 -0400 Subject: [PATCH 09/38] feat(forwarder): log peer node id for P2P control-plane verbs Thread the authenticated peer NodeId from connection admission into the control loop and emit a tracing event (peer, verb, request_id, target, outcome) at every reader-control and remote-config dispatch arm, including capability-gate denials. Extend RemoteConfigHandler::set_config to take the peer so config writes are attributed in the UI log: ForwarderRemoteConfigHandler now holds the shared UiLogger and logs both accepted and rejected remote config writes with the requesting receiver's node id. --- services/forwarder/src/main.rs | 1 + services/forwarder/src/p2p/control.rs | 107 ++++++++- services/forwarder/src/p2p/endpoint.rs | 154 +++++++++++++ services/forwarder/src/p2p/remote_config.rs | 237 +++++++++++++++----- 4 files changed, 440 insertions(+), 59 deletions(-) diff --git a/services/forwarder/src/main.rs b/services/forwarder/src/main.rs index 30c44565..830c3d75 100644 --- a/services/forwarder/src/main.rs +++ b/services/forwarder/src/main.rs @@ -303,6 +303,7 @@ async fn main() { config_state.clone(), status_server.subsystem_arc(), status_server.ui_sender(), + status_server.logger(), restart_signal.clone(), )); let reader_control_handler = Arc::new(forwarder::p2p::ForwarderReaderControlHandler::new( diff --git a/services/forwarder/src/p2p/control.rs b/services/forwarder/src/p2p/control.rs index 8eaf3d2d..bf306ece 100644 --- a/services/forwarder/src/p2p/control.rs +++ b/services/forwarder/src/p2p/control.rs @@ -23,7 +23,7 @@ use std::time::Duration; use prost::Message; #[cfg(test)] use rt_iroh::Connection; -use rt_iroh::{RecvStream, SendStream}; +use rt_iroh::{NodeId, RecvStream, SendStream}; use rt_p2p_protocol::{ CAP_CONTROL_EVENTS, CAP_READER_CONTROL, CAP_REMOTE_CONFIG, ConfigGetRequest, ConfigGetResponse, ConfigSetRequest, ConfigSetResponse, ControlC2F, ControlF2C, DownloadProgress, Hello, @@ -219,7 +219,11 @@ pub trait RemoteConfigHandler: std::fmt::Debug + Send + Sync + 'static { /// Persists `config_json` (same shape as get) to the forwarder's TOML /// config file and marks a restart as needed. On validation or IO failure /// returns `ok = false` with a descriptive error and never panics. - fn set_config(&self, request: ConfigSetRequest) -> ConfigSetFuture<'_>; + /// + /// `peer` is the authenticated node id of the receiver that sent the + /// request; implementations use it to attribute the config write in + /// UI-visible logs. + fn set_config(&self, request: ConfigSetRequest, peer: NodeId) -> ConfigSetFuture<'_>; /// Triggers the same graceful restart path as the HTTP restart endpoint. fn restart(&self, request: RestartRequest) -> RestartFuture<'_>; @@ -234,8 +238,8 @@ impl RemoteConfigHandler for Arc { (**self).get_config(request) } - fn set_config(&self, request: ConfigSetRequest) -> ConfigSetFuture<'_> { - (**self).set_config(request) + fn set_config(&self, request: ConfigSetRequest, peer: NodeId) -> ConfigSetFuture<'_> { + (**self).set_config(request, peer) } fn restart(&self, request: RestartRequest) -> RestartFuture<'_> { @@ -271,7 +275,7 @@ impl RemoteConfigHandler for NoopRemoteConfigHandler { }) } - fn set_config(&self, request: ConfigSetRequest) -> ConfigSetFuture<'_> { + fn set_config(&self, request: ConfigSetRequest, _peer: NodeId) -> ConfigSetFuture<'_> { Box::pin(async move { ConfigSetResponse { request_id: request.request_id, @@ -427,10 +431,13 @@ pub(crate) async fn negotiate_control_stream( /// control stream until the peer disconnects cleanly (`Ok`) or is declared dead /// (`Err`). Dispatches reader-control and remote-config requests to the /// supplied handlers and forwards status updates from the optional -/// `outbound_events` channel to the peer. +/// `outbound_events` channel to the peer. `peer` is the authenticated node id +/// of the connected receiver; every verb dispatch is attributed to it in logs. +#[allow(clippy::too_many_arguments)] pub(crate) async fn run_control_stream_loop( send: SendStream, recv: RecvStream, + peer: NodeId, capabilities: Vec, heartbeat: HeartbeatConfig, outbound_events: Option, @@ -440,6 +447,7 @@ pub(crate) async fn run_control_stream_loop( run_control_loop( send, recv, + peer, capabilities, heartbeat, reader_control, @@ -461,10 +469,12 @@ pub(crate) async fn serve_control_with_typed_control( outbound_events: Option, remote_config: Arc, ) -> Result<(), BoxError> { + let peer = connection.remote_node_id()?; let (send, recv) = connection.accept_bi().await?; serve_control_stream_with_typed_control( send, recv, + peer, catalog, handshake_timeout, heartbeat, @@ -480,6 +490,7 @@ pub(crate) async fn serve_control_with_typed_control( pub(crate) async fn serve_control_stream_with_typed_control( send: SendStream, recv: RecvStream, + peer: NodeId, catalog: &dyn CatalogProvider, handshake_timeout: Duration, heartbeat: HeartbeatConfig, @@ -501,6 +512,7 @@ pub(crate) async fn serve_control_stream_with_typed_control( run_control_loop( send, recv, + peer, capabilities, heartbeat, reader_control, @@ -595,9 +607,15 @@ async fn negotiate_and_serve_catalog_stream( /// Runs the typed control loop until the peer misses `max_missed` consecutive /// pongs (returns `Err`) or disconnects cleanly (returns `Ok`). +/// +/// Every reader-control and remote-config verb dispatch emits a `tracing` +/// event attributed to `peer` (the authenticated remote node id), including +/// capability-gate denials, so control-plane operations are auditable. +#[allow(clippy::too_many_arguments)] async fn run_control_loop( mut send: SendStream, mut recv: RecvStream, + peer: NodeId, capabilities: Vec, config: HeartbeatConfig, reader_control: Arc, @@ -667,6 +685,15 @@ async fn run_control_loop( } Some(control_c2f::Msg::ReaderControlRequest(request)) => { if !reader_control_negotiated { + tracing::info!( + %peer, + verb = "reader_control", + command = %request.command, + request_id = %request.request_id, + stream_id = %String::from_utf8_lossy(&request.stream_id), + outcome = "denied", + "p2p: control verb denied (capability not negotiated)" + ); // Denials are written inline: the response channels are // drained by other arms of this same select loop, so // sending on them from here could fill the bounded channel @@ -686,17 +713,34 @@ async fn run_control_loop( let stream_id = request.stream_id.clone(); let request_id = request.request_id.clone(); + let command = request.command.clone(); let reader_control = Arc::clone(&reader_control); let control_response_tx = control_response_tx.clone(); control_tasks.spawn(async move { let mut response = reader_control.handle(request).await; response.stream_id = stream_id; response.request_id = request_id; + tracing::info!( + %peer, + verb = "reader_control", + command = %command, + request_id = %response.request_id, + stream_id = %String::from_utf8_lossy(&response.stream_id), + outcome = if response.success { "ok" } else { "error" }, + "p2p: control verb handled" + ); let _ = control_response_tx.send(response).await; }); } Some(control_c2f::Msg::ConfigGetRequest(request)) => { if !remote_config_negotiated { + tracing::info!( + %peer, + verb = "config_get", + request_id = %request.request_id, + outcome = "denied", + "p2p: control verb denied (capability not negotiated)" + ); let response = ConfigGetResponse { request_id: request.request_id, config_json: String::new(), @@ -719,6 +763,17 @@ async fn run_control_loop( let config_response_tx = config_response_tx.clone(); control_tasks.spawn(async move { let response = remote_config.get_config(request).await; + tracing::info!( + %peer, + verb = "config_get", + request_id = %response.request_id, + outcome = if response.config_json.is_empty() { + "error" + } else { + "ok" + }, + "p2p: control verb handled" + ); let frame = ControlF2C { msg: Some(control_f2c::Msg::ConfigGetResponse(response)), }; @@ -727,6 +782,13 @@ async fn run_control_loop( } Some(control_c2f::Msg::ConfigSetRequest(request)) => { if !remote_config_negotiated { + tracing::info!( + %peer, + verb = "config_set", + request_id = %request.request_id, + outcome = "denied", + "p2p: control verb denied (capability not negotiated)" + ); let response = ConfigSetResponse { request_id: request.request_id, ok: false, @@ -745,7 +807,14 @@ async fn run_control_loop( let remote_config = Arc::clone(&remote_config); let config_response_tx = config_response_tx.clone(); control_tasks.spawn(async move { - let response = remote_config.set_config(request).await; + let response = remote_config.set_config(request, peer).await; + tracing::info!( + %peer, + verb = "config_set", + request_id = %response.request_id, + outcome = if response.ok { "ok" } else { "error" }, + "p2p: control verb handled" + ); let frame = ControlF2C { msg: Some(control_f2c::Msg::ConfigSetResponse(response)), }; @@ -754,6 +823,13 @@ async fn run_control_loop( } Some(control_c2f::Msg::RestartRequest(request)) => { if !remote_config_negotiated { + tracing::info!( + %peer, + verb = "restart", + request_id = %request.request_id, + outcome = "denied", + "p2p: control verb denied (capability not negotiated)" + ); let response = RestartResponse { request_id: request.request_id, accepted: false, @@ -772,6 +848,13 @@ async fn run_control_loop( let config_response_tx = config_response_tx.clone(); control_tasks.spawn(async move { let response = remote_config.restart(request).await; + tracing::info!( + %peer, + verb = "restart", + request_id = %response.request_id, + outcome = if response.accepted { "ok" } else { "error" }, + "p2p: control verb handled" + ); let frame = ControlF2C { msg: Some(control_f2c::Msg::RestartResponse(response)), }; @@ -1101,6 +1184,7 @@ mod tests { config_json: String, restart_needed: bool, last_set: Mutex>, + last_set_peer: Mutex>, get_calls: AtomicUsize, set_calls: AtomicUsize, restart_calls: AtomicUsize, @@ -1113,6 +1197,7 @@ mod tests { config_json: r#"{"schema_version":1}"#.to_owned(), restart_needed: false, last_set: Mutex::new(None), + last_set_peer: Mutex::new(None), get_calls: AtomicUsize::new(0), set_calls: AtomicUsize::new(0), restart_calls: AtomicUsize::new(0), @@ -1143,7 +1228,7 @@ mod tests { }) } - fn set_config(&self, request: ConfigSetRequest) -> ConfigSetFuture<'_> { + fn set_config(&self, request: ConfigSetRequest, peer: NodeId) -> ConfigSetFuture<'_> { Box::pin(async move { if !self.allow { return ConfigSetResponse { @@ -1155,6 +1240,7 @@ mod tests { } self.set_calls.fetch_add(1, Ordering::SeqCst); *self.last_set.lock().unwrap() = Some(request.config_json.clone()); + *self.last_set_peer.lock().unwrap() = Some(peer); ConfigSetResponse { request_id: request.request_id, ok: true, @@ -1482,6 +1568,11 @@ mod tests { Some(payload), "handler must receive the config_json sent over the wire" ); + assert_eq!( + *handler.last_set_peer.lock().unwrap(), + Some(receiver.node_id()), + "handler must be told which peer sent the config write" + ); connection.close(0u32.into(), b"done"); handle.abort(); diff --git a/services/forwarder/src/p2p/endpoint.rs b/services/forwarder/src/p2p/endpoint.rs index 002a1657..c72db594 100644 --- a/services/forwarder/src/p2p/endpoint.rs +++ b/services/forwarder/src/p2p/endpoint.rs @@ -307,6 +307,7 @@ async fn handle_connection( let control_result = run_control_stream_loop( control_send, control_recv, + node_id, capabilities, config.heartbeat, outbound_events, @@ -749,11 +750,15 @@ mod tests { )?; let (ui_tx, _ui_rx) = broadcast::channel(16); + let ui_logger = Arc::new(rt_ui_log::UiLogger::new(ui_tx.clone(), |entry| { + crate::ui_events::ForwarderUiEvent::LogEntry { entry } + })); let handler = Arc::new(crate::p2p::ForwarderRemoteConfigHandler::new( true, Arc::new(crate::status_http::ConfigState::new(config_path)), Arc::new(Mutex::new(SubsystemStatus::ready())), ui_tx, + ui_logger, Arc::new(tokio::sync::Notify::new()), )); @@ -822,6 +827,155 @@ mod tests { Ok(()) } + /// A remote config write over P2P must surface a UI log entry that + /// attributes the write to the requesting receiver's node id. + #[tokio::test] + async fn remote_config_set_over_p2p_emits_ui_log_with_peer_node_id() -> TestResult { + use rt_p2p_protocol::{CAP_REMOTE_CONFIG, ConfigGetRequest, ConfigSetRequest, Pong}; + + let receiver = EndpointBuilder::test([84; 32]).bind().await?; + let allow_list = AllowList::new([receiver.node_id()]); + + let dir = tempfile::tempdir()?; + let token_path = dir.path().join("token"); + std::fs::write(&token_path, "tok\n")?; + let config_path = dir.path().join("forwarder.toml"); + std::fs::write( + &config_path, + format!( + "schema_version = 1\n\n[auth]\ntoken_file = '{}'\n\n[[readers]]\ntarget = \"192.168.1.100\"\n\n[control]\nallow_remote_config = true\n", + token_path.display() + ), + )?; + + let (ui_tx, mut ui_rx) = broadcast::channel(64); + let ui_logger = Arc::new(rt_ui_log::UiLogger::new(ui_tx.clone(), |entry| { + crate::ui_events::ForwarderUiEvent::LogEntry { entry } + })); + let handler = Arc::new(crate::p2p::ForwarderRemoteConfigHandler::new( + true, + Arc::new(crate::status_http::ConfigState::new(config_path)), + Arc::new(Mutex::new(SubsystemStatus::ready())), + ui_tx, + ui_logger, + Arc::new(tokio::sync::Notify::new()), + )); + + let forwarder = P2pEndpoint::bind_test([85; 32], allow_list) + .await? + .with_remote_config(handler); + let forwarder_addr = forwarder.node_addr().await; + + let accept = { + let forwarder = forwarder.clone(); + tokio::spawn(async move { forwarder.run().await }) + }; + + let (connection, _hello_ok, mut send, mut recv) = tokio::time::timeout( + Duration::from_secs(5), + dial_hello_connection_with_capabilities( + &receiver, + forwarder_addr, + vec![CAP_CONTROL_EVENTS.to_owned(), CAP_REMOTE_CONFIG.to_owned()], + ), + ) + .await??; + + // Fetch the current config, edit an operational field, and write it + // back, answering heartbeat pings while waiting for responses. + write_frame( + &mut send, + &ControlC2F { + msg: Some(control_c2f::Msg::ConfigGetRequest(ConfigGetRequest { + request_id: "get".to_owned(), + })), + }, + ) + .await?; + let get_response = loop { + let frame = + tokio::time::timeout(Duration::from_secs(5), read_frame::(&mut recv)) + .await??; + match frame.msg { + Some(control_f2c::Msg::ConfigGetResponse(response)) => break response, + Some(control_f2c::Msg::Ping(ping)) => { + write_frame( + &mut send, + &ControlC2F { + msg: Some(control_c2f::Msg::Pong(Pong { nonce: ping.nonce })), + }, + ) + .await?; + } + other => return Err(format!("unexpected control frame: {other:?}").into()), + } + }; + let mut value: serde_json::Value = serde_json::from_str(&get_response.config_json)?; + value["display_name"] = serde_json::json!("Peer Attributed"); + + write_frame( + &mut send, + &ControlC2F { + msg: Some(control_c2f::Msg::ConfigSetRequest(ConfigSetRequest { + request_id: "set".to_owned(), + config_json: serde_json::to_string(&value)?, + })), + }, + ) + .await?; + let set_response = loop { + let frame = + tokio::time::timeout(Duration::from_secs(5), read_frame::(&mut recv)) + .await??; + match frame.msg { + Some(control_f2c::Msg::ConfigSetResponse(response)) => break response, + Some(control_f2c::Msg::Ping(ping)) => { + write_frame( + &mut send, + &ControlC2F { + msg: Some(control_c2f::Msg::Pong(Pong { nonce: ping.nonce })), + }, + ) + .await?; + } + other => return Err(format!("unexpected control frame: {other:?}").into()), + } + }; + assert!( + set_response.ok, + "remote config set should succeed: {}", + set_response.error + ); + + // The UI event stream must carry a log entry attributing the write to + // the receiver's node id. + let peer_id = receiver.node_id().to_string(); + let attributed = tokio::time::timeout(Duration::from_secs(5), async { + loop { + match ui_rx.recv().await { + Ok(crate::ui_events::ForwarderUiEvent::LogEntry { entry }) + if entry.contains(&peer_id) => + { + break true; + } + Ok(_) => {} + Err(_) => break false, + } + } + }) + .await?; + assert!( + attributed, + "UI log must attribute the remote config write to peer {peer_id}" + ); + + connection.close(0u32.into(), b"done"); + accept.abort(); + receiver.close().await; + forwarder.endpoint().close().await; + Ok(()) + } + #[tokio::test] async fn serves_data_streams_after_control_handshake_on_same_connection() -> TestResult { let receiver = EndpointBuilder::test([26; 32]).bind().await?; diff --git a/services/forwarder/src/p2p/remote_config.rs b/services/forwarder/src/p2p/remote_config.rs index 628b5937..afad47e7 100644 --- a/services/forwarder/src/p2p/remote_config.rs +++ b/services/forwarder/src/p2p/remote_config.rs @@ -23,6 +23,7 @@ use std::sync::Arc; +use rt_iroh::NodeId; use rt_p2p_protocol::{ ConfigGetRequest, ConfigGetResponse, ConfigSetRequest, ConfigSetResponse, RestartRequest, RestartResponse, @@ -45,6 +46,7 @@ pub struct ForwarderRemoteConfigHandler { config_state: Arc, subsystem: Arc>, ui_tx: broadcast::Sender, + ui_logger: Arc>, restart_signal: Arc, } @@ -60,13 +62,15 @@ impl std::fmt::Debug for ForwarderRemoteConfigHandler { impl ForwarderRemoteConfigHandler { /// Builds a handler. `allow_remote_config` mirrors /// `control.allow_remote_config` and gates both capability advertisement - /// and every verb. + /// and every verb. `ui_logger` is the forwarder's shared UI logger; config + /// writes are logged there attributed to the requesting peer's node id. #[must_use] pub fn new( allow_remote_config: bool, config_state: Arc, subsystem: Arc>, ui_tx: broadcast::Sender, + ui_logger: Arc>, restart_signal: Arc, ) -> Self { Self { @@ -74,6 +78,7 @@ impl ForwarderRemoteConfigHandler { config_state, subsystem, ui_tx, + ui_logger, restart_signal, } } @@ -115,7 +120,7 @@ impl RemoteConfigHandler for ForwarderRemoteConfigHandler { }) } - fn set_config(&self, request: ConfigSetRequest) -> ConfigSetFuture<'_> { + fn set_config(&self, request: ConfigSetRequest, peer: NodeId) -> ConfigSetFuture<'_> { Box::pin(async move { if !self.allow_remote_config { return ConfigSetResponse { @@ -133,14 +138,23 @@ impl RemoteConfigHandler for ForwarderRemoteConfigHandler { ) .await { - Ok(()) => ConfigSetResponse { - request_id: request.request_id, - ok: true, - restart_needed: true, - error: String::new(), - }, + Ok(()) => { + self.ui_logger.log(format!( + "Config updated remotely by receiver {peer} (restart required)" + )); + ConfigSetResponse { + request_id: request.request_id, + ok: true, + restart_needed: true, + error: String::new(), + } + } Err(error) => { - tracing::warn!(%error, "p2p: remote config set failed"); + tracing::warn!(%peer, %error, "p2p: remote config set failed"); + self.ui_logger.log_at( + rt_ui_log::UiLogLevel::Warn, + format!("Remote config write from receiver {peer} rejected: {error}"), + ); let restart_needed = self.subsystem.lock().await.restart_needed(); ConfigSetResponse { request_id: request.request_id, @@ -209,14 +223,39 @@ mod tests { config_path: PathBuf, restart_signal: Arc, ) -> ForwarderRemoteConfigHandler { + handler_with_logger(allow, config_path, restart_signal).0 + } + + /// Like [`handler`] but also returns the buffered UI logger so tests can + /// assert on emitted UI log entries. + fn handler_with_logger( + allow: bool, + config_path: PathBuf, + restart_signal: Arc, + ) -> ( + ForwarderRemoteConfigHandler, + Arc>, + ) { let (ui_tx, _ui_rx) = broadcast::channel(16); - ForwarderRemoteConfigHandler::new( + let logger = Arc::new(rt_ui_log::UiLogger::with_buffer( + ui_tx.clone(), + |entry| ForwarderUiEvent::LogEntry { entry }, + 32, + )); + let h = ForwarderRemoteConfigHandler::new( allow, Arc::new(ConfigState::new(config_path)), Arc::new(Mutex::new(SubsystemStatus::ready())), ui_tx, + Arc::clone(&logger), restart_signal, - ) + ); + (h, logger) + } + + /// A deterministic peer node id for attributing test config writes. + fn test_peer() -> NodeId { + rt_iroh::SecretKey::from_bytes(&[42u8; 32]).public() } #[tokio::test] @@ -285,10 +324,13 @@ mod tests { let edited = serde_json::to_string(&value).unwrap(); let response = h - .set_config(ConfigSetRequest { - request_id: "s1".to_owned(), - config_json: edited, - }) + .set_config( + ConfigSetRequest { + request_id: "s1".to_owned(), + config_json: edited, + }, + test_peer(), + ) .await; assert!(response.ok, "set should succeed: {}", response.error); @@ -317,10 +359,13 @@ mod tests { .unwrap(); value["schema_version"] = serde_json::json!(2); let response = h - .set_config(ConfigSetRequest { - request_id: "s2".to_owned(), - config_json: serde_json::to_string(&value).unwrap(), - }) + .set_config( + ConfigSetRequest { + request_id: "s2".to_owned(), + config_json: serde_json::to_string(&value).unwrap(), + }, + test_peer(), + ) .await; assert!(!response.ok); @@ -339,10 +384,13 @@ mod tests { let h = handler(false, config_path, Arc::new(Notify::new())); let response = h - .set_config(ConfigSetRequest { - request_id: "s3".to_owned(), - config_json: r#"{"schema_version":1}"#.to_owned(), - }) + .set_config( + ConfigSetRequest { + request_id: "s3".to_owned(), + config_json: r#"{"schema_version":1}"#.to_owned(), + }, + test_peer(), + ) .await; assert!(!response.ok); @@ -364,10 +412,13 @@ mod tests { value["p2p"]["static_allowed_receivers"] = serde_json::json!(["ff".repeat(32)]); let response = h - .set_config(ConfigSetRequest { - request_id: "s".into(), - config_json: serde_json::to_string(&value).unwrap(), - }) + .set_config( + ConfigSetRequest { + request_id: "s".into(), + config_json: serde_json::to_string(&value).unwrap(), + }, + test_peer(), + ) .await; assert!(!response.ok); @@ -397,10 +448,13 @@ mod tests { value["auth"]["token_file"] = serde_json::json!(alternate_token_path.display().to_string()); let response = h - .set_config(ConfigSetRequest { - request_id: "s".into(), - config_json: serde_json::to_string(&value).unwrap(), - }) + .set_config( + ConfigSetRequest { + request_id: "s".into(), + config_json: serde_json::to_string(&value).unwrap(), + }, + test_peer(), + ) .await; assert!(!response.ok); @@ -428,10 +482,13 @@ mod tests { value["control"]["allow_remote_config"] = serde_json::json!(false); let response = h - .set_config(ConfigSetRequest { - request_id: "s".into(), - config_json: serde_json::to_string(&value).unwrap(), - }) + .set_config( + ConfigSetRequest { + request_id: "s".into(), + config_json: serde_json::to_string(&value).unwrap(), + }, + test_peer(), + ) .await; assert!(!response.ok); @@ -471,10 +528,13 @@ mod tests { } ]); let response = h - .set_config(ConfigSetRequest { - request_id: "s".into(), - config_json: serde_json::to_string(&value).unwrap(), - }) + .set_config( + ConfigSetRequest { + request_id: "s".into(), + config_json: serde_json::to_string(&value).unwrap(), + }, + test_peer(), + ) .await; assert!(response.ok, "set should succeed: {}", response.error); @@ -511,10 +571,13 @@ mod tests { // Only an operational change; protected sections round-trip untouched. value["display_name"] = serde_json::json!("Round Trip"); let response = h - .set_config(ConfigSetRequest { - request_id: "s".into(), - config_json: serde_json::to_string(&value).unwrap(), - }) + .set_config( + ConfigSetRequest { + request_id: "s".into(), + config_json: serde_json::to_string(&value).unwrap(), + }, + test_peer(), + ) .await; assert!( @@ -547,10 +610,13 @@ mod tests { "allow_reader_control": null, }); let response = h - .set_config(ConfigSetRequest { - request_id: "s".into(), - config_json: serde_json::to_string(&value).unwrap(), - }) + .set_config( + ConfigSetRequest { + request_id: "s".into(), + config_json: serde_json::to_string(&value).unwrap(), + }, + test_peer(), + ) .await; assert!(response.ok, "set should succeed: {}", response.error); @@ -571,10 +637,13 @@ mod tests { value.as_object_mut().unwrap().remove("control"); let response = h - .set_config(ConfigSetRequest { - request_id: "s".into(), - config_json: serde_json::to_string(&value).unwrap(), - }) + .set_config( + ConfigSetRequest { + request_id: "s".into(), + config_json: serde_json::to_string(&value).unwrap(), + }, + test_peer(), + ) .await; assert!(!response.ok); @@ -587,6 +656,72 @@ mod tests { assert!(cfg.control.allow_remote_config); } + #[tokio::test] + async fn set_success_ui_log_mentions_peer_node_id() { + let (config_path, _dir) = temp_config(""); + let (h, logger) = handler_with_logger(true, config_path, Arc::new(Notify::new())); + let mut value: serde_json::Value = serde_json::from_str( + &h.get_config(ConfigGetRequest { + request_id: "g".into(), + }) + .await + .config_json, + ) + .unwrap(); + + value["display_name"] = serde_json::json!("Attributed"); + let response = h + .set_config( + ConfigSetRequest { + request_id: "s".into(), + config_json: serde_json::to_string(&value).unwrap(), + }, + test_peer(), + ) + .await; + + assert!(response.ok, "set should succeed: {}", response.error); + let peer = test_peer().to_string(); + let entries = logger.entries(); + assert!( + entries.iter().any(|entry| entry.contains(&peer)), + "UI log must attribute the config write to peer {peer}, got {entries:?}" + ); + } + + #[tokio::test] + async fn set_rejected_ui_log_mentions_peer_node_id() { + let (config_path, _dir) = temp_config("[p2p]\nenabled = false\n"); + let (h, logger) = handler_with_logger(true, config_path, Arc::new(Notify::new())); + let mut value: serde_json::Value = serde_json::from_str( + &h.get_config(ConfigGetRequest { + request_id: "g".into(), + }) + .await + .config_json, + ) + .unwrap(); + + value["p2p"]["enabled"] = serde_json::json!(true); + let response = h + .set_config( + ConfigSetRequest { + request_id: "s".into(), + config_json: serde_json::to_string(&value).unwrap(), + }, + test_peer(), + ) + .await; + + assert!(!response.ok, "protected-section write must be rejected"); + let peer = test_peer().to_string(); + let entries = logger.entries(); + assert!( + entries.iter().any(|entry| entry.contains(&peer)), + "UI log must attribute the rejected config write to peer {peer}, got {entries:?}" + ); + } + #[cfg(unix)] #[tokio::test] async fn restart_notifies_signal_without_killing_process() { From 3ad2a0f6ab52cb986d1b01897f46aba47460dda3 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 12:38:31 -0400 Subject: [PATCH 10/38] fix(forwarder): log capability denials at warn, escape wire-controlled log fields, drop duplicate warn --- services/forwarder/src/p2p/control.rs | 20 ++++++++++++-------- services/forwarder/src/p2p/remote_config.rs | 3 ++- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/services/forwarder/src/p2p/control.rs b/services/forwarder/src/p2p/control.rs index bf306ece..e2354d6c 100644 --- a/services/forwarder/src/p2p/control.rs +++ b/services/forwarder/src/p2p/control.rs @@ -685,12 +685,12 @@ async fn run_control_loop( } Some(control_c2f::Msg::ReaderControlRequest(request)) => { if !reader_control_negotiated { - tracing::info!( + tracing::warn!( %peer, verb = "reader_control", - command = %request.command, + command = ?request.command, request_id = %request.request_id, - stream_id = %String::from_utf8_lossy(&request.stream_id), + stream_id = ?String::from_utf8_lossy(&request.stream_id), outcome = "denied", "p2p: control verb denied (capability not negotiated)" ); @@ -723,9 +723,9 @@ async fn run_control_loop( tracing::info!( %peer, verb = "reader_control", - command = %command, + command = ?command, request_id = %response.request_id, - stream_id = %String::from_utf8_lossy(&response.stream_id), + stream_id = ?String::from_utf8_lossy(&response.stream_id), outcome = if response.success { "ok" } else { "error" }, "p2p: control verb handled" ); @@ -734,7 +734,7 @@ async fn run_control_loop( } Some(control_c2f::Msg::ConfigGetRequest(request)) => { if !remote_config_negotiated { - tracing::info!( + tracing::warn!( %peer, verb = "config_get", request_id = %request.request_id, @@ -763,6 +763,10 @@ async fn run_control_loop( let config_response_tx = config_response_tx.clone(); control_tasks.spawn(async move { let response = remote_config.get_config(request).await; + // Empty config_json is also the handler's "disabled" + // signal, but the capability gate above fires first when + // remote config is disabled, so on this path emptiness + // means a real read/serialize failure. tracing::info!( %peer, verb = "config_get", @@ -782,7 +786,7 @@ async fn run_control_loop( } Some(control_c2f::Msg::ConfigSetRequest(request)) => { if !remote_config_negotiated { - tracing::info!( + tracing::warn!( %peer, verb = "config_set", request_id = %request.request_id, @@ -823,7 +827,7 @@ async fn run_control_loop( } Some(control_c2f::Msg::RestartRequest(request)) => { if !remote_config_negotiated { - tracing::info!( + tracing::warn!( %peer, verb = "restart", request_id = %request.request_id, diff --git a/services/forwarder/src/p2p/remote_config.rs b/services/forwarder/src/p2p/remote_config.rs index afad47e7..7ec6b3ba 100644 --- a/services/forwarder/src/p2p/remote_config.rs +++ b/services/forwarder/src/p2p/remote_config.rs @@ -150,7 +150,8 @@ impl RemoteConfigHandler for ForwarderRemoteConfigHandler { } } Err(error) => { - tracing::warn!(%peer, %error, "p2p: remote config set failed"); + // UiLogger::log_at also echoes to tracing at warn level, so + // no separate tracing::warn! here (it would double-log). self.ui_logger.log_at( rt_ui_log::UiLogLevel::Warn, format!("Remote config write from receiver {peer} rejected: {error}"), From cd8c5bcfffce787eff2acc92cda690609e0e98f7 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 12:49:41 -0400 Subject: [PATCH 11/38] test(e2e): assert remote [control] writes are rejected; gate remote config via local TOML edit --- scripts/e2e/run_stack.py | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/scripts/e2e/run_stack.py b/scripts/e2e/run_stack.py index c920d6b3..bb4e6388 100755 --- a/scripts/e2e/run_stack.py +++ b/scripts/e2e/run_stack.py @@ -1017,18 +1017,42 @@ def forwarder_disconnected(): results.expect_eq("remote config: updated document is readable", updated_doc.get("display_name"), "E2E Remote Config Updated") - updated_doc.setdefault("control", {})["allow_remote_config"] = False + # [control] is a protected section: a remote attempt to flip + # allow_remote_config must be rejected and must not persist. + protected_doc = json.loads(json.dumps(updated_doc)) + protected_doc.setdefault("control", {})["allow_remote_config"] = False disable_response = bridge_invoke( bridge_url, "set_forwarder_config", - {"endpoint_id": forwarder_endpoint_id, "config_json": json.dumps(updated_doc)}, + {"endpoint_id": forwarder_endpoint_id, "config_json": json.dumps(protected_doc)}, + ) + results.expect_eq("remote config protection: remote [control] flip is rejected", + disable_response.get("ok"), False) + results.check("remote config protection: error names the protected section", + "control" in (disable_response.get("error") or ""), + disable_response.get("error") or "") + recheck_response = bridge_invoke( + bridge_url, + "get_forwarder_config", + {"endpoint_id": forwarder_endpoint_id}, ) - results.expect_eq("remote config gating: disabling remote config succeeds", - disable_response.get("ok"), True) - results.expect_eq("remote config gating: disable reports restart_needed", - disable_response.get("restart_needed"), True) + recheck_doc = json.loads(recheck_response.get("config_json", "{}")) + results.expect_eq("remote config protection: allow_remote_config unchanged after rejection", + (recheck_doc.get("control") or {}).get("allow_remote_config"), True) forwarder.stop() + + # Gate remote config the supported way: a local (trusted) config edit. + # Remote writes to [control] are rejected above, so the gating scenario + # flips the flag directly in the forwarder's TOML before the restart. + config_text = forwarder_config.read_text() + if "allow_remote_config = true" not in config_text: + raise AssertionError( + f"expected 'allow_remote_config = true' in forwarder config, got:\n{config_text}" + ) + forwarder_config.write_text( + config_text.replace("allow_remote_config = true", "allow_remote_config = false") + ) forwarder2 = stack.add(make_forwarder("2")) forwarder2.start() wait_for_log(forwarder2.log_path, "p2p iroh server started", timeout=30, From 33014ea6626d38514fe25ec6977a596a7150a053 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 12:56:41 -0400 Subject: [PATCH 12/38] fix(forwarder): validate per-section config writes with the canonical loader before persisting --- services/forwarder/src/status_http.rs | 69 ++++++++++++++++++++ services/forwarder/tests/config_endpoints.rs | 13 ++++ 2 files changed, 82 insertions(+) diff --git a/services/forwarder/src/status_http.rs b/services/forwarder/src/status_http.rs index 908b1608..bdba11cd 100644 --- a/services/forwarder/src/status_http.rs +++ b/services/forwarder/src/status_http.rs @@ -1356,6 +1356,14 @@ async fn update_config_file( ) })?; + crate::config::load_config_from_str(&new_toml, &config_state.path).map_err(|e| { + ( + 400u16, + serde_json::json!({"ok": false, "error": format!("config validation failed: {e}")}) + .to_string(), + ) + })?; + write_atomic(&config_state.path, &new_toml).map_err(|e| { ( 500u16, @@ -5499,6 +5507,67 @@ target = "192.168.1.100:10000" ); } + #[tokio::test] + async fn config_p2p_section_rejects_cross_field_loader_validation_failure() { + use std::io::Write; + use tempfile::NamedTempFile; + + let token_file = NamedTempFile::new().expect("create temp token"); + let token_path = token_file.path().display().to_string().replace('\\', "/"); + std::fs::write(token_file.path(), "test-token\n").expect("write temp token"); + + let mut config_file = NamedTempFile::new().expect("create temp config"); + write!( + config_file, + r#"schema_version = 1 +[p2p] +enabled = false +max_concurrent_bidi_streams = 1 +server_url = "https://timing.example.com" +[auth] +token_file = "{token_path}" +[[readers]] +target = "192.168.1.100:10000" +"# + ) + .expect("write config"); + + let restart_signal = Arc::new(Notify::new()); + let server = StatusServer::start_with_config( + StatusConfig { + bind: "127.0.0.1:0".to_owned(), + forwarder_version: "0.2.0".to_owned(), + }, + SubsystemStatus::ready(), + Arc::new(Mutex::new(NoJournal)), + Arc::new(ConfigState::new(config_file.path().to_path_buf())), + restart_signal, + ) + .await + .expect("start status server"); + + let client = reqwest::Client::new(); + let resp = client + .post(format!("http://{}/api/v1/config/p2p", server.local_addr())) + .header("content-type", "application/json") + .body(r#"{"enabled":true}"#) + .send() + .await + .expect("post p2p config update"); + + let status = resp.status(); + let body = resp.text().await.expect("read response body"); + assert_eq!(status, StatusCode::BAD_REQUEST, "response body: {body}"); + assert!( + body.contains("config validation failed") + && body.contains("max_concurrent_bidi_streams"), + "unexpected response body: {body}" + ); + + let loaded = crate::config::load_config_from_path(config_file.path()).expect("load config"); + assert!(!loaded.p2p.enabled); + } + #[cfg(any(feature = "eink", feature = "lcd"))] #[tokio::test] async fn config_screen_endpoint_updates_screen_and_sets_restart_needed() { diff --git a/services/forwarder/tests/config_endpoints.rs b/services/forwarder/tests/config_endpoints.rs index b8e792db..bc4bf31f 100644 --- a/services/forwarder/tests/config_endpoints.rs +++ b/services/forwarder/tests/config_endpoints.rs @@ -5,6 +5,14 @@ use forwarder::status_http::{StatusConfig, StatusServer, SubsystemStatus}; use std::net::SocketAddr; use std::time::Duration; +fn write_token_fixture(path: &str) { + std::fs::write(path, "test-token\n").unwrap_or_else(|e| panic!("write token {path}: {e}")); +} + +fn write_legacy_fake_token_fixture() { + write_token_fixture("/tmp/fake-token"); +} + async fn http_get(addr: SocketAddr, path: &str) -> (u16, String) { use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; @@ -39,6 +47,8 @@ async fn http_post(addr: SocketAddr, path: &str, body: &str) -> (u16, String) { use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; + write_legacy_fake_token_fixture(); + let mut stream = TcpStream::connect(addr).await.expect("connect failed"); let request = format!( "POST {} HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", @@ -71,6 +81,8 @@ async fn http_post_split(addr: SocketAddr, path: &str, body: &str) -> (u16, Stri use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; + write_legacy_fake_token_fixture(); + let mut stream = TcpStream::connect(addr).await.expect("connect failed"); let headers = format!( "POST {} HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", @@ -544,6 +556,7 @@ target = "192.168.1.100:10000" .expect("start failed"); let addr = server.local_addr(); tokio::time::sleep(Duration::from_millis(50)).await; + write_token_fixture("/tmp/new-token"); let (status, response) = http_post( addr, From 8097fa92d7750934d8ec426ebfd219c92b736144 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 13:10:22 -0400 Subject: [PATCH 13/38] test(forwarder): make config token-file fixtures hermetic (no shared /tmp state) --- services/forwarder/src/status_http.rs | 33 +++-- services/forwarder/tests/config_endpoints.rs | 130 +++++++++++-------- 2 files changed, 103 insertions(+), 60 deletions(-) diff --git a/services/forwarder/src/status_http.rs b/services/forwarder/src/status_http.rs index bdba11cd..e67b1aca 100644 --- a/services/forwarder/src/status_http.rs +++ b/services/forwarder/src/status_http.rs @@ -3849,6 +3849,15 @@ mod tests { format!("ab{body}{lrc:02x}") } + fn temp_token_file() -> (tempfile::TempDir, String) { + let dir = tempfile::tempdir().expect("create temp token dir"); + let path = dir.path().join("fake-token"); + std::fs::write(&path, "test-token\n") + .unwrap_or_else(|e| panic!("write token {}: {e}", path.display())); + let token_path = path.display().to_string().replace('\\', "/"); + (dir, token_path) + } + #[tokio::test] async fn update_apply_sets_failed_status_when_staged_file_missing() { let server = StatusServer::start( @@ -5342,6 +5351,7 @@ mod tests { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp config"); + let (_token_dir, token_path) = temp_token_file(); write!( config_file, r#"schema_version = 1 @@ -5349,7 +5359,7 @@ display_name = "Start Line" [p2p] server_url = "https://timing.example.com" [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -5402,13 +5412,14 @@ target = "192.168.1.100:10000" use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp config"); + let (_token_dir, token_path) = temp_token_file(); write!( config_file, r#"schema_version = 1 [p2p] server_url = "https://timing.example.com" [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -5462,13 +5473,14 @@ target = "192.168.1.100:10000" use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp config"); + let (_token_dir, token_path) = temp_token_file(); write!( config_file, r#"schema_version = 1 [p2p] server_url = "https://timing.example.com" [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -5575,13 +5587,14 @@ target = "192.168.1.100:10000" use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp config"); + let (_token_dir, token_path) = temp_token_file(); write!( config_file, r#"schema_version = 1 [p2p] server_url = "https://timing.example.com" [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -5636,13 +5649,14 @@ target = "192.168.1.100:10000" use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp config"); + let (_token_dir, token_path) = temp_token_file(); write!( config_file, r#"schema_version = 1 [p2p] server_url = "https://timing.example.com" [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -7126,13 +7140,14 @@ target = "192.168.1.100:10000" use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp config"); + let (_token_dir, token_path) = temp_token_file(); write!( config_file, r#"schema_version = 1 [p2p] server_url = "https://timing.example.com" [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -7173,13 +7188,14 @@ target = "192.168.1.100:10000" use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp config"); + let (_token_dir, token_path) = temp_token_file(); write!( config_file, r#"schema_version = 1 [p2p] server_url = "https://timing.example.com" [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -7217,13 +7233,14 @@ target = "192.168.1.100:10000" use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp config"); + let (_token_dir, token_path) = temp_token_file(); write!( config_file, r#"schema_version = 1 [p2p] server_url = "https://timing.example.com" [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# diff --git a/services/forwarder/tests/config_endpoints.rs b/services/forwarder/tests/config_endpoints.rs index bc4bf31f..d641675e 100644 --- a/services/forwarder/tests/config_endpoints.rs +++ b/services/forwarder/tests/config_endpoints.rs @@ -5,12 +5,17 @@ use forwarder::status_http::{StatusConfig, StatusServer, SubsystemStatus}; use std::net::SocketAddr; use std::time::Duration; -fn write_token_fixture(path: &str) { - std::fs::write(path, "test-token\n").unwrap_or_else(|e| panic!("write token {path}: {e}")); +fn create_token_fixture(filename: &str) -> (tempfile::TempDir, String) { + let dir = tempfile::tempdir().expect("create temp token dir"); + let path = dir.path().join(filename); + std::fs::write(&path, "test-token\n") + .unwrap_or_else(|e| panic!("write token {}: {e}", path.display())); + let token_path = path.display().to_string().replace('\\', "/"); + (dir, token_path) } -fn write_legacy_fake_token_fixture() { - write_token_fixture("/tmp/fake-token"); +fn create_fake_token_fixture() -> (tempfile::TempDir, String) { + create_token_fixture("fake-token") } async fn http_get(addr: SocketAddr, path: &str) -> (u16, String) { @@ -47,8 +52,6 @@ async fn http_post(addr: SocketAddr, path: &str, body: &str) -> (u16, String) { use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; - write_legacy_fake_token_fixture(); - let mut stream = TcpStream::connect(addr).await.expect("connect failed"); let request = format!( "POST {} HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", @@ -81,8 +84,6 @@ async fn http_post_split(addr: SocketAddr, path: &str, body: &str) -> (u16, Stri use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; - write_legacy_fake_token_fixture(); - let mut stream = TcpStream::connect(addr).await.expect("connect failed"); let headers = format!( "POST {} HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", @@ -147,6 +148,7 @@ async fn get_config_returns_json() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 @@ -154,7 +156,7 @@ display_name = "Start Line" [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" @@ -202,13 +204,14 @@ async fn post_config_general_updates_display_name() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" @@ -272,13 +275,14 @@ async fn post_config_general_updates_readonly_config_via_atomic_replace() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" @@ -343,13 +347,14 @@ async fn post_config_general_preserves_file_mode_on_atomic_replace() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" @@ -407,13 +412,14 @@ async fn post_config_general_accepts_fragmented_http_body() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" @@ -470,11 +476,12 @@ async fn post_config_optional_sections_reject_non_object_payloads() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -526,11 +533,12 @@ async fn post_config_auth_updates_token_file() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_old_token_dir, old_token_path) = create_token_fixture("old-token"); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/old-token" +token_file = "{old_token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -556,14 +564,10 @@ target = "192.168.1.100:10000" .expect("start failed"); let addr = server.local_addr(); tokio::time::sleep(Duration::from_millis(50)).await; - write_token_fixture("/tmp/new-token"); + let (_new_token_dir, new_token_path) = create_token_fixture("new-token"); + let body = format!(r#"{{"token_file":"{new_token_path}"}}"#); - let (status, response) = http_post( - addr, - "/api/v1/config/auth", - r#"{"token_file":"/tmp/new-token"}"#, - ) - .await; + let (status, response) = http_post(addr, "/api/v1/config/auth", &body).await; assert_eq!(status, 200); let body = response_body(&response); let json: serde_json::Value = serde_json::from_str(body).expect("parse JSON"); @@ -584,11 +588,12 @@ async fn post_config_auth_requires_token_file() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -625,11 +630,12 @@ async fn post_config_auth_rejects_whitespace_token_file() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -666,11 +672,12 @@ async fn post_config_journal_updates_sqlite_path() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -728,11 +735,12 @@ async fn post_config_journal_rejects_out_of_range_prune_watermark() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -777,11 +785,12 @@ async fn post_config_journal_rejects_non_numeric_prune_watermark() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -822,17 +831,18 @@ target = "192.168.1.100:10000" /// Helper: start a config-editing server backed by a minimal temp config file. /// Returns the running server and the config file guard (kept to preserve the /// temp file for the duration of the test). -async fn start_config_server() -> (StatusServer, tempfile::NamedTempFile) { +async fn start_config_server() -> (StatusServer, tempfile::NamedTempFile, tempfile::TempDir) { use forwarder::status_http::ConfigState; use std::io::Write; use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -856,12 +866,12 @@ target = "192.168.1.100:10000" .await .expect("start failed"); tokio::time::sleep(Duration::from_millis(50)).await; - (server, config_file) + (server, config_file, _token_dir) } #[tokio::test] async fn post_config_p2p_updates_server_settings() { - let (server, config_file) = start_config_server().await; + let (server, config_file, _token_dir) = start_config_server().await; let addr = server.local_addr(); let (status, response) = http_post( @@ -896,7 +906,7 @@ async fn post_config_p2p_updates_server_settings() { #[tokio::test] async fn post_config_p2p_rejects_unsupported_server_url_scheme() { - let (server, _config_file) = start_config_server().await; + let (server, _config_file, _token_dir) = start_config_server().await; let addr = server.local_addr(); let (status, _response) = http_post( @@ -910,7 +920,7 @@ async fn post_config_p2p_rejects_unsupported_server_url_scheme() { #[tokio::test] async fn post_config_journal_rejects_invalid_retention_suffix() { - let (server, _config_file) = start_config_server().await; + let (server, _config_file, _token_dir) = start_config_server().await; let addr = server.local_addr(); let (status, _) = http_post(addr, "/api/v1/config/journal", r#"{"min_retention":"7x"}"#).await; @@ -922,7 +932,7 @@ async fn post_config_journal_rejects_invalid_retention_suffix() { #[tokio::test] async fn post_config_journal_rejects_min_max_inversion() { - let (server, _config_file) = start_config_server().await; + let (server, _config_file, _token_dir) = start_config_server().await; let addr = server.local_addr(); let (status, _) = http_post( @@ -936,7 +946,7 @@ async fn post_config_journal_rejects_min_max_inversion() { #[tokio::test] async fn post_config_journal_rejects_zero_emergency_rows() { - let (server, _config_file) = start_config_server().await; + let (server, _config_file, _token_dir) = start_config_server().await; let addr = server.local_addr(); let (status, _) = http_post( @@ -950,7 +960,7 @@ async fn post_config_journal_rejects_zero_emergency_rows() { #[tokio::test] async fn post_config_journal_accepts_valid_retention() { - let (server, config_file) = start_config_server().await; + let (server, config_file, _token_dir) = start_config_server().await; let addr = server.local_addr(); let (status, response) = http_post( @@ -982,11 +992,12 @@ async fn post_config_status_http_updates_bind() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -1039,11 +1050,12 @@ async fn post_config_status_http_rejects_invalid_ipv4_and_port() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -1085,11 +1097,12 @@ async fn post_config_status_http_rejects_hostname_bind() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -1131,11 +1144,12 @@ async fn post_config_status_http_rejects_ipv6_bind() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -1177,11 +1191,12 @@ async fn post_config_readers_replaces_list() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -1245,11 +1260,12 @@ async fn post_config_readers_validates_target() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -1292,11 +1308,12 @@ async fn post_config_readers_rejects_out_of_range_local_fallback_port() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -1341,11 +1358,12 @@ async fn post_config_readers_requires_at_least_one() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -1382,11 +1400,12 @@ async fn post_config_control_updates_allow_power_actions() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -1448,11 +1467,12 @@ async fn post_config_control_rejects_non_boolean_allow_power_actions() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -1500,11 +1520,12 @@ async fn post_config_control_action_restart_device_requires_allow_power_actions_ use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [control] allow_power_actions = false [[readers]] @@ -1554,11 +1575,12 @@ async fn restart_endpoint_returns_ok() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -1628,11 +1650,12 @@ async fn control_restart_service_endpoint_returns_ok() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [[readers]] target = "192.168.1.100:10000" "# @@ -1686,11 +1709,12 @@ async fn control_restart_device_requires_allow_power_actions_true() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [control] allow_power_actions = false [[readers]] @@ -1735,11 +1759,12 @@ async fn control_shutdown_device_requires_allow_power_actions_true() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [control] allow_power_actions = false [[readers]] @@ -1784,11 +1809,12 @@ async fn control_action_errors_are_written_to_ui_logs() { use tempfile::NamedTempFile; let mut config_file = NamedTempFile::new().expect("create temp file"); + let (_token_dir, token_path) = create_fake_token_fixture(); write!( config_file, r#"schema_version = 1 [auth] -token_file = "/tmp/fake-token" +token_file = "{token_path}" [control] allow_power_actions = false [[readers]] From 8a01f2274c1c47019bfebad79de7ffd9d9902187 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 13:21:23 -0400 Subject: [PATCH 14/38] fix(forwarder): retry journal appends with backoff instead of dropping consumed reads --- services/forwarder/src/lib.rs | 1 + services/forwarder/src/main.rs | 4 +- services/forwarder/src/reader_task.rs | 293 +++++++++++++++++++--- services/forwarder/src/storage/journal.rs | 10 + 4 files changed, 274 insertions(+), 34 deletions(-) diff --git a/services/forwarder/src/lib.rs b/services/forwarder/src/lib.rs index 720919df..f16a8864 100644 --- a/services/forwarder/src/lib.rs +++ b/services/forwarder/src/lib.rs @@ -10,6 +10,7 @@ pub mod p2p; pub mod pisugar_client; pub mod reader_control; pub mod reader_control_service; +pub mod reader_task; pub mod replay; pub mod status_http; pub mod storage; diff --git a/services/forwarder/src/main.rs b/services/forwarder/src/main.rs index 830c3d75..208e44b1 100644 --- a/services/forwarder/src/main.rs +++ b/services/forwarder/src/main.rs @@ -3,8 +3,6 @@ // Runtime event loop: wires together journal, local fanout, IPICO TCP readers, // the P2P endpoint, and the status HTTP server. -mod reader_task; - use forwarder::discovery::expand_target; use forwarder::local_fanout::FanoutServer; use forwarder::status_http::{ @@ -22,7 +20,7 @@ use tokio::time::{Duration, sleep}; use tracing::{error, info, warn}; // run_reader is used by main() in production; NOT cfg(test)-gated. -use reader_task::run_reader; +use forwarder::reader_task::run_reader; // --------------------------------------------------------------------------- // Helpers diff --git a/services/forwarder/src/reader_task.rs b/services/forwarder/src/reader_task.rs index 8d226624..2dd00ba0 100644 --- a/services/forwarder/src/reader_task.rs +++ b/services/forwarder/src/reader_task.rs @@ -1,9 +1,9 @@ // reader_task: IPICO reader TCP connect/read loop, journal append, local fanout. -use forwarder::local_fanout::FanoutServer; -use forwarder::status_http::{ReaderConnectionState, StatusServer}; -use forwarder::storage::journal::Journal; -use forwarder::ui_events::ForwarderUiEvent; +use crate::local_fanout::FanoutServer; +use crate::status_http::{ReaderConnectionState, StatusServer}; +use crate::storage::journal::Journal; +use crate::ui_events::ForwarderUiEvent; use ipico_core::read::ChipRead; use rt_ui_log::UiLogLevel; use std::convert::TryFrom; @@ -46,6 +46,86 @@ pub(crate) fn append_read_to_journal( .map_err(|e| JournalAppendError::Append(e.to_string())) } +fn append_retry_backoff(failed_attempts: u64) -> Duration { + match failed_attempts { + 1 => Duration::from_millis(100), + 2 => Duration::from_millis(500), + 3 => Duration::from_secs(1), + 4 => Duration::from_secs(2), + _ => Duration::from_secs(5), + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn append_with_retry( + journal: &Arc>, + stream_key: &str, + reader_timestamp: Option<&str>, + raw_frame: &[u8], + read_type: &str, + shutdown_rx: &mut watch::Receiver, + logger: &Arc>, + reader_ip: &str, +) -> Option<(i64, i64)> { + let mut failed_attempts = 0_u64; + let mut last_retry_log = tokio::time::Instant::now(); + + loop { + if *shutdown_rx.borrow() { + return None; + } + + let append_result = { + let mut journal = journal.lock().await; + append_read_to_journal( + &mut journal, + stream_key, + reader_timestamp, + raw_frame, + read_type, + ) + }; + + match append_result { + Ok(value) => { + if failed_attempts > 0 { + logger.log(format!( + "reader {reader_ip} journal append recovered after {failed_attempts} failed attempts" + )); + } + return Some(value); + } + Err(JournalAppendError::Append(error)) => { + failed_attempts += 1; + if failed_attempts == 1 { + logger.log_at( + UiLogLevel::Error, + format!("reader {reader_ip} journal append failed: {error}; retrying"), + ); + } else if last_retry_log.elapsed() >= Duration::from_secs(60) { + logger.log_at( + UiLogLevel::Warn, + format!( + "reader {reader_ip} still retrying journal append after {failed_attempts} failed attempts; last error: {error}" + ), + ); + last_retry_log = tokio::time::Instant::now(); + } + } + } + + let delay = append_retry_backoff(failed_attempts); + tokio::select! { + _ = sleep(delay) => {} + changed = shutdown_rx.changed() => { + if changed.is_ok() && *shutdown_rx.borrow() { + return None; + } + } + } + } +} + pub(crate) fn download_progress_advanced_or_started( was_downloading: &mut bool, last_download_progress: &mut u32, @@ -90,7 +170,7 @@ pub(crate) fn stall_outcome_for_download( } pub(crate) async fn fail_active_download( - tracker: &tokio::sync::Mutex, + tracker: &tokio::sync::Mutex, message: String, ) { let mut dt = tracker.lock().await; @@ -104,7 +184,7 @@ pub(crate) async fn fail_active_download( // --------------------------------------------------------------------------- #[allow(clippy::too_many_arguments)] -pub(crate) async fn run_reader( +pub async fn run_reader( reader_ip: String, reader_port: u16, fanout_addr: SocketAddr, @@ -214,12 +294,12 @@ pub(crate) async fn run_reader( // Set up control channels let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::channel::>(16); - let (control_client, control_sink) = forwarder::reader_control::ControlClient::new(cmd_tx); + let (control_client, control_sink) = crate::reader_control::ControlClient::new(cmd_tx); let control_client = Arc::new(control_client); status.register_control_client(&target_addr, control_client.clone()); let download_tracker = Arc::new(tokio::sync::Mutex::new( - forwarder::reader_control::DownloadTracker::new(), + crate::reader_control::DownloadTracker::new(), )); status.register_download_tracker(&target_addr, download_tracker.clone()); @@ -248,7 +328,7 @@ pub(crate) async fn run_reader( let poll_download_tracker = download_tracker.clone(); let poll_handle = tokio::spawn(async move { // Run initial connection sequence - let reader_info = forwarder::reader_control::run_connect_sequence(&poll_client).await; + let reader_info = crate::reader_control::run_connect_sequence(&poll_client).await; if reader_info.connect_failures == 6 { poll_logger.log_at( rt_ui_log::UiLogLevel::Error, @@ -284,7 +364,7 @@ pub(crate) async fn run_reader( loop { tokio::select! { _ = interval.tick() => { - forwarder::reader_control::run_status_poll(&poll_client, &mut info).await; + crate::reader_control::run_status_poll(&poll_client, &mut info).await; poll_status .update_reader_info_unless_disconnected(&poll_target_addr, info.clone()) .await; @@ -486,27 +566,23 @@ pub(crate) async fn run_reader( } // Write to journal. Keep status updates out of the DB lock scope. - let append_result = { - let mut j = journal.lock().await; - append_read_to_journal( - &mut j, - &target_addr, - reader_timestamp.as_deref(), - &frame_buf, - &parsed_read_type, - ) - }; - let (epoch, seq) = match append_result { - Ok(v) => v, - Err(e) => { - let JournalAppendError::Append(inner) = &e; - logger.log_at( - UiLogLevel::Error, - format!("reader {} journal error (append): {}", reader_ip, inner), - ); - disconnect_and_notify(&status, &target_addr).await; - break; - } + let Some((epoch, seq)) = append_with_retry( + &journal, + &target_addr, + reader_timestamp.as_deref(), + &frame_buf, + &parsed_read_type, + &mut shutdown_rx, + &logger, + &reader_ip, + ) + .await + else { + info!(reader_ip = %reader_ip, "reader task stopping (shutdown)"); + status.deregister_control_client(&target_addr); + status.deregister_download_tracker(&target_addr); + status.deregister_reconnect_notify(&target_addr); + return; }; debug!( @@ -560,3 +636,158 @@ pub(crate) async fn run_reader( backoff_secs = (backoff_secs * 2).min(30); } } + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + use tokio::sync::broadcast; + use tokio::time::{Duration, advance}; + + const STREAM_KEY: &str = "192.0.2.10:10000"; + const RAW_FRAME: &[u8] = b"aa000000000000000000000000000000000000000000\r\n"; + + fn test_logger() -> Arc> { + let (tx, _) = broadcast::channel(16); + Arc::new(rt_ui_log::UiLogger::with_buffer( + tx, + |entry| ForwarderUiEvent::LogEntry { entry }, + 64, + )) + } + + fn test_journal() -> (TempDir, Arc>) { + let tempdir = tempfile::tempdir().expect("create tempdir"); + let mut journal = + Journal::open(&tempdir.path().join("journal.sqlite")).expect("open journal"); + journal + .ensure_stream_state(STREAM_KEY, 1) + .expect("ensure stream"); + (tempdir, Arc::new(Mutex::new(journal))) + } + + async fn set_query_only(journal: &Arc>, on: bool) { + let journal = journal.lock().await; + journal.set_query_only(on).expect("set query_only"); + } + + fn spawn_append_with_retry( + journal: Arc>, + mut shutdown_rx: watch::Receiver, + logger: Arc>, + ) -> tokio::task::JoinHandle> { + tokio::spawn(async move { + append_with_retry( + &journal, + STREAM_KEY, + Some("123456"), + RAW_FRAME, + "tag", + &mut shutdown_rx, + &logger, + "192.0.2.10", + ) + .await + }) + } + + #[tokio::test(start_paused = true)] + async fn append_with_retry_keeps_retrying_until_shutdown() { + let (_tempdir, journal) = test_journal(); + set_query_only(&journal, true).await; + let logger = test_logger(); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + + let task = spawn_append_with_retry(journal, shutdown_rx, logger.clone()); + tokio::task::yield_now().await; + + advance(Duration::from_millis(100)).await; + tokio::task::yield_now().await; + advance(Duration::from_millis(500)).await; + tokio::task::yield_now().await; + advance(Duration::from_secs(60)).await; + tokio::task::yield_now().await; + + assert!(!task.is_finished(), "append should still be retrying"); + let entries = logger.entries(); + assert!( + entries.iter().any(|entry| entry.contains("[ERROR]") + && entry.contains("journal append failed") + && entry.contains("retrying")), + "first append failure must be logged at error level: {entries:?}" + ); + assert!( + entries.iter().any(|entry| entry.contains("[WARN]") + && entry.contains("still retrying journal append") + && entry.contains("attempt")), + "long-running retry must re-log a warn: {entries:?}" + ); + + shutdown_tx.send(true).expect("send shutdown"); + assert_eq!(task.await.expect("task join"), None); + } + + #[tokio::test(start_paused = true)] + async fn append_with_retry_logs_recovery_after_failures() { + let (_tempdir, journal) = test_journal(); + set_query_only(&journal, true).await; + let logger = test_logger(); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + + let task = spawn_append_with_retry(journal.clone(), shutdown_rx, logger.clone()); + tokio::task::yield_now().await; + advance(Duration::from_millis(100)).await; + tokio::task::yield_now().await; + advance(Duration::from_millis(500)).await; + tokio::task::yield_now().await; + + set_query_only(&journal, false).await; + advance(Duration::from_secs(1)).await; + let result = task.await.expect("task join"); + assert_eq!(result, Some((1, 1))); + drop(shutdown_tx); + + let entries = logger.entries(); + assert!( + entries.iter().any(|entry| entry.contains("[ERROR]") + && entry.contains("journal append failed") + && entry.contains("retrying")), + "first append failure must be logged at error level: {entries:?}" + ); + assert!( + entries.iter().any(|entry| entry.contains("[INFO]") + && entry.contains("journal append recovered") + && entry.contains("attempt")), + "recovery must be logged: {entries:?}" + ); + } + + #[tokio::test(start_paused = true)] + async fn append_with_retry_returns_immediately_for_healthy_journal() { + let (_tempdir, journal) = test_journal(); + let logger = test_logger(); + let (_shutdown_tx, mut shutdown_rx) = watch::channel(false); + + let result = append_with_retry( + &journal, + STREAM_KEY, + Some("123456"), + RAW_FRAME, + "tag", + &mut shutdown_rx, + &logger, + "192.0.2.10", + ) + .await; + + assert_eq!(result, Some((1, 1))); + let entries = logger.entries(); + assert!( + entries + .iter() + .all(|entry| !entry.contains("journal append failed") + && !entry.contains("journal append recovered")), + "healthy append should not log retry/recovery: {entries:?}" + ); + } +} diff --git a/services/forwarder/src/storage/journal.rs b/services/forwarder/src/storage/journal.rs index 65efd1d8..b6f6ed8c 100644 --- a/services/forwarder/src/storage/journal.rs +++ b/services/forwarder/src/storage/journal.rs @@ -132,6 +132,16 @@ impl Journal { }) } + #[cfg(test)] + pub fn set_query_only(&self, on: bool) -> Result<(), JournalError> { + self.conn.execute_batch(if on { + "PRAGMA query_only = ON" + } else { + "PRAGMA query_only = OFF" + })?; + Ok(()) + } + /// Return a shareable handle to this journal's per-stream wake registry. /// /// Subscribers clone the `Arc` and call From 912a8ff8c8ae8299fab535f56bbb30fb1028d75c Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 13:37:20 -0400 Subject: [PATCH 15/38] feat(server): self-scoped GET /forwarder/catalog for stream high-water restore --- crates/rt-server-api/src/catalog.rs | 8 + services/server/src/http/catalog.rs | 171 ++++++++++++++++++++- services/server/src/http/mod.rs | 26 +++- services/server/src/registry/forwarders.rs | 25 +++ services/server/src/registry/mod.rs | 3 +- 5 files changed, 230 insertions(+), 3 deletions(-) diff --git a/crates/rt-server-api/src/catalog.rs b/crates/rt-server-api/src/catalog.rs index d9b2e3f7..938f5c3b 100644 --- a/crates/rt-server-api/src/catalog.rs +++ b/crates/rt-server-api/src/catalog.rs @@ -22,3 +22,11 @@ pub struct ForwarderCatalogResponse { pub endpoint_id: String, pub stream_count: usize, } + +/// Wire-format response for `GET /forwarder/catalog`: the caller's own stored +/// stream catalog (per-stream epoch/next_seq high-water), used by a forwarder +/// to restore stream identity after local journal loss. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ForwarderOwnCatalogResponse { + pub streams: Vec, +} diff --git a/services/server/src/http/catalog.rs b/services/server/src/http/catalog.rs index 34ad2296..7992c225 100644 --- a/services/server/src/http/catalog.rs +++ b/services/server/src/http/catalog.rs @@ -8,12 +8,13 @@ use axum::{ }; use rt_server_api::catalog::{ ForwarderCatalogRequest, ForwarderCatalogResponse, ForwarderCatalogStream, + ForwarderOwnCatalogResponse, }; use crate::http::validate::{ MAX_ADDR_LEN, MAX_CATALOG_STREAMS, MAX_DIRECT_ADDRS, MAX_ID_LEN, MAX_NAME_LEN, check_len, }; -use crate::http::{AppState, authorize_forwarder_catalog}; +use crate::http::{AppState, authorize_forwarder_catalog, authorize_forwarder_self}; use crate::registry::{self, ForwarderCatalogStreamRecord}; /// `POST /forwarder/catalog` — M2M forwarder identity and stream catalog push. @@ -81,6 +82,46 @@ pub async fn push_catalog( } } +/// `GET /forwarder/catalog` — self-scoped read of the caller's stored stream +/// catalog (per-stream `epoch`/`next_seq` high-water). +/// +/// Authorized by the forwarder's own minted device token (any approval state, +/// the same auth boundary as the push above); the caller's endpoint id comes +/// from the token, so a forwarder can only ever see its own streams. Used to +/// restore stream identity after the forwarder loses its local journal. +pub async fn get_own_catalog(State(state): State, headers: HeaderMap) -> Response { + let endpoint_id = match authorize_forwarder_self(&state, &headers) { + Ok(endpoint_id) => endpoint_id, + Err(status) => return status.into_response(), + }; + + let streams = { + let conn = state.conn.lock().expect("registry mutex poisoned"); + match registry::list_forwarder_streams_for_endpoint(&conn, &endpoint_id) { + Ok(streams) => streams, + Err(err) => { + tracing::error!(error = %err, "failed to list forwarder catalog streams"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + } + }; + + let streams = streams + .into_iter() + .map(|stream| ForwarderCatalogStream { + stream_id: stream.stream_id, + epoch: stream.epoch, + next_seq: stream.next_seq, + }) + .collect(); + + ( + StatusCode::OK, + Json(ForwarderOwnCatalogResponse { streams }), + ) + .into_response() +} + fn validate_catalog(req: &ForwarderCatalogRequest) -> Result<(), String> { let check = |field: &'static str, value: &str, max: usize| { check_len(field, value, max).map_err(|field| format!("{field} too long")) @@ -254,6 +295,134 @@ mod tests { assert_eq!(resp.status(), StatusCode::BAD_REQUEST); } + fn get_catalog_request(token: &str) -> Request { + Request::builder() + .method("GET") + .uri("/forwarder/catalog") + .header("Authorization", format!("Bearer {token}")) + .body(Body::empty()) + .unwrap() + } + + async fn response_json(resp: axum::response::Response) -> serde_json::Value { + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + serde_json::from_slice(&bytes).unwrap() + } + + #[tokio::test] + async fn get_catalog_returns_only_the_calling_forwarders_streams() { + let state = test_state(); + let token_a = forwarder_token(&state, "ep-fwd-a"); + let token_b = forwarder_token(&state, "ep-fwd-b"); + + // Push distinct catalogs for two forwarders. + let resp = router(state.clone()) + .oneshot(catalog_request_with_body( + &token_a, + &serde_json::json!({ + "endpoint_id": "ep-fwd-a", + "display_name": "A", + "direct_addrs": [], + "streams": [ + {"stream_id": "10.0.0.5:10000", "epoch": 3, "next_seq": 4200}, + {"stream_id": "10.0.0.6:10000", "epoch": 1, "next_seq": 7} + ] + }), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let resp = router(state.clone()) + .oneshot(catalog_request_with_body( + &token_b, + &serde_json::json!({ + "endpoint_id": "ep-fwd-b", + "display_name": "B", + "direct_addrs": [], + "streams": [ + {"stream_id": "10.0.1.9:10000", "epoch": 2, "next_seq": 99} + ] + }), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + // Forwarder A sees exactly its own pushed streams. + let resp = router(state.clone()) + .oneshot(get_catalog_request(&token_a)) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = response_json(resp).await; + let streams = body["streams"].as_array().unwrap(); + assert_eq!(streams.len(), 2); + assert_eq!(streams[0]["stream_id"], "10.0.0.5:10000"); + assert_eq!(streams[0]["epoch"], 3); + assert_eq!(streams[0]["next_seq"], 4200); + assert_eq!(streams[1]["stream_id"], "10.0.0.6:10000"); + + // Forwarder B sees only its own stream, not A's. + let resp = router(state) + .oneshot(get_catalog_request(&token_b)) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = response_json(resp).await; + let streams = body["streams"].as_array().unwrap(); + assert_eq!(streams.len(), 1); + assert_eq!(streams[0]["stream_id"], "10.0.1.9:10000"); + assert_eq!(streams[0]["epoch"], 2); + assert_eq!(streams[0]["next_seq"], 99); + } + + #[tokio::test] + async fn get_catalog_before_any_push_returns_empty_streams() { + let state = test_state(); + let token = forwarder_token(&state, "ep-fwd"); + let resp = router(state) + .oneshot(get_catalog_request(&token)) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let body = response_json(resp).await; + assert_eq!(body["streams"].as_array().unwrap().len(), 0); + } + + #[tokio::test] + async fn get_catalog_rejects_receiver_token_and_missing_auth() { + let state = test_state(); + let rx_token = { + let conn = state.conn.lock().unwrap(); + crate::registry::register_device_minted( + &conn, + "ep-receiver", + crate::registry::DeviceKind::Receiver, + ) + .unwrap() + .device_token + }; + let resp = router(state.clone()) + .oneshot(get_catalog_request(&rx_token)) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); + + let resp = router(state) + .oneshot( + Request::builder() + .method("GET") + .uri("/forwarder/catalog") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); + } + #[tokio::test] async fn catalog_rejects_forwarder_token_for_mismatched_endpoint() { let state = test_state(); diff --git a/services/server/src/http/mod.rs b/services/server/src/http/mod.rs index 232335e9..cee836ba 100644 --- a/services/server/src/http/mod.rs +++ b/services/server/src/http/mod.rs @@ -140,6 +140,27 @@ pub(crate) fn authorize_forwarder_catalog( } } +/// Authorize a self-scoped forwarder request, resolving the caller's endpoint +/// id from its own minted device token (any approval state, matching the +/// catalog-push posture). Receiver tokens and unknown bearers are rejected. +pub(crate) fn authorize_forwarder_self( + state: &AppState, + headers: &HeaderMap, +) -> Result { + let Some(raw) = register::bearer_token(headers) else { + return Err(StatusCode::UNAUTHORIZED); + }; + let conn = state.conn.lock().expect("registry mutex poisoned"); + match registry::authenticate_device(&conn, raw) { + Ok(Some(record)) if record.device_kind == DeviceKind::Forwarder => Ok(record.endpoint_id), + Ok(_) => Err(StatusCode::UNAUTHORIZED), + Err(err) => { + tracing::error!(error = %err, "device authentication failed"); + Err(StatusCode::INTERNAL_SERVER_ERROR) + } + } +} + /// Build the server HTTP router. /// /// Routes are grouped by the auth posture documented in [`status`]: @@ -170,7 +191,10 @@ pub fn router(state: AppState) -> Router { ) // M2M/device endpoints — per-device minted-token auth. .route("/register", post(register::register)) - .route("/forwarder/catalog", post(catalog::push_catalog)) + .route( + "/forwarder/catalog", + post(catalog::push_catalog).get(catalog::get_own_catalog), + ) .route("/announcer/rows", post(announcer::push_row)) .route("/announcer/takeover", post(announcer::takeover)) .route("/allowlist/receivers", get(allowlist::receiver_allowlist)) diff --git a/services/server/src/registry/forwarders.rs b/services/server/src/registry/forwarders.rs index 8079b0b7..e419366e 100644 --- a/services/server/src/registry/forwarders.rs +++ b/services/server/src/registry/forwarders.rs @@ -209,6 +209,31 @@ pub fn list_approved_forwarders_with_streams( Ok(result) } +/// List one forwarder's stored stream catalog rows, ordered by stream id. +/// +/// Backs the self-scoped `GET /forwarder/catalog` endpoint: the caller's +/// per-stream `epoch`/`next_seq` high-water, used to restore stream identity +/// after the forwarder loses its local journal. +pub fn list_forwarder_streams_for_endpoint( + conn: &Connection, + endpoint_id: &str, +) -> rusqlite::Result> { + let mut stmt = conn.prepare( + "SELECT stream_id, epoch, next_seq + FROM forwarder_streams + WHERE endpoint_id = ?1 + ORDER BY stream_id", + )?; + let rows = stmt.query_map([endpoint_id], |row| { + Ok(ForwarderCatalogStreamRecord { + stream_id: row.get(0)?, + epoch: sql_i64_to_u64(row.get(1)?, 1)?, + next_seq: sql_i64_to_u64(row.get(2)?, 2)?, + }) + })?; + rows.collect() +} + /// List the backup forwarder stream catalog rows, ordered by stream id. pub fn list_forwarder_streams(conn: &Connection) -> rusqlite::Result> { let mut stmt = conn.prepare( diff --git a/services/server/src/registry/mod.rs b/services/server/src/registry/mod.rs index 023791ea..dbee4308 100644 --- a/services/server/src/registry/mod.rs +++ b/services/server/src/registry/mod.rs @@ -30,7 +30,8 @@ pub use enrollment::{ pub use forwarders::{ ApprovedForwarderWithStreams, DiscoveredForwarderStream, ForwarderCatalogStreamRecord, ForwarderRecord, ForwarderStreamRecord, list_approved_forwarders_with_streams, - list_forwarder_streams, list_forwarders, upsert_forwarder_catalog, + list_forwarder_streams, list_forwarder_streams_for_endpoint, list_forwarders, + upsert_forwarder_catalog, }; pub use schema::migrate; pub use tokens::{authenticate_device, hash_token, verify_token}; From 192d90c908db09df39dc2ea76cc879a9b21b6642 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 13:48:12 -0400 Subject: [PATCH 16/38] fix(forwarder): restore stream epoch/next_seq from server registry after journal loss --- crates/rt-server-api/src/catalog.rs | 2 +- services/forwarder/src/main.rs | 93 +++++++ services/forwarder/src/p2p/allowlist.rs | 22 ++ services/forwarder/src/p2p/mod.rs | 13 +- services/forwarder/src/reader_task.rs | 6 +- services/forwarder/src/storage/journal.rs | 6 +- services/forwarder/src/storage/mod.rs | 1 + services/forwarder/src/storage/restore.rs | 238 +++++++++++++++++ .../tests/stream_identity_restore.rs | 252 ++++++++++++++++++ 9 files changed, 628 insertions(+), 5 deletions(-) create mode 100644 services/forwarder/src/storage/restore.rs create mode 100644 services/forwarder/tests/stream_identity_restore.rs diff --git a/crates/rt-server-api/src/catalog.rs b/crates/rt-server-api/src/catalog.rs index 938f5c3b..43169df9 100644 --- a/crates/rt-server-api/src/catalog.rs +++ b/crates/rt-server-api/src/catalog.rs @@ -24,7 +24,7 @@ pub struct ForwarderCatalogResponse { } /// Wire-format response for `GET /forwarder/catalog`: the caller's own stored -/// stream catalog (per-stream epoch/next_seq high-water), used by a forwarder +/// stream catalog (per-stream `epoch`/`next_seq` high-water), used by a forwarder /// to restore stream identity after local journal loss. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ForwarderOwnCatalogResponse { diff --git a/services/forwarder/src/main.rs b/services/forwarder/src/main.rs index 208e44b1..8c62e8bc 100644 --- a/services/forwarder/src/main.rs +++ b/services/forwarder/src/main.rs @@ -218,6 +218,99 @@ async fn main() { std::process::exit(1); } + // Restore stream identity for any journal-missing stream keys BEFORE + // `start_forwarder_p2p` (which idempotently seeds advertised streams at + // seq 1 and would pre-empt a restore) and before reader tasks spawn. + // Receivers dedup on (stream_id, seq); if the journal was lost, restarting + // an existing stream key at seq 1 would make them silently discard reads. + // See `forwarder::storage::restore` for the high-water + slack rationale. + { + use forwarder::storage::restore::{ + RegistryFetch, RegistryStreamRecord, fetch_registry_snapshot_with_retries, + restore_streams_at_startup, + }; + + let stream_keys: Vec = all_readers.iter().map(|(addr, _)| addr.clone()).collect(); + let any_missing = { + let j = journal.lock().await; + stream_keys + .iter() + .any(|key| !j.stream_exists(key).unwrap_or(false)) + }; + if any_missing { + let fetch = if cfg.p2p.enabled && cfg.p2p.server_url.is_some() { + // Reuse the persisted minted device token (it lives at a + // different path than the journal, so it typically survives a + // journal-loss reboot). Never bootstrap here: minting needs the + // P2P endpoint id, which does not exist yet. A missing token is + // treated as "registry unavailable" (loud fallback); on a true + // first boot the journal is expected to be empty anyway. + let token_path = forwarder::p2p::device_token_path(&cfg.p2p); + match forwarder::p2p::read_device_token(&token_path) { + Ok(Some(device_token)) => { + let client = forwarder::p2p::ServerCatalogClient::with_timeout( + cfg.p2p.server_url.clone().unwrap_or_default(), + device_token, + Duration::from_secs(cfg.p2p.allowlist_request_timeout_secs), + ); + fetch_registry_snapshot_with_retries( + || { + let client = client.clone(); + async move { + client.fetch_own_catalog().await.map(|streams| { + streams + .into_iter() + .map(|s| RegistryStreamRecord { + stream_id: s.stream_id, + epoch: s.epoch, + next_seq: s.next_seq, + }) + .collect() + }) + } + }, + 3, + Duration::from_secs(5), + ) + .await + } + Ok(None) => { + warn!( + path = %token_path.display(), + "no persisted device token; cannot fetch registry high-water for stream restore" + ); + RegistryFetch::Unavailable + } + Err(e) => { + warn!( + path = %token_path.display(), + error = %e, + "failed to read device token for stream restore" + ); + RegistryFetch::Unavailable + } + } + } else { + RegistryFetch::NotConfigured + }; + + let restore_result = { + let mut j = journal.lock().await; + restore_streams_at_startup(&mut j, &stream_keys, &fetch, Some(logger.as_ref())) + }; + if let Err(e) = restore_result { + // Reader tasks retry `ensure_stream_state` as a safety net, so + // local capture still proceeds; but a failed restore means a + // previously forwarded stream key may restart at seq 1. + error!(error = %e, "startup stream identity restore failed"); + logger.log_at( + UiLogLevel::Error, + format!("stream identity restore failed: {e}"), + ); + } + } + } + // Seed historical totals once at startup to avoid per-request DB counting. for (reader_addr, _) in &all_readers { let total = { diff --git a/services/forwarder/src/p2p/allowlist.rs b/services/forwarder/src/p2p/allowlist.rs index 51a73ffd..329d6cbd 100644 --- a/services/forwarder/src/p2p/allowlist.rs +++ b/services/forwarder/src/p2p/allowlist.rs @@ -34,6 +34,7 @@ use std::time::Duration; use rt_iroh::{Connection, EndpointId}; pub use rt_server_api::allowlist::ReceiverAllowListResponse as ReceiverAllowListUpdate; +use rt_server_api::catalog::ForwarderOwnCatalogResponse; pub use rt_server_api::catalog::{ ForwarderCatalogRequest as ForwarderCatalog, ForwarderCatalogStream, }; @@ -416,6 +417,27 @@ impl ServerCatalogClient { .ok_or(CatalogPushError::MissingMintedToken) } + /// Fetches this forwarder's own stored stream catalog from the server + /// (`GET /forwarder/catalog`, self-scoped by the device token). + /// + /// Used at startup to restore per-stream epoch/next_seq high-water after + /// local journal loss. An older server without this route returns 404, + /// which surfaces as an `Err` here — callers treat any error as + /// "registry unavailable". + pub async fn fetch_own_catalog(&self) -> Result, CatalogPushError> { + let url = format!("{}/forwarder/catalog", self.base_url); + let response = self + .http + .get(url) + .bearer_auth(self.bearer_token.as_ref()) + .send() + .await? + .error_for_status()? + .json::() + .await?; + Ok(response.streams) + } + /// Pushes this forwarder's latest identity and stream catalog. pub async fn push_catalog(&self, catalog: &ForwarderCatalog) -> Result<(), CatalogPushError> { let url = format!("{}/forwarder/catalog", self.base_url); diff --git a/services/forwarder/src/p2p/mod.rs b/services/forwarder/src/p2p/mod.rs index f98e65f2..502b2dcc 100644 --- a/services/forwarder/src/p2p/mod.rs +++ b/services/forwarder/src/p2p/mod.rs @@ -122,6 +122,10 @@ pub async fn start_forwarder_p2p( // this a freshly configured reader could be advertised and subscribed before // its reader task has appended anything, and the subscription would fail on // missing stream state instead of returning SubscribeOk/CaughtUp. + // + // NOTE: startup stream-identity restore (main.rs) runs BEFORE this, so this + // idempotent seq-1 seeding never pre-empts a registry high-water restore + // for a journal-lost stream. { let mut journal = journal.lock().await; for stream in reader_streams { @@ -348,7 +352,10 @@ fn server_credentials(config: &P2pConfig) -> Result, P2 /// Writable path for the minted per-device token: the configured /// `device_token_file`, else a `p2p-device-token` sibling of the secret-key path. -fn device_token_path(config: &P2pConfig) -> PathBuf { +/// +/// `pub` so startup stream-identity restore (main.rs) can read the persisted +/// token before the P2P endpoint exists. +pub fn device_token_path(config: &P2pConfig) -> PathBuf { if let Some(path) = &config.device_token_file { return PathBuf::from(path); } @@ -395,7 +402,9 @@ async fn resolve_device_token( } } -fn read_device_token(path: &Path) -> Result, P2pStartError> { +/// Read the persisted minted device token, if any. `pub` for startup +/// stream-identity restore; it never bootstraps (that needs the endpoint id). +pub fn read_device_token(path: &Path) -> Result, P2pStartError> { match std::fs::read_to_string(path) { Ok(contents) => { let trimmed = contents.trim(); diff --git a/services/forwarder/src/reader_task.rs b/services/forwarder/src/reader_task.rs index 2dd00ba0..4bf7a99e 100644 --- a/services/forwarder/src/reader_task.rs +++ b/services/forwarder/src/reader_task.rs @@ -258,7 +258,11 @@ pub async fn run_reader( .update_reader_state(&target_addr, ReaderConnectionState::Connected) .await; - // Ensure journal has stream state for this reader (idempotent) + // Ensure journal has stream state for this reader (idempotent). + // Startup stream-identity restore (main.rs, before reader tasks spawn) + // already seeded every configured stream key — possibly from the server + // registry high-water — so this is only a cheap safety net for stream + // keys startup did not seed. Do not remove it. { let mut j = journal.lock().await; let epoch = 1_i64; diff --git a/services/forwarder/src/storage/journal.rs b/services/forwarder/src/storage/journal.rs index b6f6ed8c..7e77af8c 100644 --- a/services/forwarder/src/storage/journal.rs +++ b/services/forwarder/src/storage/journal.rs @@ -718,7 +718,11 @@ impl Journal { current_epoch(&self.conn, stream_key) } - fn stream_exists(&self, stream_id: &str) -> Result { + /// Whether the journal has any stream state for `stream_id`. + /// + /// Startup restore uses this to detect journal loss for configured reader + /// stream keys before reader tasks or the P2P catalog seed them. + pub fn stream_exists(&self, stream_id: &str) -> Result { self.conn .query_row( "SELECT EXISTS(SELECT 1 FROM streams WHERE stream_id = ?1)", diff --git a/services/forwarder/src/storage/mod.rs b/services/forwarder/src/storage/mod.rs index 9bb39939..d6c4d445 100644 --- a/services/forwarder/src/storage/mod.rs +++ b/services/forwarder/src/storage/mod.rs @@ -4,4 +4,5 @@ pub mod journal; pub mod migrations; +pub mod restore; pub mod wake; diff --git a/services/forwarder/src/storage/restore.rs b/services/forwarder/src/storage/restore.rs new file mode 100644 index 00000000..2464523d --- /dev/null +++ b/services/forwarder/src/storage/restore.rs @@ -0,0 +1,238 @@ +//! Startup stream-identity restore from the server registry high-water. +//! +//! Receivers deduplicate durable events by `(stream_id, seq)`, and the stream +//! id is the reader network address (`ip:port`). If the local journal file is +//! lost/recreated, the same stream key would restart at seq 1 and receivers +//! would silently discard the new reads as duplicates. Before reader tasks (or +//! the P2P catalog seeding) touch stream state, startup checks each enabled +//! stream key and — for keys missing from the journal — restores +//! `epoch`/`next_seq` from the server's stored catalog +//! (`GET /forwarder/catalog`), with fixed slack added on top. +//! +//! Known design gap (accepted): with no coordination server configured there +//! is no durable high-water source, so journal loss on a server-less +//! deployment still risks seq reuse — the loud log is the mitigation. Even +//! with a server, the restored high-water can lag the lost journal by up to +//! one catalog-push interval; [`RESTORE_SEQ_SLACK`] is the mitigation for +//! that. + +use std::time::Duration; + +use rt_ui_log::{UiLogLevel, UiLogger}; + +use crate::storage::journal::{Journal, JournalError, RegistryStreamRestore}; +use crate::ui_events::ForwarderUiEvent; + +/// Fixed slack added on top of the server-restored `next_seq`. +/// +/// The server snapshot can lag the lost journal by up to one catalog-push +/// interval (30s), so the true high-water may be above the restored value. +/// Sequence *gaps* are benign under at-least-once delivery with +/// `(stream_id, seq)` dedup (receivers already tolerate gaps from retention +/// pruning); sequence *reuse* silently drops reads. So we always over-shoot. +pub const RESTORE_SEQ_SLACK: i64 = 100_000; + +/// One stream record from the server's stored catalog for this forwarder. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RegistryStreamRecord { + pub stream_id: String, + pub epoch: u64, + pub next_seq: u64, +} + +/// Outcome of trying to fetch this forwarder's registry catalog snapshot. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RegistryFetch { + /// The server responded with this forwarder's stored catalog (possibly + /// empty). Streams absent from the snapshot are first boots. + Snapshot(Vec), + /// No server is configured — there is no durable high-water source at all + /// (accepted design gap; see module docs). + NotConfigured, + /// The server was unreachable or errored after bounded retries. This + /// includes 404 from an older server without `GET /forwarder/catalog` and + /// a missing device token. + Unavailable, +} + +/// Per-stream decision made by [`restore_streams_at_startup`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum StreamRestoreOutcome { + /// The journal already has state for this stream; nothing to do. + Existing, + /// Missing locally but present in the registry snapshot: seeded with the + /// server high-water plus [`RESTORE_SEQ_SLACK`]. + Restored { epoch: i64, next_seq: i64 }, + /// Missing locally and absent from a successfully fetched snapshot: + /// expected first boot, seeded at seq 1. + SeededFirstBoot, + /// Missing locally with no usable registry data: seeded at seq 1 with a + /// loud warning — if this host previously forwarded this stream, receiver + /// dedup may silently discard the new reads. + SeededWithoutRegistry, +} + +/// Fetch the registry snapshot with bounded retries. +/// +/// Calls `fetch` up to `attempts` times, sleeping `retry_delay` between +/// attempts (injectable so tests run instantly). Returns +/// [`RegistryFetch::Unavailable`] once every attempt has failed; never blocks +/// startup indefinitely. +pub async fn fetch_registry_snapshot_with_retries( + fetch: F, + attempts: u32, + retry_delay: Duration, +) -> RegistryFetch +where + F: Fn() -> Fut, + Fut: Future, E>>, + E: std::fmt::Display, +{ + for attempt in 1..=attempts { + match fetch().await { + Ok(streams) => return RegistryFetch::Snapshot(streams), + Err(error) => { + tracing::warn!( + %error, + attempt, + attempts, + "registry catalog fetch for stream restore failed" + ); + } + } + if attempt < attempts { + tokio::time::sleep(retry_delay).await; + } + } + RegistryFetch::Unavailable +} + +/// Seed journal stream state for every `stream_keys` entry missing from the +/// journal, restoring epoch/next_seq from `fetch` where available. +/// +/// Runs BEFORE reader tasks spawn and before `start_forwarder_p2p` (whose +/// idempotent `ensure_stream_state(stream, 1)` seeding would otherwise +/// pre-empt a restore with seq 1). Existing streams are never touched. +/// +/// Case (c) (`Unavailable`) emits an `error!` plus a UI log entry via +/// `logger`; case (b) (snapshot without the stream) is an expected first boot +/// and logs at info. +pub fn restore_streams_at_startup( + journal: &mut Journal, + stream_keys: &[String], + fetch: &RegistryFetch, + logger: Option<&UiLogger>, +) -> Result, JournalError> { + let mut outcomes = Vec::with_capacity(stream_keys.len()); + for key in stream_keys { + let outcome = restore_one_stream(journal, key, fetch, logger)?; + outcomes.push((key.clone(), outcome)); + } + Ok(outcomes) +} + +fn restore_one_stream( + journal: &mut Journal, + key: &str, + fetch: &RegistryFetch, + logger: Option<&UiLogger>, +) -> Result { + if journal.stream_exists(key)? { + return Ok(StreamRestoreOutcome::Existing); + } + + if let RegistryFetch::Snapshot(records) = fetch + && let Some(record) = records.iter().find(|r| r.stream_id == key) + { + // The server validates epoch/next_seq fit in i64 on push; clamp + // defensively (a floor of 1 satisfies the journal's seed validation, + // saturation keeps a corrupt huge value from wrapping). + let epoch = i64::try_from(record.epoch).unwrap_or(i64::MAX).max(1); + let next_seq = i64::try_from(record.next_seq) + .unwrap_or(i64::MAX) + .max(1) + .saturating_add(RESTORE_SEQ_SLACK); + journal.ensure_stream_after_startup( + key, + Some(key), + key, + 1, + Some(RegistryStreamRestore { + stream_id: key, + epoch, + next_seq, + }), + )?; + tracing::info!( + stream = %key, + epoch, + next_seq, + registry_next_seq = record.next_seq, + slack = RESTORE_SEQ_SLACK, + "restored stream identity from server registry high-water" + ); + if let Some(logger) = logger { + logger.log(format!( + "stream {key}: restored epoch {epoch} / next seq {next_seq} from server registry" + )); + } + return Ok(StreamRestoreOutcome::Restored { epoch, next_seq }); + } + + // No registry record for this stream. `ensure_stream_after_startup` errors + // when prior state is missing, no restore is provided, and prior_stream_id + // equals new_stream_id — the stream key IS the reader address here, so the + // fallback paths seed via plain `ensure_stream_state` instead. + journal.ensure_stream_state(key, 1)?; + + match fetch { + RegistryFetch::Snapshot(_) => { + tracing::info!( + stream = %key, + "no server registry record for stream; seeding fresh at seq 1 (expected first boot)" + ); + if let Some(logger) = logger { + logger.log(format!("stream {key}: first boot, starting at seq 1")); + } + Ok(StreamRestoreOutcome::SeededFirstBoot) + } + RegistryFetch::NotConfigured => { + tracing::warn!( + stream = %key, + "no coordination server configured: stream seeded at seq 1 with no durable \ + high-water source; if this host previously forwarded this stream, receiver \ + dedup may silently discard its reads" + ); + if let Some(logger) = logger { + logger.log_at( + UiLogLevel::Warn, + format!( + "stream {key}: seeded at seq 1 without a server high-water source; if \ + this host previously forwarded this stream, receivers may discard its \ + reads as duplicates" + ), + ); + } + Ok(StreamRestoreOutcome::SeededWithoutRegistry) + } + RegistryFetch::Unavailable => { + tracing::error!( + stream = %key, + "server registry unavailable during stream restore: stream seeded at seq 1; if \ + this host previously forwarded this stream, receiver dedup may silently \ + discard its reads" + ); + if let Some(logger) = logger { + logger.log_at( + UiLogLevel::Error, + format!( + "stream {key}: server registry unavailable, seeded at seq 1 — if this \ + host previously forwarded this stream, receivers may silently discard \ + its reads as duplicates" + ), + ); + } + Ok(StreamRestoreOutcome::SeededWithoutRegistry) + } + } +} diff --git a/services/forwarder/tests/stream_identity_restore.rs b/services/forwarder/tests/stream_identity_restore.rs new file mode 100644 index 00000000..5de2b37c --- /dev/null +++ b/services/forwarder/tests/stream_identity_restore.rs @@ -0,0 +1,252 @@ +//! Startup stream-identity restore after journal loss. +//! +//! If the journal file is lost/recreated, the same stream key (reader network +//! address) must NOT restart at seq 1: receivers dedup on `(stream_id, seq)` +//! and would silently discard the new reads as duplicates. Startup restores +//! epoch/next_seq from the server registry high-water (plus fixed slack), and +//! falls back to seq 1 — loudly when the registry was unavailable. + +use std::sync::Arc; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::time::Duration; + +use forwarder::storage::journal::Journal; +use forwarder::storage::restore::{ + RESTORE_SEQ_SLACK, RegistryFetch, RegistryStreamRecord, StreamRestoreOutcome, + fetch_registry_snapshot_with_retries, restore_streams_at_startup, +}; +use forwarder::ui_events::ForwarderUiEvent; + +const STREAM_KEY: &str = "10.0.0.5:10000"; + +fn test_logger() -> Arc> { + let (tx, _) = tokio::sync::broadcast::channel(64); + Arc::new(rt_ui_log::UiLogger::with_buffer( + tx, + |entry| ForwarderUiEvent::LogEntry { entry }, + 100, + )) +} + +/// Simulate journal loss: journal with appended events is deleted and a fresh +/// one is created at the same path (same stream key). +fn lose_journal(dir: &tempfile::TempDir) -> (Journal, i64) { + let path = dir.path().join("journal.sqlite"); + let mut journal = Journal::open(&path).unwrap(); + journal.ensure_stream_state(STREAM_KEY, 1).unwrap(); + let mut last_seq = 0; + for _ in 0..5 { + let (_, seq) = journal + .append_read(STREAM_KEY, None, b"aa0000000000000000000000000000", "RAW") + .unwrap(); + last_seq = seq; + } + assert_eq!(last_seq, 5, "expected seq to have advanced"); + drop(journal); + + // Delete the journal (plus SQLite sidecar files) — the loss scenario. + for suffix in ["", "-wal", "-shm"] { + let mut os = path.clone().into_os_string(); + os.push(suffix); + let _ = std::fs::remove_file(std::path::PathBuf::from(os)); + } + + let journal = Journal::open(&path).unwrap(); + (journal, last_seq + 1) // pre-loss next_seq +} + +/// Case (a): the server has a record for the stream — restore continues from +/// the registry high-water plus slack, never back at seq 1. +#[test] +fn restore_continues_seq_from_registry_high_water_after_journal_loss() { + let dir = tempfile::tempdir().unwrap(); + let (mut journal, pre_loss_next_seq) = lose_journal(&dir); + + let fetch = RegistryFetch::Snapshot(vec![RegistryStreamRecord { + stream_id: STREAM_KEY.to_owned(), + epoch: 3, + next_seq: u64::try_from(pre_loss_next_seq).unwrap(), + }]); + let logger = test_logger(); + let outcomes = restore_streams_at_startup( + &mut journal, + &[STREAM_KEY.to_owned()], + &fetch, + Some(&logger), + ) + .unwrap(); + + assert_eq!(outcomes.len(), 1); + assert_eq!(outcomes[0].0, STREAM_KEY); + assert!( + matches!(outcomes[0].1, StreamRestoreOutcome::Restored { .. }), + "expected registry restore, got {:?}", + outcomes[0].1 + ); + + let (epoch, seq) = journal + .append_read(STREAM_KEY, None, b"aa0000000000000000000000000000", "RAW") + .unwrap(); + assert_eq!(epoch, 3, "restored epoch must be preserved"); + assert_eq!( + seq, + pre_loss_next_seq + RESTORE_SEQ_SLACK, + "append must continue from restored next_seq + slack, not 1" + ); +} + +/// Case (b): server reachable but has no record for this stream — expected +/// first boot; seed seq 1 without any error-level UI log. +#[test] +fn restore_seeds_seq_one_when_server_has_no_record() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("journal.sqlite"); + let mut journal = Journal::open(&path).unwrap(); + + let fetch = RegistryFetch::Snapshot(vec![]); + let logger = test_logger(); + let outcomes = restore_streams_at_startup( + &mut journal, + &[STREAM_KEY.to_owned()], + &fetch, + Some(&logger), + ) + .unwrap(); + + assert_eq!(outcomes[0].1, StreamRestoreOutcome::SeededFirstBoot); + let (_, seq) = journal + .append_read(STREAM_KEY, None, b"aa0000000000000000000000000000", "RAW") + .unwrap(); + assert_eq!(seq, 1); + assert!( + !logger.entries().iter().any(|e| e.contains("[ERROR]")), + "first boot must not emit an error UI log: {:?}", + logger.entries() + ); +} + +/// Case (c): registry unavailable (unreachable, errored, or 404 from an older +/// server) — seed seq 1 but emit a loud error UI log warning that receiver +/// dedup may discard reads. +#[test] +fn restore_seeds_seq_one_and_logs_error_when_registry_unavailable() { + let dir = tempfile::tempdir().unwrap(); + let (mut journal, _) = lose_journal(&dir); + + let logger = test_logger(); + let outcomes = restore_streams_at_startup( + &mut journal, + &[STREAM_KEY.to_owned()], + &RegistryFetch::Unavailable, + Some(&logger), + ) + .unwrap(); + + assert_eq!(outcomes[0].1, StreamRestoreOutcome::SeededWithoutRegistry); + let (_, seq) = journal + .append_read(STREAM_KEY, None, b"aa0000000000000000000000000000", "RAW") + .unwrap(); + assert_eq!(seq, 1); + + let entries = logger.entries(); + let error_entry = entries + .iter() + .find(|e| e.contains("[ERROR]") && e.contains(STREAM_KEY)); + assert!( + error_entry.is_some_and(|e| e.contains("discard")), + "expected loud error UI log about receiver dedup discarding reads, got {entries:?}" + ); +} + +/// Streams that still have local journal state are untouched, even when the +/// registry snapshot carries a (stale, lower) high-water for them. +#[test] +fn restore_leaves_existing_streams_untouched() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("journal.sqlite"); + let mut journal = Journal::open(&path).unwrap(); + journal.ensure_stream_state(STREAM_KEY, 1).unwrap(); + for _ in 0..3 { + journal + .append_read(STREAM_KEY, None, b"aa0000000000000000000000000000", "RAW") + .unwrap(); + } + + let fetch = RegistryFetch::Snapshot(vec![RegistryStreamRecord { + stream_id: STREAM_KEY.to_owned(), + epoch: 9, + next_seq: 2, + }]); + let outcomes = + restore_streams_at_startup(&mut journal, &[STREAM_KEY.to_owned()], &fetch, None).unwrap(); + + assert_eq!(outcomes[0].1, StreamRestoreOutcome::Existing); + let (epoch, seq) = journal + .append_read(STREAM_KEY, None, b"aa0000000000000000000000000000", "RAW") + .unwrap(); + assert_eq!(epoch, 1); + assert_eq!(seq, 4, "existing local state must win over the snapshot"); +} + +/// The fetch wrapper retries a bounded number of times and then reports +/// `Unavailable`; it never retries after a success. +#[tokio::test] +async fn fetch_retries_are_bounded_then_unavailable() { + let calls = Arc::new(AtomicU32::new(0)); + let calls_in_fetch = Arc::clone(&calls); + let fetch = fetch_registry_snapshot_with_retries( + move || { + let calls = Arc::clone(&calls_in_fetch); + async move { + calls.fetch_add(1, Ordering::SeqCst); + Err::, String>("connection refused".to_owned()) + } + }, + 3, + Duration::ZERO, + ) + .await; + + assert_eq!(fetch, RegistryFetch::Unavailable); + assert_eq!( + calls.load(Ordering::SeqCst), + 3, + "exactly 3 bounded attempts" + ); +} + +/// A success on a later attempt yields the snapshot (no further retries). +#[tokio::test] +async fn fetch_succeeds_after_transient_failures() { + let calls = Arc::new(AtomicU32::new(0)); + let calls_in_fetch = Arc::clone(&calls); + let fetch = fetch_registry_snapshot_with_retries( + move || { + let calls = Arc::clone(&calls_in_fetch); + async move { + if calls.fetch_add(1, Ordering::SeqCst) < 2 { + Err::, String>("boom".to_owned()) + } else { + Ok(vec![RegistryStreamRecord { + stream_id: STREAM_KEY.to_owned(), + epoch: 1, + next_seq: 42, + }]) + } + } + }, + 3, + Duration::ZERO, + ) + .await; + + assert_eq!( + fetch, + RegistryFetch::Snapshot(vec![RegistryStreamRecord { + stream_id: STREAM_KEY.to_owned(), + epoch: 1, + next_seq: 42, + }]) + ); + assert_eq!(calls.load(Ordering::SeqCst), 3); +} From ce4b240c851b1efa44f6694871bab1a50b7a159d Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 14:05:48 -0400 Subject: [PATCH 17/38] fix(forwarder): treat fresh p2p identity as benign first boot in stream restore --- services/forwarder/src/main.rs | 15 +++++++-- services/forwarder/src/p2p/mod.rs | 20 +++++++++++ services/forwarder/src/storage/restore.rs | 25 ++++++++++++-- .../tests/stream_identity_restore.rs | 33 +++++++++++++++++++ 4 files changed, 89 insertions(+), 4 deletions(-) diff --git a/services/forwarder/src/main.rs b/services/forwarder/src/main.rs index 8c62e8bc..e014af8a 100644 --- a/services/forwarder/src/main.rs +++ b/services/forwarder/src/main.rs @@ -238,7 +238,7 @@ async fn main() { .any(|key| !j.stream_exists(key).unwrap_or(false)) }; if any_missing { - let fetch = if cfg.p2p.enabled && cfg.p2p.server_url.is_some() { + let fetch = if let (true, Some(server_url)) = (cfg.p2p.enabled, &cfg.p2p.server_url) { // Reuse the persisted minted device token (it lives at a // different path than the journal, so it typically survives a // journal-loss reboot). Never bootstrap here: minting needs the @@ -249,7 +249,7 @@ async fn main() { match forwarder::p2p::read_device_token(&token_path) { Ok(Some(device_token)) => { let client = forwarder::p2p::ServerCatalogClient::with_timeout( - cfg.p2p.server_url.clone().unwrap_or_default(), + server_url.clone(), device_token, Duration::from_secs(cfg.p2p.allowlist_request_timeout_secs), ); @@ -274,6 +274,17 @@ async fn main() { ) .await } + Ok(None) if !forwarder::p2p::persistent_identity_exists(&cfg.p2p) => { + // No token AND no P2P identity yet: a true first boot. + // The registry cannot hold records for an identity + // that is about to be freshly generated, so this is + // benign (info), not a lost token (error). + info!( + path = %token_path.display(), + "no device token and no p2p identity yet: first boot, skipping registry restore" + ); + RegistryFetch::FreshIdentity + } Ok(None) => { warn!( path = %token_path.display(), diff --git a/services/forwarder/src/p2p/mod.rs b/services/forwarder/src/p2p/mod.rs index 502b2dcc..5c031de6 100644 --- a/services/forwarder/src/p2p/mod.rs +++ b/services/forwarder/src/p2p/mod.rs @@ -370,6 +370,26 @@ pub fn device_token_path(config: &P2pConfig) -> PathBuf { .join("p2p-device-token") } +/// Whether a persistent P2P identity already exists for this config: true +/// when a deterministic seed is configured or the secret-key file is present. +/// +/// `pub` so startup stream-identity restore (main.rs) can distinguish a true +/// first boot (fresh identity → the server registry cannot hold records for +/// it) from a lost device token on an established identity. +#[must_use] +pub fn persistent_identity_exists(config: &P2pConfig) -> bool { + if config.secret_key_seed_hex.is_some() { + return true; + } + Path::new( + config + .secret_key_path + .as_deref() + .unwrap_or(DEFAULT_P2P_SECRET_KEY_PATH), + ) + .exists() +} + /// Resolve the minted per-device token: return the persisted one if present, /// otherwise bootstrap via the voucher and persist the result. /// diff --git a/services/forwarder/src/storage/restore.rs b/services/forwarder/src/storage/restore.rs index 2464523d..b6df97a1 100644 --- a/services/forwarder/src/storage/restore.rs +++ b/services/forwarder/src/storage/restore.rs @@ -9,12 +9,15 @@ //! `epoch`/`next_seq` from the server's stored catalog //! (`GET /forwarder/catalog`), with fixed slack added on top. //! -//! Known design gap (accepted): with no coordination server configured there +//! Known design gaps (accepted): with no coordination server configured there //! is no durable high-water source, so journal loss on a server-less //! deployment still risks seq reuse — the loud log is the mitigation. Even //! with a server, the restored high-water can lag the lost journal by up to //! one catalog-push interval; [`RESTORE_SEQ_SLACK`] is the mitigation for -//! that. +//! that. Finally, a server whose registry database was reset returns a +//! successful *empty* snapshot that is indistinguishable from a first boot, +//! so simultaneous journal + server-registry loss still silently reseeds at +//! seq 1 — the server is the durable source of truth by design. use std::time::Duration; @@ -49,6 +52,11 @@ pub enum RegistryFetch { /// No server is configured — there is no durable high-water source at all /// (accepted design gap; see module docs). NotConfigured, + /// A server is configured but the P2P endpoint identity is about to be + /// freshly generated (no secret-key file and no deterministic seed), so + /// the registry cannot hold records for it: a true first boot, not a + /// lost token. Streams are seeded at seq 1 with an info log only. + FreshIdentity, /// The server was unreachable or errored after bounded retries. This /// includes 404 from an older server without `GET /forwarder/catalog` and /// a missing device token. @@ -196,6 +204,19 @@ fn restore_one_stream( } Ok(StreamRestoreOutcome::SeededFirstBoot) } + RegistryFetch::FreshIdentity => { + tracing::info!( + stream = %key, + "p2p identity freshly generated: no registry record can exist; seeding at seq 1 \ + (first boot)" + ); + if let Some(logger) = logger { + logger.log(format!( + "stream {key}: first boot with new p2p identity, starting at seq 1" + )); + } + Ok(StreamRestoreOutcome::SeededFirstBoot) + } RegistryFetch::NotConfigured => { tracing::warn!( stream = %key, diff --git a/services/forwarder/tests/stream_identity_restore.rs b/services/forwarder/tests/stream_identity_restore.rs index 5de2b37c..719428f9 100644 --- a/services/forwarder/tests/stream_identity_restore.rs +++ b/services/forwarder/tests/stream_identity_restore.rs @@ -125,6 +125,39 @@ fn restore_seeds_seq_one_when_server_has_no_record() { ); } +/// First boot with a server configured but no P2P identity yet: the registry +/// cannot hold records for a freshly generated identity, so seeding at seq 1 +/// is benign — info log only, no warn/error noise. +#[test] +fn restore_fresh_identity_seeds_seq_one_without_error_noise() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("journal.sqlite"); + let mut journal = Journal::open(&path).unwrap(); + + let logger = test_logger(); + let outcomes = restore_streams_at_startup( + &mut journal, + &[STREAM_KEY.to_owned()], + &RegistryFetch::FreshIdentity, + Some(&logger), + ) + .unwrap(); + + assert_eq!(outcomes[0].1, StreamRestoreOutcome::SeededFirstBoot); + let (_, seq) = journal + .append_read(STREAM_KEY, None, b"aa0000000000000000000000000000", "RAW") + .unwrap(); + assert_eq!(seq, 1); + assert!( + !logger + .entries() + .iter() + .any(|e| e.contains("[ERROR]") || e.contains("[WARN]")), + "fresh-identity first boot must not emit warn/error UI logs: {:?}", + logger.entries() + ); +} + /// Case (c): registry unavailable (unreachable, errored, or 404 from an older /// server) — seed seq 1 but emit a loud error UI log warning that receiver /// dedup may discard reads. From 65ddb3d93a2732c3f9cf8796f7262300ad60e784 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 14:21:00 -0400 Subject: [PATCH 18/38] perf(forwarder): serve P2P replay reads from per-subscriber read-only SQLite connections --- services/forwarder/src/main.rs | 1 + services/forwarder/src/p2p/control.rs | 22 +- services/forwarder/src/p2p/data.rs | 64 ++++- services/forwarder/src/p2p/endpoint.rs | 14 +- services/forwarder/src/p2p/mod.rs | 5 +- services/forwarder/src/p2p/remote_config.rs | 6 +- services/forwarder/src/replay.rs | 6 +- services/forwarder/src/storage/journal.rs | 275 ++++++++++++++++++-- 8 files changed, 328 insertions(+), 65 deletions(-) diff --git a/services/forwarder/src/main.rs b/services/forwarder/src/main.rs index e014af8a..24a44477 100644 --- a/services/forwarder/src/main.rs +++ b/services/forwarder/src/main.rs @@ -415,6 +415,7 @@ async fn main() { )); let p2p_runtime = match forwarder::p2p::start_forwarder_p2p( &cfg.p2p, + journal_path, Arc::clone(&journal), &all_readers .iter() diff --git a/services/forwarder/src/p2p/control.rs b/services/forwarder/src/p2p/control.rs index e2354d6c..2d8ce239 100644 --- a/services/forwarder/src/p2p/control.rs +++ b/services/forwarder/src/p2p/control.rs @@ -23,7 +23,7 @@ use std::time::Duration; use prost::Message; #[cfg(test)] use rt_iroh::Connection; -use rt_iroh::{NodeId, RecvStream, SendStream}; +use rt_iroh::{EndpointId, RecvStream, SendStream}; use rt_p2p_protocol::{ CAP_CONTROL_EVENTS, CAP_READER_CONTROL, CAP_REMOTE_CONFIG, ConfigGetRequest, ConfigGetResponse, ConfigSetRequest, ConfigSetResponse, ControlC2F, ControlF2C, DownloadProgress, Hello, @@ -223,7 +223,7 @@ pub trait RemoteConfigHandler: std::fmt::Debug + Send + Sync + 'static { /// `peer` is the authenticated node id of the receiver that sent the /// request; implementations use it to attribute the config write in /// UI-visible logs. - fn set_config(&self, request: ConfigSetRequest, peer: NodeId) -> ConfigSetFuture<'_>; + fn set_config(&self, request: ConfigSetRequest, peer: EndpointId) -> ConfigSetFuture<'_>; /// Triggers the same graceful restart path as the HTTP restart endpoint. fn restart(&self, request: RestartRequest) -> RestartFuture<'_>; @@ -238,7 +238,7 @@ impl RemoteConfigHandler for Arc { (**self).get_config(request) } - fn set_config(&self, request: ConfigSetRequest, peer: NodeId) -> ConfigSetFuture<'_> { + fn set_config(&self, request: ConfigSetRequest, peer: EndpointId) -> ConfigSetFuture<'_> { (**self).set_config(request, peer) } @@ -275,7 +275,7 @@ impl RemoteConfigHandler for NoopRemoteConfigHandler { }) } - fn set_config(&self, request: ConfigSetRequest, _peer: NodeId) -> ConfigSetFuture<'_> { + fn set_config(&self, request: ConfigSetRequest, _peer: EndpointId) -> ConfigSetFuture<'_> { Box::pin(async move { ConfigSetResponse { request_id: request.request_id, @@ -437,7 +437,7 @@ pub(crate) async fn negotiate_control_stream( pub(crate) async fn run_control_stream_loop( send: SendStream, recv: RecvStream, - peer: NodeId, + peer: EndpointId, capabilities: Vec, heartbeat: HeartbeatConfig, outbound_events: Option, @@ -469,7 +469,7 @@ pub(crate) async fn serve_control_with_typed_control( outbound_events: Option, remote_config: Arc, ) -> Result<(), BoxError> { - let peer = connection.remote_node_id()?; + let peer = connection.remote_id(); let (send, recv) = connection.accept_bi().await?; serve_control_stream_with_typed_control( send, @@ -490,7 +490,7 @@ pub(crate) async fn serve_control_with_typed_control( pub(crate) async fn serve_control_stream_with_typed_control( send: SendStream, recv: RecvStream, - peer: NodeId, + peer: EndpointId, catalog: &dyn CatalogProvider, handshake_timeout: Duration, heartbeat: HeartbeatConfig, @@ -615,7 +615,7 @@ async fn negotiate_and_serve_catalog_stream( async fn run_control_loop( mut send: SendStream, mut recv: RecvStream, - peer: NodeId, + peer: EndpointId, capabilities: Vec, config: HeartbeatConfig, reader_control: Arc, @@ -1188,7 +1188,7 @@ mod tests { config_json: String, restart_needed: bool, last_set: Mutex>, - last_set_peer: Mutex>, + last_set_peer: Mutex>, get_calls: AtomicUsize, set_calls: AtomicUsize, restart_calls: AtomicUsize, @@ -1232,7 +1232,7 @@ mod tests { }) } - fn set_config(&self, request: ConfigSetRequest, peer: NodeId) -> ConfigSetFuture<'_> { + fn set_config(&self, request: ConfigSetRequest, peer: EndpointId) -> ConfigSetFuture<'_> { Box::pin(async move { if !self.allow { return ConfigSetResponse { @@ -1574,7 +1574,7 @@ mod tests { ); assert_eq!( *handler.last_set_peer.lock().unwrap(), - Some(receiver.node_id()), + Some(receiver.endpoint_id()), "handler must be told which peer sent the config write" ); diff --git a/services/forwarder/src/p2p/data.rs b/services/forwarder/src/p2p/data.rs index ca73d8f1..59c74c95 100644 --- a/services/forwarder/src/p2p/data.rs +++ b/services/forwarder/src/p2p/data.rs @@ -5,6 +5,7 @@ //! the journal wake registry for live records. Acknowledgements update the //! receiver cursor table that retention uses as its floor. +use std::path::{Path, PathBuf}; use std::sync::Arc; use rt_iroh::{Connection, RecvStream, SendStream}; @@ -16,7 +17,7 @@ use tokio::sync::{Mutex, mpsc}; use tokio::task::{JoinHandle, JoinSet}; use crate::replay::ReplayEngine; -use crate::storage::journal::{Journal, JournalEvent}; +use crate::storage::journal::{Journal, JournalError, JournalEvent, ReadJournal}; use super::control::{read_frame, write_frame}; @@ -36,17 +37,55 @@ impl Drop for AbortOnDrop { } } +type ReadJournalFactory = Arc Result + Send + Sync>; + /// Runtime knobs for data-stream delivery. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone)] pub struct DataConfig { /// Maximum journal records sent in one [`EventBatch`]. pub max_events_per_batch: usize, + read_journal_factory: ReadJournalFactory, +} + +impl DataConfig { + #[must_use] + pub fn with_read_journal_path(mut self, path: impl AsRef) -> Self { + let path: PathBuf = path.as_ref().to_owned(); + self.read_journal_factory = Arc::new(move || Journal::open_read_only(&path)); + self + } + + #[must_use] + pub fn with_read_journal_factory( + mut self, + factory: impl Fn() -> Result + Send + Sync + 'static, + ) -> Self { + self.read_journal_factory = Arc::new(factory); + self + } + + fn open_read_journal(&self) -> Result { + (self.read_journal_factory)() + } +} + +impl std::fmt::Debug for DataConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DataConfig") + .field("max_events_per_batch", &self.max_events_per_batch) + .finish_non_exhaustive() + } } impl Default for DataConfig { fn default() -> Self { Self { max_events_per_batch: 256, + read_journal_factory: Arc::new(|| { + Err(JournalError::InvalidData( + "read journal factory is not configured".to_owned(), + )) + }), } } } @@ -80,6 +119,7 @@ pub async fn serve_data_streams( while tasks.try_join_next().is_some() {} let journal = Arc::clone(&journal); let receiver_id = receiver_id.clone(); + let config = config.clone(); tasks.spawn(async move { if let Err(error) = serve_data_stream(send, recv, journal, receiver_id, config).await { tracing::warn!(%error, "p2p: data stream failed"); @@ -113,13 +153,11 @@ async fn serve_data_stream( let stream_key = wire_stream_key(&subscribe.stream_id)?; let stream_id = subscribe.stream_id.clone(); let max = config.max_events_per_batch.max(1); - let (earliest, latest_at_open) = { - let journal = journal.lock().await; - ( - journal.retention_state(&stream_key)?.earliest_available_seq, - journal.latest_committed_seq(&stream_key)?, - ) - }; + let read_journal = config.open_read_journal()?; + let earliest = read_journal + .retention_state(&stream_key)? + .earliest_available_seq; + let latest_at_open = read_journal.latest_committed_seq(&stream_key)?; // Only `Unspecified` falls back to replay; an unknown enum value is a // protocol violation and fails the stream rather than silently replaying. let mode = SubscribeMode::try_from(subscribe.mode) @@ -185,10 +223,7 @@ async fn serve_data_stream( ) .await?; - let batch = { - let journal = journal.lock().await; - replay.read_after(&journal, &stream_key, cursor, max)? - }; + let batch = replay.read_after(&read_journal, &stream_key, cursor, max)?; if let Some(gap) = batch.gap { write_frame( @@ -464,6 +499,7 @@ mod tests { async fn start_harness(config: DataConfig) -> TestResult { let dir = tempfile::tempdir()?; let journal_path = dir.path().join("journal.db"); + let config = config.with_read_journal_path(&journal_path); let journal = Arc::new(Mutex::new(Journal::open(&journal_path)?)); let forwarder = EndpointBuilder::test([91; 32]).bind().await?; let receiver = EndpointBuilder::test([92; 32]).bind().await?; @@ -771,6 +807,7 @@ mod tests { async fn epoch_started_precedes_new_epoch_events() -> TestResult { let harness = start_harness(DataConfig { max_events_per_batch: 1, + ..DataConfig::default() }) .await?; { @@ -822,6 +859,7 @@ mod tests { async fn slow_receiver_does_not_block_others() -> TestResult { let harness = start_harness(DataConfig { max_events_per_batch: 1, + ..DataConfig::default() }) .await?; let slow_stream = OTHER_STREAM_KEY.to_owned(); diff --git a/services/forwarder/src/p2p/endpoint.rs b/services/forwarder/src/p2p/endpoint.rs index c72db594..b2e08dbf 100644 --- a/services/forwarder/src/p2p/endpoint.rs +++ b/services/forwarder/src/p2p/endpoint.rs @@ -192,7 +192,7 @@ impl P2pEndpoint { let catalog = Arc::clone(&self.catalog); let journal = Arc::clone(&self.journal); let config = ConnectionConfig { - data_config: self.data_config, + data_config: self.data_config.clone(), status_feed: self.status_feed.clone(), handshake_timeout: self.handshake_timeout, heartbeat: self.heartbeat, @@ -285,7 +285,7 @@ async fn handle_connection( data_connection, journal, endpoint_id.to_string(), - config.data_config, + config.data_config.clone(), ) .await { @@ -307,7 +307,7 @@ async fn handle_connection( let control_result = run_control_stream_loop( control_send, control_recv, - node_id, + endpoint_id, capabilities, config.heartbeat, outbound_events, @@ -536,7 +536,7 @@ mod tests { allow_list, catalog: empty_catalog(), journal: Arc::new(Mutex::new(journal)), - data_config: DataConfig::default(), + data_config: DataConfig::default().with_read_journal_path(&journal_path), status_feed: None, handshake_timeout: DEFAULT_HANDSHAKE_TIMEOUT, heartbeat: HeartbeatConfig::default(), @@ -834,7 +834,7 @@ mod tests { use rt_p2p_protocol::{CAP_REMOTE_CONFIG, ConfigGetRequest, ConfigSetRequest, Pong}; let receiver = EndpointBuilder::test([84; 32]).bind().await?; - let allow_list = AllowList::new([receiver.node_id()]); + let allow_list = AllowList::new([receiver.endpoint_id()]); let dir = tempfile::tempdir()?; let token_path = dir.path().join("token"); @@ -864,7 +864,7 @@ mod tests { let forwarder = P2pEndpoint::bind_test([85; 32], allow_list) .await? .with_remote_config(handler); - let forwarder_addr = forwarder.node_addr().await; + let forwarder_addr = forwarder.endpoint_addr().await; let accept = { let forwarder = forwarder.clone(); @@ -949,7 +949,7 @@ mod tests { // The UI event stream must carry a log entry attributing the write to // the receiver's node id. - let peer_id = receiver.node_id().to_string(); + let peer_id = receiver.endpoint_id().to_string(); let attributed = tokio::time::timeout(Duration::from_secs(5), async { loop { match ui_rx.recv().await { diff --git a/services/forwarder/src/p2p/mod.rs b/services/forwarder/src/p2p/mod.rs index 5c031de6..986f2236 100644 --- a/services/forwarder/src/p2p/mod.rs +++ b/services/forwarder/src/p2p/mod.rs @@ -104,6 +104,7 @@ impl P2pRuntime { #[allow(clippy::too_many_arguments)] pub async fn start_forwarder_p2p( config: &P2pConfig, + journal_path: &Path, journal: Arc>, reader_streams: &[String], display_name: Option, @@ -139,7 +140,7 @@ pub async fn start_forwarder_p2p( allow_list.clone(), catalog, Arc::clone(&journal), - DataConfig::default(), + DataConfig::default().with_read_journal_path(journal_path), ) .await? .with_status_feed(status_feed) @@ -687,6 +688,7 @@ mod tests { let runtime = start_forwarder_p2p( &p2p_config(receiver.endpoint_id().to_string()), + &journal_path, Arc::clone(&journal), &[stream_key.to_owned()], None, @@ -798,6 +800,7 @@ mod tests { let runtime = start_forwarder_p2p( &p2p_config(receiver.endpoint_id().to_string()), + &journal_path, Arc::clone(&journal), &[stream_key.to_owned()], None, diff --git a/services/forwarder/src/p2p/remote_config.rs b/services/forwarder/src/p2p/remote_config.rs index 7ec6b3ba..ca7ce679 100644 --- a/services/forwarder/src/p2p/remote_config.rs +++ b/services/forwarder/src/p2p/remote_config.rs @@ -23,7 +23,7 @@ use std::sync::Arc; -use rt_iroh::NodeId; +use rt_iroh::EndpointId; use rt_p2p_protocol::{ ConfigGetRequest, ConfigGetResponse, ConfigSetRequest, ConfigSetResponse, RestartRequest, RestartResponse, @@ -120,7 +120,7 @@ impl RemoteConfigHandler for ForwarderRemoteConfigHandler { }) } - fn set_config(&self, request: ConfigSetRequest, peer: NodeId) -> ConfigSetFuture<'_> { + fn set_config(&self, request: ConfigSetRequest, peer: EndpointId) -> ConfigSetFuture<'_> { Box::pin(async move { if !self.allow_remote_config { return ConfigSetResponse { @@ -255,7 +255,7 @@ mod tests { } /// A deterministic peer node id for attributing test config writes. - fn test_peer() -> NodeId { + fn test_peer() -> EndpointId { rt_iroh::SecretKey::from_bytes(&[42u8; 32]).public() } diff --git a/services/forwarder/src/replay.rs b/services/forwarder/src/replay.rs index 5db01992..90eb559f 100644 --- a/services/forwarder/src/replay.rs +++ b/services/forwarder/src/replay.rs @@ -3,7 +3,7 @@ //! Used by the P2P session to determine which events need to be //! (re-)transmitted after a reconnect or on initial connect. -use crate::storage::journal::{Journal, JournalError, JournalEvent}; +use crate::storage::journal::{Journal, JournalError, JournalEvent, ReplayJournal}; // --------------------------------------------------------------------------- // Public types @@ -53,9 +53,9 @@ impl ReplayEngine { /// advancing the cursor and call this again after append wake-ups. If the /// cursor is older than the retained prefix, the batch carries a gap notice /// and no records so the caller can jump to `earliest - 1` explicitly. - pub fn read_after( + pub fn read_after( &self, - journal: &Journal, + journal: &J, stream_id: &str, cursor: i64, max: usize, diff --git a/services/forwarder/src/storage/journal.rs b/services/forwarder/src/storage/journal.rs index 7e77af8c..39db537b 100644 --- a/services/forwarder/src/storage/journal.rs +++ b/services/forwarder/src/storage/journal.rs @@ -6,7 +6,7 @@ use crate::storage::migrations; use crate::storage::wake::WakeRegistry; -use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params}; +use rusqlite::{Connection, OpenFlags, OptionalExtension, TransactionBehavior, params}; use std::path::Path; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; @@ -115,6 +115,80 @@ impl From for JournalError { } } +/// Read-only journal API used by replay subscribers. +pub trait ReplayJournal { + fn retention_state(&self, stream_key: &str) -> Result; + + fn latest_committed_seq(&self, stream_key: &str) -> Result; + + fn read_events_after( + &self, + stream_key: &str, + after_seq: i64, + max: usize, + ) -> Result, JournalError>; +} + +/// A read-only SQLite journal connection for replay queries. +pub struct ReadJournal { + conn: Connection, +} + +impl ReadJournal { + pub fn open(path: &Path) -> Result { + let conn = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; + apply_read_pragmas(&conn)?; + Ok(Self { conn }) + } + + #[cfg(test)] + pub fn connection_for_test(&self) -> &Connection { + &self.conn + } + + pub fn current_epoch_and_next_seq(&self, stream_key: &str) -> Result<(i64, i64), JournalError> { + let epoch = current_epoch(&self.conn, stream_key)?; + let next_seq = next_seq(&self.conn, stream_key)?; + Ok((epoch, next_seq)) + } + + pub fn retention_state(&self, stream_key: &str) -> Result { + retention_state(&self.conn, stream_key) + } + + pub fn latest_committed_seq(&self, stream_key: &str) -> Result { + latest_committed_seq(&self.conn, stream_key) + } + + pub fn read_events_after( + &self, + stream_key: &str, + after_seq: i64, + max: usize, + ) -> Result, JournalError> { + read_events_after(&self.conn, stream_key, after_seq, max) + } +} + +impl ReplayJournal for ReadJournal { + fn retention_state(&self, stream_key: &str) -> Result { + self.retention_state(stream_key) + } + + fn latest_committed_seq(&self, stream_key: &str) -> Result { + self.latest_committed_seq(stream_key) + } + + fn read_events_after( + &self, + stream_key: &str, + after_seq: i64, + max: usize, + ) -> Result, JournalError> { + self.read_events_after(stream_key, after_seq, max) + } +} + /// The durable SQLite journal for a single forwarder instance. pub struct Journal { conn: Connection, @@ -132,6 +206,10 @@ impl Journal { }) } + pub fn open_read_only(path: &Path) -> Result { + ReadJournal::open(path) + } + #[cfg(test)] pub fn set_query_only(&self, on: bool) -> Result<(), JournalError> { self.conn.execute_batch(if on { @@ -485,7 +563,7 @@ impl Journal { } pub fn latest_committed_seq(&self, stream_key: &str) -> Result { - Ok(next_seq(&self.conn, stream_key)?.saturating_sub(1)) + latest_committed_seq(&self.conn, stream_key) } pub fn read_events_after( @@ -494,17 +572,7 @@ impl Journal { after_seq: i64, max: usize, ) -> Result, JournalError> { - let limit = i64::try_from(max).unwrap_or(i64::MAX); - let mut stmt = self.conn.prepare( - "SELECT rowid, stream_id, epoch, seq, reader_timestamp, raw_frame, read_kind, - CAST(received_unix_ms AS TEXT) - FROM events - WHERE stream_id = ?1 AND seq > ?2 - ORDER BY seq ASC - LIMIT ?3", - )?; - let rows = stmt.query_map(params![stream_key, after_seq, limit], map_event)?; - collect_events(rows) + read_events_after(&self.conn, stream_key, after_seq, max) } /// Return all unacked events for a stream epoch after `after_seq`. @@ -664,20 +732,7 @@ impl Journal { } pub fn retention_state(&self, stream_key: &str) -> Result { - self.conn - .query_row( - "SELECT earliest_available_seq, forced_gap_count - FROM stream_retention - WHERE stream_id = ?1", - params![stream_key], - |row| { - Ok(RetentionState { - earliest_available_seq: row.get(0)?, - forced_gap_count: row.get(1)?, - }) - }, - ) - .map_err(Into::into) + retention_state(&self.conn, stream_key) } pub fn clear_stream(&mut self, stream_key: &str) -> Result<(), JournalError> { @@ -742,6 +797,25 @@ impl Journal { } } +impl ReplayJournal for Journal { + fn retention_state(&self, stream_key: &str) -> Result { + self.retention_state(stream_key) + } + + fn latest_committed_seq(&self, stream_key: &str) -> Result { + self.latest_committed_seq(stream_key) + } + + fn read_events_after( + &self, + stream_key: &str, + after_seq: i64, + max: usize, + ) -> Result, JournalError> { + self.read_events_after(stream_key, after_seq, max) + } +} + fn validate_stream_seed(stream_id: &str, epoch: i64, next_seq: i64) -> Result<(), JournalError> { if stream_id.is_empty() { return Err(JournalError::InvalidData( @@ -917,6 +991,55 @@ fn delete_candidates( Ok((deleted, forced)) } +fn apply_read_pragmas(conn: &Connection) -> Result<(), JournalError> { + conn.execute_batch( + "PRAGMA query_only=ON; + PRAGMA wal_autocheckpoint=1000; + PRAGMA busy_timeout=5000; + PRAGMA foreign_keys=ON;", + )?; + Ok(()) +} + +fn retention_state(conn: &Connection, stream_key: &str) -> Result { + conn.query_row( + "SELECT earliest_available_seq, forced_gap_count + FROM stream_retention + WHERE stream_id = ?1", + params![stream_key], + |row| { + Ok(RetentionState { + earliest_available_seq: row.get(0)?, + forced_gap_count: row.get(1)?, + }) + }, + ) + .map_err(Into::into) +} + +fn latest_committed_seq(conn: &Connection, stream_key: &str) -> Result { + Ok(next_seq(conn, stream_key)?.saturating_sub(1)) +} + +fn read_events_after( + conn: &Connection, + stream_key: &str, + after_seq: i64, + max: usize, +) -> Result, JournalError> { + let limit = i64::try_from(max).unwrap_or(i64::MAX); + let mut stmt = conn.prepare( + "SELECT rowid, stream_id, epoch, seq, reader_timestamp, raw_frame, read_kind, + CAST(received_unix_ms AS TEXT) + FROM events + WHERE stream_id = ?1 AND seq > ?2 + ORDER BY seq ASC + LIMIT ?3", + )?; + let rows = stmt.query_map(params![stream_key, after_seq, limit], map_event)?; + collect_events(rows) +} + /// Return the next stream-wide sequence number. /// /// The seq is stream-wide and never resets across epochs. It is derived from @@ -992,3 +1115,101 @@ fn unix_ms() -> i64 { .try_into() .unwrap_or(i64::MAX) } + +#[cfg(test)] +mod tests { + use super::{Journal, TransactionBehavior, params}; + + #[test] + fn read_only_journal_sees_committed_rows() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("journal.db"); + let mut journal = Journal::open(&path).expect("open write journal"); + journal + .ensure_stream_state("stream-a", 7) + .expect("ensure stream"); + journal + .append_read("stream-a", Some("1234"), b"one", "RAW") + .expect("append read"); + + let read = Journal::open_read_only(&path).expect("open read journal"); + + let retention = read.retention_state("stream-a").expect("retention"); + assert_eq!(retention.earliest_available_seq, 1); + assert_eq!(read.latest_committed_seq("stream-a").expect("latest"), 1); + assert_eq!( + read.current_epoch_and_next_seq("stream-a") + .expect("epoch and next seq"), + (7, 2) + ); + let events = read + .read_events_after("stream-a", 0, 10) + .expect("read events"); + assert_eq!(events.len(), 1); + assert_eq!(events[0].seq, 1); + assert_eq!(events[0].raw_frame, b"one"); + } + + #[test] + fn read_only_journal_rejects_raw_sql_writes() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("journal.db"); + let _journal = Journal::open(&path).expect("open write journal"); + let read = Journal::open_read_only(&path).expect("open read journal"); + + let result = read.connection_for_test().execute( + "INSERT INTO receivers (endpoint_id, display_name, approved_unix_ms) + VALUES ('receiver-a', 'receiver-a', 0)", + [], + ); + + assert!(result.is_err(), "read-only connection accepted a write"); + } + + #[test] + fn read_only_batch_read_succeeds_during_in_flight_write_transaction() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("journal.db"); + let mut journal = Journal::open(&path).expect("open write journal"); + journal + .ensure_stream_state("stream-a", 1) + .expect("ensure stream"); + journal + .append_read("stream-a", None, b"committed", "RAW") + .expect("append committed read"); + let read = Journal::open_read_only(&path).expect("open read journal"); + let busy_timeout: i64 = read + .connection_for_test() + .query_row("PRAGMA busy_timeout", [], |row| row.get(0)) + .expect("busy timeout"); + assert_eq!(busy_timeout, 5000); + + let tx = journal + .conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .expect("begin write tx"); + tx.execute( + "INSERT INTO events + (stream_id, seq, epoch, raw_frame, read_kind, reader_timestamp, received_unix_ms) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![ + "stream-a", + 2, + 1, + b"uncommitted", + "RAW", + Option::<&str>::None, + 0 + ], + ) + .expect("insert uncommitted read"); + + let events = read + .read_events_after("stream-a", 0, 10) + .expect("batch read during write tx"); + assert_eq!(events.len(), 1); + assert_eq!(events[0].raw_frame, b"committed"); + + tx.rollback().expect("rollback write tx"); + } +} From da1905789f5e7071c35c3d7b2afc0ad54b3daa6c Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 14:37:09 -0400 Subject: [PATCH 19/38] refactor(forwarder): drop dead P2pEndpoint::bind and trim ReadJournal surface --- services/forwarder/src/p2p/endpoint.rs | 29 +++++------------------ services/forwarder/src/storage/journal.rs | 15 +++--------- 2 files changed, 9 insertions(+), 35 deletions(-) diff --git a/services/forwarder/src/p2p/endpoint.rs b/services/forwarder/src/p2p/endpoint.rs index b2e08dbf..8fffc459 100644 --- a/services/forwarder/src/p2p/endpoint.rs +++ b/services/forwarder/src/p2p/endpoint.rs @@ -8,13 +8,10 @@ //! negotiation, serves the [`StreamCatalog`](rt_p2p_protocol::StreamCatalog), //! and runs the heartbeat until the peer disconnects or is declared dead. -use std::path::Path; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use rt_iroh::{ - Connection, Endpoint, EndpointAddr, EndpointBuilder, EndpointId, load_or_create_secret_key, -}; +use rt_iroh::{Connection, Endpoint, EndpointAddr, EndpointBuilder, EndpointId}; use rt_p2p_protocol::{CAP_CONTROL_EVENTS, has_capability}; use tokio::sync::{Mutex, broadcast, mpsc}; @@ -91,28 +88,14 @@ impl std::fmt::Debug for P2pEndpoint { } impl P2pEndpoint { - /// Binds the endpoint, loading or creating the persistent secret key at - /// `secret_key_path`. Admitted peers are served the catalog from `catalog`. - pub async fn bind( - secret_key_path: impl AsRef, - allow_list: AllowList, - catalog: Arc, - journal: Arc>, - ) -> Result { - let secret_key = load_or_create_secret_key(secret_key_path)?; - Self::bind_with_builder( - EndpointBuilder::default().secret_key(secret_key), - allow_list, - catalog, - journal, - DataConfig::default(), - ) - .await - } - /// Binds an endpoint from an explicit builder. Production startup uses this /// to apply loopback/test transport knobs from config without exposing raw /// iroh APIs to the binary. + /// + /// This is the only constructor: it forces callers to supply a + /// [`DataConfig`], whose read-journal factory the data plane requires + /// (a default config would serve an endpoint whose data subscriptions + /// always fail). pub async fn bind_with_builder( builder: EndpointBuilder, allow_list: AllowList, diff --git a/services/forwarder/src/storage/journal.rs b/services/forwarder/src/storage/journal.rs index 39db537b..a3516532 100644 --- a/services/forwarder/src/storage/journal.rs +++ b/services/forwarder/src/storage/journal.rs @@ -146,12 +146,6 @@ impl ReadJournal { &self.conn } - pub fn current_epoch_and_next_seq(&self, stream_key: &str) -> Result<(i64, i64), JournalError> { - let epoch = current_epoch(&self.conn, stream_key)?; - let next_seq = next_seq(&self.conn, stream_key)?; - Ok((epoch, next_seq)) - } - pub fn retention_state(&self, stream_key: &str) -> Result { retention_state(&self.conn, stream_key) } @@ -992,9 +986,11 @@ fn delete_candidates( } fn apply_read_pragmas(conn: &Connection) -> Result<(), JournalError> { + // No journal_mode here: the writer sets WAL, and a read-only connection + // cannot change it. wal_autocheckpoint is also omitted: read-only + // connections never commit, so they never trigger autocheckpoints. conn.execute_batch( "PRAGMA query_only=ON; - PRAGMA wal_autocheckpoint=1000; PRAGMA busy_timeout=5000; PRAGMA foreign_keys=ON;", )?; @@ -1137,11 +1133,6 @@ mod tests { let retention = read.retention_state("stream-a").expect("retention"); assert_eq!(retention.earliest_available_seq, 1); assert_eq!(read.latest_committed_seq("stream-a").expect("latest"), 1); - assert_eq!( - read.current_epoch_and_next_seq("stream-a") - .expect("epoch and next seq"), - (7, 2) - ); let events = read .read_events_after("stream-a", 0, 10) .expect("read events"); From b8787a6fb46a278333e3432eead0b57f7159d62d Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 14:55:47 -0400 Subject: [PATCH 20/38] refactor(forwarder): extract config persistence into config_service module --- services/forwarder/src/config_service.rs | 735 +++++++++++++++++++ services/forwarder/src/lib.rs | 1 + services/forwarder/src/main.rs | 4 +- services/forwarder/src/p2p/endpoint.rs | 4 +- services/forwarder/src/p2p/remote_config.rs | 11 +- services/forwarder/src/status_http.rs | 734 +----------------- services/forwarder/tests/config_endpoints.rs | 58 +- services/forwarder/tests/health_endpoints.rs | 4 +- 8 files changed, 784 insertions(+), 767 deletions(-) create mode 100644 services/forwarder/src/config_service.rs diff --git a/services/forwarder/src/config_service.rs b/services/forwarder/src/config_service.rs new file mode 100644 index 00000000..c9b8ebd5 --- /dev/null +++ b/services/forwarder/src/config_service.rs @@ -0,0 +1,735 @@ +//! Config persistence for the forwarder. +//! +//! Owns the on-disk TOML config file: serialized read-modify-write access +//! (via [`ConfigState`]), atomic writes, JSON serialization shared by the +//! local status HTTP surface and the P2P remote-config verbs, and the +//! per-section update logic behind `POST /api/v1/config/{section}`. + +use std::io::Write as _; +use std::net::SocketAddrV4; +use std::sync::Arc; + +use tokio::sync::Mutex; + +use crate::status_http::{ + SubsystemStatus, apply_control_action_from_config, bad_request_error, + mark_restart_needed_and_emit, require_object_payload, +}; + +/// Holds the config file path and a write lock for read-modify-write operations. +pub struct ConfigState { + pub path: std::path::PathBuf, + pub(crate) write_lock: Mutex<()>, +} + +impl ConfigState { + pub fn new(path: std::path::PathBuf) -> Self { + ConfigState { + path, + write_lock: Mutex::new(()), + } + } +} + +pub(crate) fn write_atomic(path: &std::path::Path, content: &str) -> std::io::Result<()> { + let original_permissions = std::fs::metadata(path).map(|m| m.permissions()).ok(); + + let parent = path.parent().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("path has no parent: {}", path.display()), + ) + })?; + let file_name = path.file_name().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("path has no file name: {}", path.display()), + ) + })?; + + let file_name = file_name.to_string_lossy(); + let pid = std::process::id(); + + for attempt in 0..=16 { + let tmp_name = format!(".{}.tmp.{}.{}", file_name, pid, attempt); + let tmp_path = parent.join(tmp_name); + match std::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&tmp_path) + { + Ok(mut temp_file) => { + let result = (|| -> std::io::Result<()> { + temp_file.write_all(content.as_bytes())?; + temp_file.sync_all()?; + + if let Some(perms) = &original_permissions { + std::fs::set_permissions(&tmp_path, perms.clone())?; + } + + std::fs::rename(&tmp_path, path)?; + if let Ok(parent_dir) = std::fs::File::open(parent) { + let _ = parent_dir.sync_all(); + } + Ok(()) + })(); + if result.is_err() { + let _ = std::fs::remove_file(&tmp_path); + } + return result; + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(e), + } + } + + Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!("failed to allocate temp path for {}", path.display()), + )) +} + +/// Read the config TOML file as a JSON value plus the current `restart_needed` +/// state, returning a plain error message on failure. +/// +/// Shared core for both the HTTP `GET /api/v1/config` endpoint and the P2P +/// remote-config get verb, so the two always agree on the serialized shape. +async fn read_config_value( + config_state: &ConfigState, + subsystem: &Arc>, +) -> Result<(serde_json::Value, bool), String> { + let _lock = config_state.write_lock.lock().await; + + let toml_str = + std::fs::read_to_string(&config_state.path).map_err(|e| format!("File read error: {e}"))?; + + let raw: crate::config::RawConfig = + toml::from_str(&toml_str).map_err(|e| format!("TOML parse error: {e}"))?; + + let json = serde_json::to_value(&raw).map_err(|e| format!("JSON serialize error: {e}"))?; + + let restart_needed = subsystem.lock().await.restart_needed(); + Ok((json, restart_needed)) +} + +/// Read the config TOML file as JSON. +/// +/// Returns `(config_json, restart_needed)` on success. +pub async fn read_config_json( + config_state: &ConfigState, + subsystem: &Arc>, +) -> Result<(serde_json::Value, bool), (u16, String)> { + read_config_value(config_state, subsystem) + .await + .map_err(|e| { + ( + 500u16, + serde_json::json!({"ok": false, "error": e}).to_string(), + ) + }) +} + +/// Serialize the current config to a JSON string (identical to the body +/// `GET /api/v1/config` returns) plus the current `restart_needed` state. +/// +/// Used by the P2P remote-config get verb so the receiver UI round-trips the +/// same document whether it reads config over HTTP or P2P. +pub async fn config_json_string( + config_state: &ConfigState, + subsystem: &Arc>, +) -> Result<(String, bool), String> { + let (value, restart_needed) = read_config_value(config_state, subsystem).await?; + let json = serde_json::to_string(&value).map_err(|e| format!("JSON serialize error: {e}"))?; + Ok((json, restart_needed)) +} + +/// Sections a P2P peer may never change: identity/credentials (`auth`), +/// transport trust (`p2p`), and the gates themselves (`control`). +/// +/// Boundary rationale: `[auth]`/`[p2p]`/`[control]` changes are trust/identity +/// escalation — permanent and self-granting; `[journal]`, `[[readers]]`, and +/// display fields are the operational surface remote management exists to serve +/// — a hostile allow-listed receiver can disrupt operations through them but +/// cannot expand its own access. `[status_http]`, `[update]`, and `[ups]` were +/// considered and left writable as operational surface; `status_http.bind` is +/// the borderline case (a receiver could rebind the local admin HTTP), but it +/// grants the receiver itself no access. +const REMOTE_PROTECTED_SECTIONS: &[&str] = &["auth", "p2p", "control"]; + +/// Persist a full config document received from P2P remote config, rejecting +/// any attempted change to privileged config sections before writing. +/// +/// The document is parsed into the same [`crate::config::RawConfig`] the get +/// path serializes, re-serialized to TOML, and validated by running the +/// canonical loader before anything is written — so a document that would fail +/// to load on restart is rejected without corrupting the on-disk file. Reuses +/// the same atomic writer and `restart_needed` signal as the per-section HTTP +/// writers. Returns a plain error message on failure. +/// +/// There is intentionally no unrestricted full-document writer: local +/// (trusted) config edits go through the per-section HTTP handlers +/// ([`apply_section_update`] / [`update_config_file`]), so every full-document +/// write path enforces [`REMOTE_PROTECTED_SECTIONS`]. +pub async fn write_config_json_restricted( + config_json: &str, + config_state: &ConfigState, + subsystem: &Arc>, + ui_tx: &tokio::sync::broadcast::Sender, +) -> Result<(), String> { + let incoming: crate::config::RawConfig = + serde_json::from_str(config_json).map_err(|e| format!("invalid config JSON: {e}"))?; + + let _lock = config_state.write_lock.lock().await; + + let current_toml = + std::fs::read_to_string(&config_state.path).map_err(|e| format!("File read error: {e}"))?; + let current: crate::config::RawConfig = + toml::from_str(¤t_toml).map_err(|e| format!("TOML parse error: {e}"))?; + + let incoming_value = + serde_json::to_value(&incoming).map_err(|e| format!("JSON serialize error: {e}"))?; + let current_value = + serde_json::to_value(¤t).map_err(|e| format!("JSON serialize error: {e}"))?; + + for section in REMOTE_PROTECTED_SECTIONS { + if normalize_section(incoming_value.get(section)) + != normalize_section(current_value.get(section)) + { + return Err(format!( + "remote config may not modify the protected [{section}] section" + )); + } + } + + write_config_json_locked(incoming, config_state, subsystem, ui_tx).await +} + +/// Shared locked write body. Caller must hold `config_state.write_lock`. +/// +/// Takes the already-parsed [`crate::config::RawConfig`] (the same value the +/// protected-section comparison ran against) so "what was compared is what is +/// written" holds structurally rather than by re-parsing the same string. +async fn write_config_json_locked( + raw: crate::config::RawConfig, + config_state: &ConfigState, + subsystem: &Arc>, + ui_tx: &tokio::sync::broadcast::Sender, +) -> Result<(), String> { + let new_toml = + toml::to_string_pretty(&raw).map_err(|e| format!("TOML serialize error: {e}"))?; + + // Validate via the canonical loader so we never persist a config that would + // fail to load on the next restart. + crate::config::load_config_from_str(&new_toml, &config_state.path) + .map_err(|e| format!("config validation failed: {e}"))?; + + write_atomic(&config_state.path, &new_toml).map_err(|e| format!("File write error: {e}"))?; + + mark_restart_needed_and_emit(subsystem, ui_tx).await; + Ok(()) +} + +/// A missing section, `null`, and an all-null object are equivalent RawConfig +/// states. +/// +/// Normalization is single-level by design: every protected raw section is a +/// flat struct of scalars/`Vec` today. If a nested struct is ever +/// added to a protected section, an all-null nested object will not be +/// flattened — the failure mode is a false *reject* (fail-closed), and the +/// populated-section round-trip test in `p2p/remote_config.rs` will catch +/// serialization drift. +fn normalize_section(v: Option<&serde_json::Value>) -> serde_json::Value { + match v { + None | Some(serde_json::Value::Null) => serde_json::Value::Null, + Some(serde_json::Value::Object(map)) if map.values().all(serde_json::Value::is_null) => { + serde_json::Value::Null + } + Some(other) => other.clone(), + } +} + +/// Read the TOML config file, apply a mutation, and write it back. +/// +/// Returns Ok(()) on success or Err((status_code, json_error_body)) on failure. +async fn update_config_file( + config_state: &ConfigState, + subsystem: &Arc>, + ui_tx: &tokio::sync::broadcast::Sender, + mutate: impl FnOnce(&mut crate::config::RawConfig) -> Result<(), String>, +) -> Result<(), (u16, String)> { + let _lock = config_state.write_lock.lock().await; + + let toml_str = std::fs::read_to_string(&config_state.path).map_err(|e| { + ( + 500u16, + serde_json::json!({"ok": false, "error": format!("File read error: {}", e)}) + .to_string(), + ) + })?; + + let mut raw: crate::config::RawConfig = toml::from_str(&toml_str).map_err(|e| { + ( + 500u16, + serde_json::json!({"ok": false, "error": format!("TOML parse error: {}", e)}) + .to_string(), + ) + })?; + + mutate(&mut raw).map_err(|e| { + ( + 400u16, + serde_json::json!({"ok": false, "error": e}).to_string(), + ) + })?; + + let new_toml = toml::to_string_pretty(&raw).map_err(|e| { + ( + 500u16, + serde_json::json!({"ok": false, "error": format!("TOML serialize error: {}", e)}) + .to_string(), + ) + })?; + + crate::config::load_config_from_str(&new_toml, &config_state.path).map_err(|e| { + ( + 400u16, + serde_json::json!({"ok": false, "error": format!("config validation failed: {e}")}) + .to_string(), + ) + })?; + + write_atomic(&config_state.path, &new_toml).map_err(|e| { + ( + 500u16, + serde_json::json!({"ok": false, "error": format!("File write error: {}", e)}) + .to_string(), + ) + })?; + + mark_restart_needed_and_emit(subsystem, ui_tx).await; + Ok(()) +} + +/// Apply a config section update by name. +/// +/// Dispatches to the right mutation logic based on `section`, validates the +/// payload, and calls `update_config_file` to persist the change. +/// +/// Recognised sections: `"general"`, `"auth"`, `"journal"`, `"status_http"`, +/// `"control"`, `"update"`, `"p2p"`, `"ups"`, `"readers"`, and `"screen"`. +/// Screen config changes require a restart to apply. +pub async fn apply_section_update( + section: &str, + payload: &serde_json::Value, + config_state: &ConfigState, + subsystem: &Arc>, + ui_tx: &tokio::sync::broadcast::Sender, + logger: Option<&rt_ui_log::UiLogger>, +) -> Result<(), (u16, String)> { + require_object_payload(payload)?; + + match section { + "general" => { + let display_name = optional_string_field(payload, "display_name")?; + update_config_file(config_state, subsystem, ui_tx, |raw| { + raw.display_name = display_name; + Ok(()) + }) + .await + } + "auth" => { + let token_file_opt = optional_string_field(payload, "token_file")?; + let token_file = require_non_empty_trimmed("token_file", token_file_opt) + .map_err(bad_request_error)?; + validate_token_file(&token_file).map_err(bad_request_error)?; + update_config_file(config_state, subsystem, ui_tx, |raw| { + raw.auth = Some(crate::config::RawAuthConfig { + token_file: Some(token_file), + }); + Ok(()) + }) + .await + } + "journal" => { + let sqlite_path = optional_string_field(payload, "sqlite_path")?; + let prune_watermark_pct = optional_u8_field(payload, "prune_watermark_pct")?; + let min_retention = optional_string_field(payload, "min_retention")?; + let max_retention = optional_string_field(payload, "max_retention")?; + let emergency_free_disk_bytes = + optional_u64_field(payload, "emergency_free_disk_bytes")?; + let emergency_max_rows = optional_u64_field(payload, "emergency_max_rows")? + .map(|value| { + i64::try_from(value) + .map_err(|_| bad_request_error("emergency_max_rows must be <= i64::MAX")) + }) + .transpose()?; + if let Some(pct) = prune_watermark_pct + && pct > 100 + { + return Err(bad_request_error( + "prune_watermark_pct must be between 0 and 100", + )); + } + crate::config::validate_retention_settings( + min_retention.as_deref(), + max_retention.as_deref(), + emergency_max_rows, + ) + .map_err(bad_request_error)?; + update_config_file(config_state, subsystem, ui_tx, |raw| { + raw.journal = Some(crate::config::RawJournalConfig { + sqlite_path, + prune_watermark_pct, + min_retention, + max_retention, + emergency_free_disk_bytes, + emergency_max_rows, + }); + Ok(()) + }) + .await + } + "status_http" => { + let bind = optional_string_field(payload, "bind")?.and_then(|s| { + let trimmed = s.trim(); + if trimmed.is_empty() { + None + } else { + Some(trimmed.to_owned()) + } + }); + if let Some(ref bind_addr) = bind { + validate_status_bind(bind_addr).map_err(bad_request_error)?; + } + update_config_file(config_state, subsystem, ui_tx, |raw| { + raw.status_http = Some(crate::config::RawStatusHttpConfig { bind }); + Ok(()) + }) + .await + } + "control" => { + let allow_power_actions = optional_bool_field(payload, "allow_power_actions")?; + let action = optional_string_field(payload, "action")?; + if let Some(action) = action { + return apply_control_action_from_config(&action, config_state, logger).await; + } + update_config_file(config_state, subsystem, ui_tx, |raw| { + // Preserve existing P2P control settings; this handler only + // mutates allow_power_actions. + let allow_remote_config = raw.control.as_ref().and_then(|c| c.allow_remote_config); + let allow_reader_control = + raw.control.as_ref().and_then(|c| c.allow_reader_control); + raw.control = Some(crate::config::RawControlConfig { + allow_power_actions, + allow_remote_config, + allow_reader_control, + }); + Ok(()) + }) + .await + } + "p2p" => { + let enabled = optional_bool_field(payload, "enabled")?; + let server_url = optional_string_field(payload, "server_url")?.and_then(|url| { + let trimmed = url.trim(); + (!trimmed.is_empty()).then(|| trimmed.to_owned()) + }); + let server_token_file = + optional_string_field(payload, "server_token_file")?.and_then(|path| { + let trimmed = path.trim(); + (!trimmed.is_empty()).then(|| trimmed.to_owned()) + }); + if let Some(ref url) = server_url { + validate_server_url(url).map_err(bad_request_error)?; + } + if let Some(ref token_file) = server_token_file { + validate_token_file(token_file).map_err(bad_request_error)?; + } + update_config_file(config_state, subsystem, ui_tx, |raw| { + let previous = raw.p2p.take(); + raw.p2p = Some(crate::config::RawP2pConfig { + enabled, + secret_key_path: previous + .as_ref() + .and_then(|cfg| cfg.secret_key_path.clone()), + secret_key_seed_hex: previous + .as_ref() + .and_then(|cfg| cfg.secret_key_seed_hex.clone()), + bind_addr_v4: previous.as_ref().and_then(|cfg| cfg.bind_addr_v4.clone()), + relay_disabled: previous.as_ref().and_then(|cfg| cfg.relay_disabled), + discovery_disabled: previous.as_ref().and_then(|cfg| cfg.discovery_disabled), + max_concurrent_bidi_streams: previous + .as_ref() + .and_then(|cfg| cfg.max_concurrent_bidi_streams), + static_allowed_receivers: previous + .as_ref() + .and_then(|cfg| cfg.static_allowed_receivers.clone()), + allowlist_cache_path: previous + .as_ref() + .and_then(|cfg| cfg.allowlist_cache_path.clone()), + server_url, + server_token_file, + device_token_file: previous + .as_ref() + .and_then(|cfg| cfg.device_token_file.clone()), + allowlist_poll_interval_secs: previous + .as_ref() + .and_then(|cfg| cfg.allowlist_poll_interval_secs), + allowlist_request_timeout_secs: previous + .as_ref() + .and_then(|cfg| cfg.allowlist_request_timeout_secs), + }); + Ok(()) + }) + .await + } + "update" => { + let mode_str = optional_string_field(payload, "mode")?; + let parsed_mode = match mode_str.as_ref() { + Some(m) => serde_json::from_value::( + serde_json::Value::String(m.clone()), + ) + .map_err(|_| { + ( + 400u16, + serde_json::json!({"ok": false, "error": format!( + "mode must be 'disabled', 'check-only', or 'check-and-download', got '{}'", m + )}) + .to_string(), + ) + })?, + None => rt_updater::UpdateMode::default(), + }; + update_config_file(config_state, subsystem, ui_tx, |raw| { + raw.update = Some(crate::config::RawUpdateConfig { mode: mode_str }); + Ok(()) + }) + .await?; + subsystem.lock().await.update_mode = parsed_mode; + Ok(()) + } + "ups" => { + let enabled = optional_bool_field(payload, "enabled")?; + let daemon_addr = optional_string_field(payload, "daemon_addr")?.and_then(|addr| { + let trimmed = addr.trim(); + (!trimmed.is_empty()).then(|| trimmed.to_owned()) + }); + let poll_interval_secs = optional_u64_field(payload, "poll_interval_secs")?; + let upstream_heartbeat_secs = optional_u64_field(payload, "upstream_heartbeat_secs")?; + + if let Some(interval) = poll_interval_secs + && !(1..=60).contains(&interval) + { + return Err(bad_request_error( + "poll_interval_secs must be between 1 and 60", + )); + } + if let Some(heartbeat) = upstream_heartbeat_secs + && !(10..=300).contains(&heartbeat) + { + return Err(bad_request_error( + "upstream_heartbeat_secs must be between 10 and 300", + )); + } + if let Some(ref addr) = daemon_addr + && addr.parse::().is_err() + { + let parts: Vec<&str> = addr.rsplitn(2, ':').collect(); + if parts.len() != 2 || parts[0].parse::().is_err() { + return Err(bad_request_error(format!( + "daemon_addr must be a valid host:port, got '{}'", + addr + ))); + } + } + + update_config_file(config_state, subsystem, ui_tx, |raw| { + raw.ups = Some(crate::config::RawUpsConfig { + enabled, + daemon_addr, + poll_interval_secs, + upstream_heartbeat_secs, + }); + Ok(()) + }) + .await + } + "readers" => { + let readers_val = payload.get("readers").ok_or_else(|| { + ( + 400u16, + serde_json::json!({"ok": false, "error": "readers field is required"}) + .to_string(), + ) + })?; + let readers_arr = readers_val.as_array().ok_or_else(|| { + ( + 400u16, + serde_json::json!({"ok": false, "error": "readers must be an array"}) + .to_string(), + ) + })?; + + if readers_arr.is_empty() { + return Err(( + 400u16, + "{\"ok\":false,\"error\":\"at least one reader is required\"}".to_owned(), + )); + } + + let mut raw_readers = Vec::with_capacity(readers_arr.len()); + for (i, entry) in readers_arr.iter().enumerate() { + let target = optional_string_field(entry, "target")?; + + let target_str = match &target { + Some(t) => t, + None => { + return Err(( + 400u16, + serde_json::json!({"ok": false, "error": format!("readers[{}].target is required", i)}).to_string(), + )); + } + }; + + if let Err(e) = crate::discovery::expand_target(target_str) { + return Err(( + 400u16, + serde_json::json!({"ok": false, "error": format!("readers[{}].target invalid: {}", i, e)}).to_string(), + )); + } + + let enabled = optional_bool_field(entry, "enabled")?; + let local_fallback_port = optional_u16_field(entry, "local_fallback_port")?; + + raw_readers.push(crate::config::RawReaderConfig { + target, + enabled, + local_fallback_port, + }); + } + + update_config_file(config_state, subsystem, ui_tx, |raw| { + raw.readers = Some(raw_readers); + Ok(()) + }) + .await + } + #[cfg(any(feature = "eink", feature = "lcd"))] + "screen" => { + let parsed = serde_json::from_value::(payload.clone()) + .map_err(|e| bad_request_error(e.to_string()))?; + // Validate before persisting so the endpoint can't accept values the + // loader would later reject (which would make the forwarder fail to + // boot on the very restart this change requests). + crate::config::validate_screen_config(&parsed) + .map_err(|e| bad_request_error(e.to_string()))?; + update_config_file(config_state, subsystem, ui_tx, |raw| { + raw.screen = Some(parsed); + Ok(()) + }) + .await + } + _ => Err(( + 400u16, + serde_json::json!({"ok": false, "error": format!("unknown section: {}", section)}) + .to_string(), + )), + } +} + +fn optional_string_field( + payload: &serde_json::Value, + field: &str, +) -> Result, (u16, String)> { + match payload.get(field) { + None | Some(serde_json::Value::Null) => Ok(None), + Some(serde_json::Value::String(s)) => Ok(Some(s.clone())), + Some(_) => Err(bad_request_error(format!( + "{} must be a string or null", + field + ))), + } +} + +fn optional_bool_field( + payload: &serde_json::Value, + field: &str, +) -> Result, (u16, String)> { + match payload.get(field) { + None | Some(serde_json::Value::Null) => Ok(None), + Some(serde_json::Value::Bool(b)) => Ok(Some(*b)), + Some(_) => Err(bad_request_error(format!( + "{} must be a boolean or null", + field + ))), + } +} + +fn optional_u64_field( + payload: &serde_json::Value, + field: &str, +) -> Result, (u16, String)> { + match payload.get(field) { + None | Some(serde_json::Value::Null) => Ok(None), + Some(value) => { + let raw = value.as_u64().ok_or_else(|| { + bad_request_error(format!("{} must be a non-negative integer or null", field)) + })?; + Ok(Some(raw)) + } + } +} + +fn optional_u16_field( + payload: &serde_json::Value, + field: &str, +) -> Result, (u16, String)> { + let raw = optional_u64_field(payload, field)?; + raw.map(|value| { + u16::try_from(value) + .map_err(|_| bad_request_error(format!("{} must be <= {}", field, u16::MAX))) + }) + .transpose() +} + +fn optional_u8_field( + payload: &serde_json::Value, + field: &str, +) -> Result, (u16, String)> { + let raw = optional_u64_field(payload, field)?; + raw.map(|value| { + u8::try_from(value) + .map_err(|_| bad_request_error(format!("{} must be <= {}", field, u8::MAX))) + }) + .transpose() +} + +fn require_non_empty_trimmed(field: &str, value: Option) -> Result { + let raw = value.ok_or_else(|| format!("{} is required", field))?; + let trimmed = raw.trim(); + if trimmed.is_empty() { + return Err(format!("{} must not be empty", field)); + } + Ok(trimmed.to_owned()) +} + +fn validate_token_file(token_file: &str) -> Result<(), String> { + if token_file.contains('\n') || token_file.contains('\r') { + return Err("token_file must be a single-line path".to_owned()); + } + Ok(()) +} + +fn validate_status_bind(bind: &str) -> Result<(), String> { + bind.parse::() + .map(|_| ()) + .map_err(|_| "bind must be a valid IPv4 address with port (e.g. 127.0.0.1:8080)".to_owned()) +} + +fn validate_server_url(url: &str) -> Result<(), String> { + if url.starts_with("http://") || url.starts_with("https://") { + Ok(()) + } else { + Err("server_url must start with http:// or https://".to_owned()) + } +} diff --git a/services/forwarder/src/lib.rs b/services/forwarder/src/lib.rs index f16a8864..27811e10 100644 --- a/services/forwarder/src/lib.rs +++ b/services/forwarder/src/lib.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; pub mod config; +pub mod config_service; pub mod discovery; pub mod local_fanout; pub mod p2p; diff --git a/services/forwarder/src/main.rs b/services/forwarder/src/main.rs index 24a44477..147239fd 100644 --- a/services/forwarder/src/main.rs +++ b/services/forwarder/src/main.rs @@ -3,11 +3,11 @@ // Runtime event loop: wires together journal, local fanout, IPICO TCP readers, // the P2P endpoint, and the status HTTP server. +use forwarder::config_service::ConfigState; use forwarder::discovery::expand_target; use forwarder::local_fanout::FanoutServer; use forwarder::status_http::{ - ConfigState, ForwarderStatusEvent, ReaderConnectionState, StatusConfig, StatusServer, - SubsystemStatus, + ForwarderStatusEvent, ReaderConnectionState, StatusConfig, StatusServer, SubsystemStatus, }; use forwarder::storage::journal::Journal; use rt_ui_log::UiLogLevel; diff --git a/services/forwarder/src/p2p/endpoint.rs b/services/forwarder/src/p2p/endpoint.rs index 8fffc459..12ef4ee6 100644 --- a/services/forwarder/src/p2p/endpoint.rs +++ b/services/forwarder/src/p2p/endpoint.rs @@ -738,7 +738,7 @@ mod tests { })); let handler = Arc::new(crate::p2p::ForwarderRemoteConfigHandler::new( true, - Arc::new(crate::status_http::ConfigState::new(config_path)), + Arc::new(crate::config_service::ConfigState::new(config_path)), Arc::new(Mutex::new(SubsystemStatus::ready())), ui_tx, ui_logger, @@ -837,7 +837,7 @@ mod tests { })); let handler = Arc::new(crate::p2p::ForwarderRemoteConfigHandler::new( true, - Arc::new(crate::status_http::ConfigState::new(config_path)), + Arc::new(crate::config_service::ConfigState::new(config_path)), Arc::new(Mutex::new(SubsystemStatus::ready())), ui_tx, ui_logger, diff --git a/services/forwarder/src/p2p/remote_config.rs b/services/forwarder/src/p2p/remote_config.rs index ca7ce679..0ec7378e 100644 --- a/services/forwarder/src/p2p/remote_config.rs +++ b/services/forwarder/src/p2p/remote_config.rs @@ -3,11 +3,11 @@ //! [`ForwarderRemoteConfigHandler`] serves the `ConfigGet` / `ConfigSet` / //! `Restart` verbs over the control stream, reusing the exact same config //! serialization, persistence, and restart mechanisms as the local status -//! HTTP surface (see [`crate::status_http`]): +//! HTTP surface (see [`crate::config_service`]): //! -//! - get -> [`crate::status_http::config_json_string`] (same body as +//! - get -> [`crate::config_service::config_json_string`] (same body as //! `GET /api/v1/config`), -//! - set -> [`crate::status_http::write_config_json_restricted`] (rejects +//! - set -> [`crate::config_service::write_config_json_restricted`] (rejects //! privileged section changes, parses the full config document, validates via //! the canonical loader, writes the TOML file atomically, and marks a restart //! as needed), @@ -30,9 +30,8 @@ use rt_p2p_protocol::{ }; use tokio::sync::{Mutex, Notify, broadcast}; -use crate::status_http::{ - ConfigState, SubsystemStatus, config_json_string, write_config_json_restricted, -}; +use crate::config_service::{ConfigState, config_json_string, write_config_json_restricted}; +use crate::status_http::SubsystemStatus; use crate::ui_events::ForwarderUiEvent; use super::control::{ diff --git a/services/forwarder/src/status_http.rs b/services/forwarder/src/status_http.rs index e67b1aca..2b05bc6e 100644 --- a/services/forwarder/src/status_http.rs +++ b/services/forwarder/src/status_http.rs @@ -46,6 +46,7 @@ //! # Security //! No authentication in v1. +use crate::config_service::ConfigState; use crate::storage::journal::Journal; use axum::Router; use axum::body::Bytes; @@ -61,8 +62,7 @@ use serde::de::DeserializeOwned; use std::collections::{HashMap, HashSet}; use std::convert::Infallible; use std::future::Future; -use std::io::Write as _; -use std::net::{SocketAddr, SocketAddrV4}; +use std::net::SocketAddr; use std::pin::Pin; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -420,21 +420,6 @@ pub struct StatusServer { cpu_temp: Arc>>, } -/// Holds the config file path and a write lock for read-modify-write operations. -pub struct ConfigState { - pub path: std::path::PathBuf, - pub(crate) write_lock: Mutex<()>, -} - -impl ConfigState { - pub fn new(path: std::path::PathBuf) -> Self { - ConfigState { - path, - write_lock: Mutex::new(()), - } - } -} - struct AppState { subsystem: Arc>, journal: Arc>, @@ -1256,127 +1241,7 @@ impl JournalAccess for NoJournal { // Helpers // --------------------------------------------------------------------------- -fn write_atomic(path: &std::path::Path, content: &str) -> std::io::Result<()> { - let original_permissions = std::fs::metadata(path).map(|m| m.permissions()).ok(); - - let parent = path.parent().ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::InvalidInput, - format!("path has no parent: {}", path.display()), - ) - })?; - let file_name = path.file_name().ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::InvalidInput, - format!("path has no file name: {}", path.display()), - ) - })?; - - let file_name = file_name.to_string_lossy(); - let pid = std::process::id(); - - for attempt in 0..=16 { - let tmp_name = format!(".{}.tmp.{}.{}", file_name, pid, attempt); - let tmp_path = parent.join(tmp_name); - match std::fs::OpenOptions::new() - .create_new(true) - .write(true) - .open(&tmp_path) - { - Ok(mut temp_file) => { - let result = (|| -> std::io::Result<()> { - temp_file.write_all(content.as_bytes())?; - temp_file.sync_all()?; - - if let Some(perms) = &original_permissions { - std::fs::set_permissions(&tmp_path, perms.clone())?; - } - - std::fs::rename(&tmp_path, path)?; - if let Ok(parent_dir) = std::fs::File::open(parent) { - let _ = parent_dir.sync_all(); - } - Ok(()) - })(); - if result.is_err() { - let _ = std::fs::remove_file(&tmp_path); - } - return result; - } - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, - Err(e) => return Err(e), - } - } - - Err(std::io::Error::new( - std::io::ErrorKind::AlreadyExists, - format!("failed to allocate temp path for {}", path.display()), - )) -} - -/// Read the TOML config file, apply a mutation, and write it back. -/// -/// Returns Ok(()) on success or Err((status_code, json_error_body)) on failure. -async fn update_config_file( - config_state: &ConfigState, - subsystem: &Arc>, - ui_tx: &tokio::sync::broadcast::Sender, - mutate: impl FnOnce(&mut crate::config::RawConfig) -> Result<(), String>, -) -> Result<(), (u16, String)> { - let _lock = config_state.write_lock.lock().await; - - let toml_str = std::fs::read_to_string(&config_state.path).map_err(|e| { - ( - 500u16, - serde_json::json!({"ok": false, "error": format!("File read error: {}", e)}) - .to_string(), - ) - })?; - - let mut raw: crate::config::RawConfig = toml::from_str(&toml_str).map_err(|e| { - ( - 500u16, - serde_json::json!({"ok": false, "error": format!("TOML parse error: {}", e)}) - .to_string(), - ) - })?; - - mutate(&mut raw).map_err(|e| { - ( - 400u16, - serde_json::json!({"ok": false, "error": e}).to_string(), - ) - })?; - - let new_toml = toml::to_string_pretty(&raw).map_err(|e| { - ( - 500u16, - serde_json::json!({"ok": false, "error": format!("TOML serialize error: {}", e)}) - .to_string(), - ) - })?; - - crate::config::load_config_from_str(&new_toml, &config_state.path).map_err(|e| { - ( - 400u16, - serde_json::json!({"ok": false, "error": format!("config validation failed: {e}")}) - .to_string(), - ) - })?; - - write_atomic(&config_state.path, &new_toml).map_err(|e| { - ( - 500u16, - serde_json::json!({"ok": false, "error": format!("File write error: {}", e)}) - .to_string(), - ) - })?; - - mark_restart_needed_and_emit(subsystem, ui_tx).await; - Ok(()) -} - -async fn mark_restart_needed_and_emit( +pub(crate) async fn mark_restart_needed_and_emit( subsystem: &Arc>, ui_tx: &tokio::sync::broadcast::Sender, ) { @@ -1389,492 +1254,6 @@ async fn mark_restart_needed_and_emit( }); } -/// Apply a config section update by name. -/// -/// Dispatches to the right mutation logic based on `section`, validates the -/// payload, and calls `update_config_file` to persist the change. -/// -/// Recognised sections: `"general"`, `"auth"`, `"journal"`, `"status_http"`, -/// `"control"`, `"update"`, `"p2p"`, `"ups"`, `"readers"`, and `"screen"`. -/// Screen config changes require a restart to apply. -pub async fn apply_section_update( - section: &str, - payload: &serde_json::Value, - config_state: &ConfigState, - subsystem: &Arc>, - ui_tx: &tokio::sync::broadcast::Sender, - logger: Option<&rt_ui_log::UiLogger>, -) -> Result<(), (u16, String)> { - require_object_payload(payload)?; - - match section { - "general" => { - let display_name = optional_string_field(payload, "display_name")?; - update_config_file(config_state, subsystem, ui_tx, |raw| { - raw.display_name = display_name; - Ok(()) - }) - .await - } - "auth" => { - let token_file_opt = optional_string_field(payload, "token_file")?; - let token_file = require_non_empty_trimmed("token_file", token_file_opt) - .map_err(bad_request_error)?; - validate_token_file(&token_file).map_err(bad_request_error)?; - update_config_file(config_state, subsystem, ui_tx, |raw| { - raw.auth = Some(crate::config::RawAuthConfig { - token_file: Some(token_file), - }); - Ok(()) - }) - .await - } - "journal" => { - let sqlite_path = optional_string_field(payload, "sqlite_path")?; - let prune_watermark_pct = optional_u8_field(payload, "prune_watermark_pct")?; - let min_retention = optional_string_field(payload, "min_retention")?; - let max_retention = optional_string_field(payload, "max_retention")?; - let emergency_free_disk_bytes = - optional_u64_field(payload, "emergency_free_disk_bytes")?; - let emergency_max_rows = optional_u64_field(payload, "emergency_max_rows")? - .map(|value| { - i64::try_from(value) - .map_err(|_| bad_request_error("emergency_max_rows must be <= i64::MAX")) - }) - .transpose()?; - if let Some(pct) = prune_watermark_pct - && pct > 100 - { - return Err(bad_request_error( - "prune_watermark_pct must be between 0 and 100", - )); - } - crate::config::validate_retention_settings( - min_retention.as_deref(), - max_retention.as_deref(), - emergency_max_rows, - ) - .map_err(bad_request_error)?; - update_config_file(config_state, subsystem, ui_tx, |raw| { - raw.journal = Some(crate::config::RawJournalConfig { - sqlite_path, - prune_watermark_pct, - min_retention, - max_retention, - emergency_free_disk_bytes, - emergency_max_rows, - }); - Ok(()) - }) - .await - } - "status_http" => { - let bind = optional_string_field(payload, "bind")?.and_then(|s| { - let trimmed = s.trim(); - if trimmed.is_empty() { - None - } else { - Some(trimmed.to_owned()) - } - }); - if let Some(ref bind_addr) = bind { - validate_status_bind(bind_addr).map_err(bad_request_error)?; - } - update_config_file(config_state, subsystem, ui_tx, |raw| { - raw.status_http = Some(crate::config::RawStatusHttpConfig { bind }); - Ok(()) - }) - .await - } - "control" => { - let allow_power_actions = optional_bool_field(payload, "allow_power_actions")?; - let action = optional_string_field(payload, "action")?; - if let Some(action) = action { - return apply_control_action_from_config(&action, config_state, logger).await; - } - update_config_file(config_state, subsystem, ui_tx, |raw| { - // Preserve existing P2P control settings; this handler only - // mutates allow_power_actions. - let allow_remote_config = raw.control.as_ref().and_then(|c| c.allow_remote_config); - let allow_reader_control = - raw.control.as_ref().and_then(|c| c.allow_reader_control); - raw.control = Some(crate::config::RawControlConfig { - allow_power_actions, - allow_remote_config, - allow_reader_control, - }); - Ok(()) - }) - .await - } - "p2p" => { - let enabled = optional_bool_field(payload, "enabled")?; - let server_url = optional_string_field(payload, "server_url")?.and_then(|url| { - let trimmed = url.trim(); - (!trimmed.is_empty()).then(|| trimmed.to_owned()) - }); - let server_token_file = - optional_string_field(payload, "server_token_file")?.and_then(|path| { - let trimmed = path.trim(); - (!trimmed.is_empty()).then(|| trimmed.to_owned()) - }); - if let Some(ref url) = server_url { - validate_server_url(url).map_err(bad_request_error)?; - } - if let Some(ref token_file) = server_token_file { - validate_token_file(token_file).map_err(bad_request_error)?; - } - update_config_file(config_state, subsystem, ui_tx, |raw| { - let previous = raw.p2p.take(); - raw.p2p = Some(crate::config::RawP2pConfig { - enabled, - secret_key_path: previous - .as_ref() - .and_then(|cfg| cfg.secret_key_path.clone()), - secret_key_seed_hex: previous - .as_ref() - .and_then(|cfg| cfg.secret_key_seed_hex.clone()), - bind_addr_v4: previous.as_ref().and_then(|cfg| cfg.bind_addr_v4.clone()), - relay_disabled: previous.as_ref().and_then(|cfg| cfg.relay_disabled), - discovery_disabled: previous.as_ref().and_then(|cfg| cfg.discovery_disabled), - max_concurrent_bidi_streams: previous - .as_ref() - .and_then(|cfg| cfg.max_concurrent_bidi_streams), - static_allowed_receivers: previous - .as_ref() - .and_then(|cfg| cfg.static_allowed_receivers.clone()), - allowlist_cache_path: previous - .as_ref() - .and_then(|cfg| cfg.allowlist_cache_path.clone()), - server_url, - server_token_file, - device_token_file: previous - .as_ref() - .and_then(|cfg| cfg.device_token_file.clone()), - allowlist_poll_interval_secs: previous - .as_ref() - .and_then(|cfg| cfg.allowlist_poll_interval_secs), - allowlist_request_timeout_secs: previous - .as_ref() - .and_then(|cfg| cfg.allowlist_request_timeout_secs), - }); - Ok(()) - }) - .await - } - "update" => { - let mode_str = optional_string_field(payload, "mode")?; - let parsed_mode = match mode_str.as_ref() { - Some(m) => serde_json::from_value::( - serde_json::Value::String(m.clone()), - ) - .map_err(|_| { - ( - 400u16, - serde_json::json!({"ok": false, "error": format!( - "mode must be 'disabled', 'check-only', or 'check-and-download', got '{}'", m - )}) - .to_string(), - ) - })?, - None => rt_updater::UpdateMode::default(), - }; - update_config_file(config_state, subsystem, ui_tx, |raw| { - raw.update = Some(crate::config::RawUpdateConfig { mode: mode_str }); - Ok(()) - }) - .await?; - subsystem.lock().await.update_mode = parsed_mode; - Ok(()) - } - "ups" => { - let enabled = optional_bool_field(payload, "enabled")?; - let daemon_addr = optional_string_field(payload, "daemon_addr")?.and_then(|addr| { - let trimmed = addr.trim(); - (!trimmed.is_empty()).then(|| trimmed.to_owned()) - }); - let poll_interval_secs = optional_u64_field(payload, "poll_interval_secs")?; - let upstream_heartbeat_secs = optional_u64_field(payload, "upstream_heartbeat_secs")?; - - if let Some(interval) = poll_interval_secs - && !(1..=60).contains(&interval) - { - return Err(bad_request_error( - "poll_interval_secs must be between 1 and 60", - )); - } - if let Some(heartbeat) = upstream_heartbeat_secs - && !(10..=300).contains(&heartbeat) - { - return Err(bad_request_error( - "upstream_heartbeat_secs must be between 10 and 300", - )); - } - if let Some(ref addr) = daemon_addr - && addr.parse::().is_err() - { - let parts: Vec<&str> = addr.rsplitn(2, ':').collect(); - if parts.len() != 2 || parts[0].parse::().is_err() { - return Err(bad_request_error(format!( - "daemon_addr must be a valid host:port, got '{}'", - addr - ))); - } - } - - update_config_file(config_state, subsystem, ui_tx, |raw| { - raw.ups = Some(crate::config::RawUpsConfig { - enabled, - daemon_addr, - poll_interval_secs, - upstream_heartbeat_secs, - }); - Ok(()) - }) - .await - } - "readers" => { - let readers_val = payload.get("readers").ok_or_else(|| { - ( - 400u16, - serde_json::json!({"ok": false, "error": "readers field is required"}) - .to_string(), - ) - })?; - let readers_arr = readers_val.as_array().ok_or_else(|| { - ( - 400u16, - serde_json::json!({"ok": false, "error": "readers must be an array"}) - .to_string(), - ) - })?; - - if readers_arr.is_empty() { - return Err(( - 400u16, - "{\"ok\":false,\"error\":\"at least one reader is required\"}".to_owned(), - )); - } - - let mut raw_readers = Vec::with_capacity(readers_arr.len()); - for (i, entry) in readers_arr.iter().enumerate() { - let target = optional_string_field(entry, "target")?; - - let target_str = match &target { - Some(t) => t, - None => { - return Err(( - 400u16, - serde_json::json!({"ok": false, "error": format!("readers[{}].target is required", i)}).to_string(), - )); - } - }; - - if let Err(e) = crate::discovery::expand_target(target_str) { - return Err(( - 400u16, - serde_json::json!({"ok": false, "error": format!("readers[{}].target invalid: {}", i, e)}).to_string(), - )); - } - - let enabled = optional_bool_field(entry, "enabled")?; - let local_fallback_port = optional_u16_field(entry, "local_fallback_port")?; - - raw_readers.push(crate::config::RawReaderConfig { - target, - enabled, - local_fallback_port, - }); - } - - update_config_file(config_state, subsystem, ui_tx, |raw| { - raw.readers = Some(raw_readers); - Ok(()) - }) - .await - } - #[cfg(any(feature = "eink", feature = "lcd"))] - "screen" => { - let parsed = serde_json::from_value::(payload.clone()) - .map_err(|e| bad_request_error(e.to_string()))?; - // Validate before persisting so the endpoint can't accept values the - // loader would later reject (which would make the forwarder fail to - // boot on the very restart this change requests). - crate::config::validate_screen_config(&parsed) - .map_err(|e| bad_request_error(e.to_string()))?; - update_config_file(config_state, subsystem, ui_tx, |raw| { - raw.screen = Some(parsed); - Ok(()) - }) - .await - } - _ => Err(( - 400u16, - serde_json::json!({"ok": false, "error": format!("unknown section: {}", section)}) - .to_string(), - )), - } -} - -/// Read the config TOML file as a JSON value plus the current `restart_needed` -/// state, returning a plain error message on failure. -/// -/// Shared core for both the HTTP `GET /api/v1/config` endpoint and the P2P -/// remote-config get verb, so the two always agree on the serialized shape. -async fn read_config_value( - config_state: &ConfigState, - subsystem: &Arc>, -) -> Result<(serde_json::Value, bool), String> { - let _lock = config_state.write_lock.lock().await; - - let toml_str = - std::fs::read_to_string(&config_state.path).map_err(|e| format!("File read error: {e}"))?; - - let raw: crate::config::RawConfig = - toml::from_str(&toml_str).map_err(|e| format!("TOML parse error: {e}"))?; - - let json = serde_json::to_value(&raw).map_err(|e| format!("JSON serialize error: {e}"))?; - - let restart_needed = subsystem.lock().await.restart_needed(); - Ok((json, restart_needed)) -} - -/// Read the config TOML file as JSON. -/// -/// Returns `(config_json, restart_needed)` on success. -pub async fn read_config_json( - config_state: &ConfigState, - subsystem: &Arc>, -) -> Result<(serde_json::Value, bool), (u16, String)> { - read_config_value(config_state, subsystem) - .await - .map_err(|e| { - ( - 500u16, - serde_json::json!({"ok": false, "error": e}).to_string(), - ) - }) -} - -/// Serialize the current config to a JSON string (identical to the body -/// `GET /api/v1/config` returns) plus the current `restart_needed` state. -/// -/// Used by the P2P remote-config get verb so the receiver UI round-trips the -/// same document whether it reads config over HTTP or P2P. -pub async fn config_json_string( - config_state: &ConfigState, - subsystem: &Arc>, -) -> Result<(String, bool), String> { - let (value, restart_needed) = read_config_value(config_state, subsystem).await?; - let json = serde_json::to_string(&value).map_err(|e| format!("JSON serialize error: {e}"))?; - Ok((json, restart_needed)) -} - -/// Sections a P2P peer may never change: identity/credentials (`auth`), -/// transport trust (`p2p`), and the gates themselves (`control`). -/// -/// Boundary rationale: `[auth]`/`[p2p]`/`[control]` changes are trust/identity -/// escalation — permanent and self-granting; `[journal]`, `[[readers]]`, and -/// display fields are the operational surface remote management exists to serve -/// — a hostile allow-listed receiver can disrupt operations through them but -/// cannot expand its own access. `[status_http]`, `[update]`, and `[ups]` were -/// considered and left writable as operational surface; `status_http.bind` is -/// the borderline case (a receiver could rebind the local admin HTTP), but it -/// grants the receiver itself no access. -const REMOTE_PROTECTED_SECTIONS: &[&str] = &["auth", "p2p", "control"]; - -/// Persist a full config document received from P2P remote config, rejecting -/// any attempted change to privileged config sections before writing. -/// -/// The document is parsed into the same [`crate::config::RawConfig`] the get -/// path serializes, re-serialized to TOML, and validated by running the -/// canonical loader before anything is written — so a document that would fail -/// to load on restart is rejected without corrupting the on-disk file. Reuses -/// the same atomic writer and `restart_needed` signal as the per-section HTTP -/// writers. Returns a plain error message on failure. -/// -/// There is intentionally no unrestricted full-document writer: local -/// (trusted) config edits go through the per-section HTTP handlers -/// ([`apply_section_update`] / [`update_config_file`]), so every full-document -/// write path enforces [`REMOTE_PROTECTED_SECTIONS`]. -pub async fn write_config_json_restricted( - config_json: &str, - config_state: &ConfigState, - subsystem: &Arc>, - ui_tx: &tokio::sync::broadcast::Sender, -) -> Result<(), String> { - let incoming: crate::config::RawConfig = - serde_json::from_str(config_json).map_err(|e| format!("invalid config JSON: {e}"))?; - - let _lock = config_state.write_lock.lock().await; - - let current_toml = - std::fs::read_to_string(&config_state.path).map_err(|e| format!("File read error: {e}"))?; - let current: crate::config::RawConfig = - toml::from_str(¤t_toml).map_err(|e| format!("TOML parse error: {e}"))?; - - let incoming_value = - serde_json::to_value(&incoming).map_err(|e| format!("JSON serialize error: {e}"))?; - let current_value = - serde_json::to_value(¤t).map_err(|e| format!("JSON serialize error: {e}"))?; - - for section in REMOTE_PROTECTED_SECTIONS { - if normalize_section(incoming_value.get(section)) - != normalize_section(current_value.get(section)) - { - return Err(format!( - "remote config may not modify the protected [{section}] section" - )); - } - } - - write_config_json_locked(incoming, config_state, subsystem, ui_tx).await -} - -/// Shared locked write body. Caller must hold `config_state.write_lock`. -/// -/// Takes the already-parsed [`crate::config::RawConfig`] (the same value the -/// protected-section comparison ran against) so "what was compared is what is -/// written" holds structurally rather than by re-parsing the same string. -async fn write_config_json_locked( - raw: crate::config::RawConfig, - config_state: &ConfigState, - subsystem: &Arc>, - ui_tx: &tokio::sync::broadcast::Sender, -) -> Result<(), String> { - let new_toml = - toml::to_string_pretty(&raw).map_err(|e| format!("TOML serialize error: {e}"))?; - - // Validate via the canonical loader so we never persist a config that would - // fail to load on the next restart. - crate::config::load_config_from_str(&new_toml, &config_state.path) - .map_err(|e| format!("config validation failed: {e}"))?; - - write_atomic(&config_state.path, &new_toml).map_err(|e| format!("File write error: {e}"))?; - - mark_restart_needed_and_emit(subsystem, ui_tx).await; - Ok(()) -} - -/// A missing section, `null`, and an all-null object are equivalent RawConfig -/// states. -/// -/// Normalization is single-level by design: every protected raw section is a -/// flat struct of scalars/`Vec` today. If a nested struct is ever -/// added to a protected section, an all-null nested object will not be -/// flattened — the failure mode is a false *reject* (fail-closed), and the -/// populated-section round-trip test in `p2p/remote_config.rs` will catch -/// serialization drift. -fn normalize_section(v: Option<&serde_json::Value>) -> serde_json::Value { - match v { - None | Some(serde_json::Value::Null) => serde_json::Value::Null, - Some(serde_json::Value::Object(map)) if map.values().all(serde_json::Value::is_null) => { - serde_json::Value::Null - } - Some(other) => other.clone(), - } -} - fn text_response(status: StatusCode, body: impl Into) -> Response { (status, [(header::CONTENT_TYPE, "text/plain")], body.into()).into_response() } @@ -1887,14 +1266,14 @@ fn parse_json_body(body: &Bytes) -> Result { serde_json::from_slice::(body).map_err(|e| format!("Invalid JSON: {}", e)) } -fn bad_request_error(message: impl Into) -> (u16, String) { +pub(crate) fn bad_request_error(message: impl Into) -> (u16, String) { ( 400u16, serde_json::json!({"ok": false, "error": message.into()}).to_string(), ) } -fn require_object_payload(payload: &serde_json::Value) -> Result<(), (u16, String)> { +pub(crate) fn require_object_payload(payload: &serde_json::Value) -> Result<(), (u16, String)> { if payload.is_object() { Ok(()) } else { @@ -1902,103 +1281,6 @@ fn require_object_payload(payload: &serde_json::Value) -> Result<(), (u16, Strin } } -fn optional_string_field( - payload: &serde_json::Value, - field: &str, -) -> Result, (u16, String)> { - match payload.get(field) { - None | Some(serde_json::Value::Null) => Ok(None), - Some(serde_json::Value::String(s)) => Ok(Some(s.clone())), - Some(_) => Err(bad_request_error(format!( - "{} must be a string or null", - field - ))), - } -} - -fn optional_bool_field( - payload: &serde_json::Value, - field: &str, -) -> Result, (u16, String)> { - match payload.get(field) { - None | Some(serde_json::Value::Null) => Ok(None), - Some(serde_json::Value::Bool(b)) => Ok(Some(*b)), - Some(_) => Err(bad_request_error(format!( - "{} must be a boolean or null", - field - ))), - } -} - -fn optional_u64_field( - payload: &serde_json::Value, - field: &str, -) -> Result, (u16, String)> { - match payload.get(field) { - None | Some(serde_json::Value::Null) => Ok(None), - Some(value) => { - let raw = value.as_u64().ok_or_else(|| { - bad_request_error(format!("{} must be a non-negative integer or null", field)) - })?; - Ok(Some(raw)) - } - } -} - -fn optional_u16_field( - payload: &serde_json::Value, - field: &str, -) -> Result, (u16, String)> { - let raw = optional_u64_field(payload, field)?; - raw.map(|value| { - u16::try_from(value) - .map_err(|_| bad_request_error(format!("{} must be <= {}", field, u16::MAX))) - }) - .transpose() -} - -fn optional_u8_field( - payload: &serde_json::Value, - field: &str, -) -> Result, (u16, String)> { - let raw = optional_u64_field(payload, field)?; - raw.map(|value| { - u8::try_from(value) - .map_err(|_| bad_request_error(format!("{} must be <= {}", field, u8::MAX))) - }) - .transpose() -} - -fn require_non_empty_trimmed(field: &str, value: Option) -> Result { - let raw = value.ok_or_else(|| format!("{} is required", field))?; - let trimmed = raw.trim(); - if trimmed.is_empty() { - return Err(format!("{} must not be empty", field)); - } - Ok(trimmed.to_owned()) -} - -fn validate_token_file(token_file: &str) -> Result<(), String> { - if token_file.contains('\n') || token_file.contains('\r') { - return Err("token_file must be a single-line path".to_owned()); - } - Ok(()) -} - -fn validate_status_bind(bind: &str) -> Result<(), String> { - bind.parse::() - .map(|_| ()) - .map_err(|_| "bind must be a valid IPv4 address with port (e.g. 127.0.0.1:8080)".to_owned()) -} - -fn validate_server_url(url: &str) -> Result<(), String> { - if url.starts_with("http://") || url.starts_with("https://") { - Ok(()) - } else { - Err("server_url must start with http:// or https://".to_owned()) - } -} - fn config_not_available() -> Response { text_response(StatusCode::NOT_FOUND, "Config editing not available") } @@ -2195,7 +1477,7 @@ async fn run_device_power_action( )) } -async fn apply_control_action_from_config( +pub(crate) async fn apply_control_action_from_config( action: &str, config_state: &ConfigState, logger: Option<&rt_ui_log::UiLogger>, @@ -3631,7 +2913,7 @@ async fn config_json_handler( None => return config_not_available(), }; - match read_config_json(&cs, &state.subsystem).await { + match crate::config_service::read_config_json(&cs, &state.subsystem).await { Ok((json_value, _restart_needed)) => { let json_str = serde_json::to_string(&json_value).unwrap_or_else(|e| { serde_json::json!({"ok": false, "error": format!("JSON serialize error: {}", e)}) @@ -3666,7 +2948,7 @@ async fn post_config_section_handler( } }; - match apply_section_update( + match crate::config_service::apply_section_update( section, &payload, &cs, diff --git a/services/forwarder/tests/config_endpoints.rs b/services/forwarder/tests/config_endpoints.rs index d641675e..afcc980d 100644 --- a/services/forwarder/tests/config_endpoints.rs +++ b/services/forwarder/tests/config_endpoints.rs @@ -143,7 +143,7 @@ async fn restart_needed_flag_defaults_to_false() { #[tokio::test] async fn get_config_returns_json() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -199,7 +199,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_general_updates_display_name() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -270,7 +270,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_general_updates_readonly_config_via_atomic_replace() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -341,7 +341,7 @@ target = "192.168.1.100:10000" #[cfg(unix)] #[tokio::test] async fn post_config_general_preserves_file_mode_on_atomic_replace() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use std::os::unix::fs::PermissionsExt; use tempfile::NamedTempFile; @@ -407,7 +407,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_general_accepts_fragmented_http_body() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -471,7 +471,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_optional_sections_reject_non_object_payloads() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -528,7 +528,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_auth_updates_token_file() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -583,7 +583,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_auth_requires_token_file() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -625,7 +625,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_auth_rejects_whitespace_token_file() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -667,7 +667,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_journal_updates_sqlite_path() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -730,7 +730,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_journal_rejects_out_of_range_prune_watermark() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -780,7 +780,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_journal_rejects_non_numeric_prune_watermark() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -832,7 +832,7 @@ target = "192.168.1.100:10000" /// Returns the running server and the config file guard (kept to preserve the /// temp file for the duration of the test). async fn start_config_server() -> (StatusServer, tempfile::NamedTempFile, tempfile::TempDir) { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -987,7 +987,7 @@ async fn post_config_journal_accepts_valid_retention() { #[tokio::test] async fn post_config_status_http_updates_bind() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -1045,7 +1045,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_status_http_rejects_invalid_ipv4_and_port() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -1092,7 +1092,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_status_http_rejects_hostname_bind() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -1139,7 +1139,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_status_http_rejects_ipv6_bind() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -1186,7 +1186,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_readers_replaces_list() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -1255,7 +1255,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_readers_validates_target() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -1303,7 +1303,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_readers_rejects_out_of_range_local_fallback_port() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -1353,7 +1353,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_readers_requires_at_least_one() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -1395,7 +1395,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_control_updates_allow_power_actions() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -1462,7 +1462,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_control_rejects_non_boolean_allow_power_actions() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -1515,7 +1515,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn post_config_control_action_restart_device_requires_allow_power_actions_true() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -1570,7 +1570,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn restart_endpoint_returns_ok() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -1645,7 +1645,7 @@ async fn restart_endpoint_returns_404_without_config() { #[tokio::test] async fn control_restart_service_endpoint_returns_ok() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -1704,7 +1704,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn control_restart_device_requires_allow_power_actions_true() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -1754,7 +1754,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn control_shutdown_device_requires_allow_power_actions_true() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; @@ -1804,7 +1804,7 @@ target = "192.168.1.100:10000" #[tokio::test] async fn control_action_errors_are_written_to_ui_logs() { - use forwarder::status_http::ConfigState; + use forwarder::config_service::ConfigState; use std::io::Write; use tempfile::NamedTempFile; diff --git a/services/forwarder/tests/health_endpoints.rs b/services/forwarder/tests/health_endpoints.rs index beeabad5..1c577587 100644 --- a/services/forwarder/tests/health_endpoints.rs +++ b/services/forwarder/tests/health_endpoints.rs @@ -466,7 +466,7 @@ target = "10.0.0.1:10000" cfg, SubsystemStatus::ready(), Arc::new(tokio::sync::Mutex::new(NoJournalForNameApi)), - Arc::new(forwarder::status_http::ConfigState::new( + Arc::new(forwarder::config_service::ConfigState::new( config_file.path().to_path_buf(), )), Arc::new(tokio::sync::Notify::new()), @@ -518,7 +518,7 @@ target = "10.0.0.1:10000" cfg, SubsystemStatus::ready(), Arc::new(tokio::sync::Mutex::new(NoJournalForNameApi)), - Arc::new(forwarder::status_http::ConfigState::new( + Arc::new(forwarder::config_service::ConfigState::new( config_file.path().to_path_buf(), )), Arc::new(tokio::sync::Notify::new()), From fcb65e93876f531ff0cc8276305b75b3856dfa53 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 15:03:38 -0400 Subject: [PATCH 21/38] refactor(forwarder): make write_atomic private to config_service --- services/forwarder/src/config_service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/forwarder/src/config_service.rs b/services/forwarder/src/config_service.rs index c9b8ebd5..2242bf3b 100644 --- a/services/forwarder/src/config_service.rs +++ b/services/forwarder/src/config_service.rs @@ -31,7 +31,7 @@ impl ConfigState { } } -pub(crate) fn write_atomic(path: &std::path::Path, content: &str) -> std::io::Result<()> { +fn write_atomic(path: &std::path::Path, content: &str) -> std::io::Result<()> { let original_permissions = std::fs::metadata(path).map(|m| m.permissions()).ok(); let parent = path.parent().ok_or_else(|| { From 9fe222504bca623cb4e4e2a7af2e0493a872a969 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 15:22:01 -0400 Subject: [PATCH 22/38] refactor(forwarder): extract status store and decouple data plane --- services/forwarder/src/config_service.rs | 4 +- services/forwarder/src/lib.rs | 1 + services/forwarder/src/main.rs | 11 +- services/forwarder/src/p2p/endpoint.rs | 5 +- services/forwarder/src/p2p/mod.rs | 5 +- services/forwarder/src/p2p/reader_control.rs | 2 +- services/forwarder/src/p2p/remote_config.rs | 2 +- .../forwarder/src/reader_control_service.rs | 2 +- services/forwarder/src/reader_task.rs | 8 +- services/forwarder/src/status_http.rs | 822 +++-------------- services/forwarder/src/status_store.rs | 841 ++++++++++++++++++ services/forwarder/src/ups_task.rs | 46 +- services/forwarder/tests/config_endpoints.rs | 3 +- services/forwarder/tests/health_endpoints.rs | 5 +- services/forwarder/tests/ui_embed_fallback.rs | 3 +- 15 files changed, 997 insertions(+), 763 deletions(-) create mode 100644 services/forwarder/src/status_store.rs diff --git a/services/forwarder/src/config_service.rs b/services/forwarder/src/config_service.rs index 2242bf3b..2aecdbc4 100644 --- a/services/forwarder/src/config_service.rs +++ b/services/forwarder/src/config_service.rs @@ -12,9 +12,9 @@ use std::sync::Arc; use tokio::sync::Mutex; use crate::status_http::{ - SubsystemStatus, apply_control_action_from_config, bad_request_error, - mark_restart_needed_and_emit, require_object_payload, + apply_control_action_from_config, bad_request_error, require_object_payload, }; +use crate::status_store::{SubsystemStatus, mark_restart_needed_and_emit}; /// Holds the config file path and a write lock for read-modify-write operations. pub struct ConfigState { diff --git a/services/forwarder/src/lib.rs b/services/forwarder/src/lib.rs index 27811e10..9535ac77 100644 --- a/services/forwarder/src/lib.rs +++ b/services/forwarder/src/lib.rs @@ -14,6 +14,7 @@ pub mod reader_control_service; pub mod reader_task; pub mod replay; pub mod status_http; +pub mod status_store; pub mod storage; pub mod ui_events; pub mod ui_server; diff --git a/services/forwarder/src/main.rs b/services/forwarder/src/main.rs index 147239fd..5fdf0a1a 100644 --- a/services/forwarder/src/main.rs +++ b/services/forwarder/src/main.rs @@ -6,9 +6,8 @@ use forwarder::config_service::ConfigState; use forwarder::discovery::expand_target; use forwarder::local_fanout::FanoutServer; -use forwarder::status_http::{ - ForwarderStatusEvent, ReaderConnectionState, StatusConfig, StatusServer, SubsystemStatus, -}; +use forwarder::status_http::{StatusConfig, StatusServer}; +use forwarder::status_store::{ForwarderStatusEvent, ReaderConnectionState, SubsystemStatus}; use forwarder::storage::journal::Journal; use rt_ui_log::UiLogLevel; use sha2::{Digest, Sha256}; @@ -700,11 +699,13 @@ async fn main() { } } + let status_store = status_server.store(); + // Spawn reader tasks for (reader_ip, reader_port, fanout_addr) in fanout_addrs { let j = journal.clone(); let rx = shutdown_rx.clone(); - let ss = status_server.clone(); + let ss = status_store.clone(); let lg = logger.clone(); tokio::spawn(async move { run_reader(reader_ip, reader_port, fanout_addr, j, rx, ss, lg).await; @@ -713,7 +714,7 @@ async fn main() { // Spawn UPS monitoring task (if enabled) let ups_handle = if cfg.ups.enabled { - let ss = status_server.clone(); + let ss = status_store.clone(); let rx = shutdown_rx.clone(); let fwd_id = forwarder_id.clone(); Some(forwarder::ups_task::spawn_ups_task( diff --git a/services/forwarder/src/p2p/endpoint.rs b/services/forwarder/src/p2p/endpoint.rs index 12ef4ee6..3cd63280 100644 --- a/services/forwarder/src/p2p/endpoint.rs +++ b/services/forwarder/src/p2p/endpoint.rs @@ -15,7 +15,7 @@ use rt_iroh::{Connection, Endpoint, EndpointAddr, EndpointBuilder, EndpointId}; use rt_p2p_protocol::{CAP_CONTROL_EVENTS, has_capability}; use tokio::sync::{Mutex, broadcast, mpsc}; -use crate::status_http::{ +use crate::status_store::{ ForwarderStatusEvent, ForwarderStatusFeed, ForwarderStatusSnapshot, ReaderConnectionState, ReaderStatus, UpsStatusState, }; @@ -487,7 +487,8 @@ mod tests { use crate::p2p::control::{ PROTOCOL_MINOR, StaticCatalog, forwarder_hello, read_frame, write_frame, }; - use crate::status_http::{ReaderConnectionState, StatusConfig, StatusServer, SubsystemStatus}; + use crate::status_http::{StatusConfig, StatusServer}; + use crate::status_store::{ReaderConnectionState, SubsystemStatus}; use rt_p2p_protocol::{ CAP_CONTROL_EVENTS, ControlC2F, ControlF2C, DataC2F, DataF2C, DataSubscribe, HelloOk, StreamCatalog, SubscribeMode, control_c2f, control_f2c, data_c2f, data_f2c, has_capability, diff --git a/services/forwarder/src/p2p/mod.rs b/services/forwarder/src/p2p/mod.rs index 986f2236..f3da48ac 100644 --- a/services/forwarder/src/p2p/mod.rs +++ b/services/forwarder/src/p2p/mod.rs @@ -30,7 +30,7 @@ use std::sync::Arc; use std::time::Duration; use crate::config::P2pConfig; -use crate::status_http::ForwarderStatusFeed; +use crate::status_store::ForwarderStatusFeed; use crate::storage::journal::Journal; use rt_iroh::{EndpointAddr, EndpointBuilder, EndpointId, RelayMode, SecretKey}; use rt_p2p_protocol::{StreamCatalog, StreamEntry}; @@ -590,7 +590,8 @@ mod tests { use super::*; use crate::config::P2pConfig; - use crate::status_http::{StatusConfig, StatusServer, SubsystemStatus}; + use crate::status_http::{StatusConfig, StatusServer}; + use crate::status_store::SubsystemStatus; use crate::storage::journal::Journal; use rt_iroh::EndpointBuilder; use rt_p2p_protocol::{ diff --git a/services/forwarder/src/p2p/reader_control.rs b/services/forwarder/src/p2p/reader_control.rs index 5bed8a16..6b4a787e 100644 --- a/services/forwarder/src/p2p/reader_control.rs +++ b/services/forwarder/src/p2p/reader_control.rs @@ -258,7 +258,7 @@ mod tests { fn handler_with_reader_control_flag( allow_reader_control: bool, ) -> (ForwarderReaderControlHandler, tempfile::TempDir) { - let subsystem = Arc::new(Mutex::new(crate::status_http::SubsystemStatus::ready())); + let subsystem = Arc::new(Mutex::new(crate::status_store::SubsystemStatus::ready())); let control_clients = Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())); let download_trackers = Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())); let reconnect_notifies = Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())); diff --git a/services/forwarder/src/p2p/remote_config.rs b/services/forwarder/src/p2p/remote_config.rs index 0ec7378e..9927aad4 100644 --- a/services/forwarder/src/p2p/remote_config.rs +++ b/services/forwarder/src/p2p/remote_config.rs @@ -31,7 +31,7 @@ use rt_p2p_protocol::{ use tokio::sync::{Mutex, Notify, broadcast}; use crate::config_service::{ConfigState, config_json_string, write_config_json_restricted}; -use crate::status_http::SubsystemStatus; +use crate::status_store::SubsystemStatus; use crate::ui_events::ForwarderUiEvent; use super::control::{ diff --git a/services/forwarder/src/reader_control_service.rs b/services/forwarder/src/reader_control_service.rs index 56f2e70a..19590e0e 100644 --- a/services/forwarder/src/reader_control_service.rs +++ b/services/forwarder/src/reader_control_service.rs @@ -1,7 +1,7 @@ //! Shared reader-control operations used by HTTP and P2P control paths. use crate::reader_control::{ControlClient, DownloadTracker}; -use crate::status_http::{ForwarderStatusEvent, SubsystemStatus}; +use crate::status_store::{ForwarderStatusEvent, SubsystemStatus}; use ipico_core::control; use std::collections::HashMap; use std::sync::Arc; diff --git a/services/forwarder/src/reader_task.rs b/services/forwarder/src/reader_task.rs index 4bf7a99e..0b448dad 100644 --- a/services/forwarder/src/reader_task.rs +++ b/services/forwarder/src/reader_task.rs @@ -1,7 +1,7 @@ // reader_task: IPICO reader TCP connect/read loop, journal append, local fanout. use crate::local_fanout::FanoutServer; -use crate::status_http::{ReaderConnectionState, StatusServer}; +use crate::status_store::{ReaderConnectionState, StatusStore}; use crate::storage::journal::Journal; use crate::ui_events::ForwarderUiEvent; use ipico_core::read::ChipRead; @@ -19,13 +19,13 @@ use tracing::{debug, info, warn}; // Helpers // --------------------------------------------------------------------------- -pub(crate) async fn mark_reader_disconnected(status: &StatusServer, reader_ip: &str) { +pub(crate) async fn mark_reader_disconnected(status: &StatusStore, reader_ip: &str) { status .update_reader_state(reader_ip, ReaderConnectionState::Disconnected) .await; } -async fn disconnect_and_notify(status: &StatusServer, stream_key: &str) { +async fn disconnect_and_notify(status: &StatusStore, stream_key: &str) { mark_reader_disconnected(status, stream_key).await; } @@ -190,7 +190,7 @@ pub async fn run_reader( fanout_addr: SocketAddr, journal: Arc>, mut shutdown_rx: watch::Receiver, - status: StatusServer, + status: StatusStore, logger: Arc>, ) { let target_addr = format!("{}:{}", reader_ip, reader_port); diff --git a/services/forwarder/src/status_http.rs b/services/forwarder/src/status_http.rs index 2b05bc6e..3d8e34b7 100644 --- a/services/forwarder/src/status_http.rs +++ b/services/forwarder/src/status_http.rs @@ -47,6 +47,12 @@ //! No authentication in v1. use crate::config_service::ConfigState; +use crate::status_store::{ + ForwarderStatusEvent, ForwarderStatusFeed, ReaderConnectionState, StatusStore, SubsystemStatus, + UpsStatusState, +}; +#[cfg(test)] +use crate::status_store::{ReaderStatus, broadcast_dirty_read_counts}; use crate::storage::journal::Journal; use axum::Router; use axum::body::Bytes; @@ -57,15 +63,16 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{get, post, put}; use rt_updater::UpdateStatus; use rt_updater::workflow::{RealChecker, WorkflowState, run_check, run_download}; -use serde::Serialize; use serde::de::DeserializeOwned; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::convert::Infallible; use std::future::Future; use std::net::SocketAddr; use std::pin::Pin; use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::Duration; +#[cfg(test)] +use std::time::Instant; use tokio::net::TcpListener; use tokio::sync::{Mutex, Notify, broadcast}; @@ -82,260 +89,6 @@ pub struct StatusConfig { pub forwarder_version: String, } -// --------------------------------------------------------------------------- -// Subsystem readiness -// --------------------------------------------------------------------------- - -/// Connection state of a reader TCP socket. -#[derive(Debug, Clone, PartialEq)] -pub enum ReaderConnectionState { - Connecting, - Connected, - Disconnected, -} - -impl From<&ReaderConnectionState> for crate::ui_events::ReaderConnectionState { - fn from(state: &ReaderConnectionState) -> Self { - match state { - ReaderConnectionState::Connected => crate::ui_events::ReaderConnectionState::Connected, - ReaderConnectionState::Connecting => { - crate::ui_events::ReaderConnectionState::Connecting - } - ReaderConnectionState::Disconnected => { - crate::ui_events::ReaderConnectionState::Disconnected - } - } - } -} - -/// Per-reader status tracked in memory. -#[derive(Debug, Clone)] -pub struct ReaderStatus { - pub state: ReaderConnectionState, - pub last_seen: Option, - pub reads_since_restart: u64, - pub reads_total: i64, - /// The local port the forwarder listens on to re-expose reads from this reader. - pub local_port: u16, - /// The name of the current epoch, if any. - pub current_epoch_name: Option, - /// Control protocol info (firmware, clock, etc.) — populated on connect. - pub reader_info: Option, -} - -/// UPS daemon availability + latest readings snapshot. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct UpsStatusState { - pub available: bool, - pub status: Option, -} - -/// Forwarder-local status updates consumed by P2P control sessions. -#[derive(Debug, Clone)] -pub enum ForwarderStatusEvent { - ReaderStatus { - stream_id: String, - status: ReaderStatus, - }, - ReaderInfo { - stream_id: String, - info: crate::reader_control::ReaderInfo, - }, - DownloadProgress { - stream_id: String, - event: crate::reader_control::DownloadEvent, - }, - UpsStatus(UpsStatusState), -} - -/// Current forwarder status snapshot consumed by a newly connected P2P peer. -#[derive(Debug, Clone, Default)] -pub struct ForwarderStatusSnapshot { - pub readers: Vec<(String, ReaderStatus)>, - pub ups_status: Option, -} - -/// Read-only status feed for P2P control sessions. -#[derive(Clone)] -pub struct ForwarderStatusFeed { - subsystem: Arc>, - status_event_tx: broadcast::Sender, -} - -impl ForwarderStatusFeed { - /// Atomically subscribe to the status broadcast and capture the current - /// snapshot under the same `SubsystemStatus` lock. - /// - /// Holding the lock across both operations guarantees the returned snapshot - /// and the delta stream do not overlap: every status update either landed in - /// the snapshot (and was broadcast before this subscription existed) or is - /// delivered as a post-snapshot delta on the returned receiver. This relies - /// on all `ForwarderStatusEvent` broadcasts being emitted while holding the - /// same lock. No `.await` is held across the lock. - pub async fn subscribe_and_snapshot( - &self, - ) -> ( - broadcast::Receiver, - ForwarderStatusSnapshot, - ) { - let ss = self.subsystem.lock().await; - let receiver = self.status_event_tx.subscribe(); - let snapshot = ForwarderStatusSnapshot { - readers: ss - .readers() - .iter() - .map(|(stream_id, status)| (stream_id.clone(), status.clone())) - .collect(), - ups_status: ss.ups_status().cloned(), - }; - (receiver, snapshot) - } -} - -/// Tracks local subsystem readiness for the `/readyz` endpoint. -/// -/// Ready = config loaded + journal open + worker tasks started. -/// P2P session connectivity is explicitly excluded from readiness. -#[derive(Debug, Clone)] -pub struct SubsystemStatus { - ready: bool, - reason: Option, - /// P2P session state is tracked for the status page but does NOT affect readiness. - p2p_connected: bool, - p2p_endpoint_id: Option, - forwarder_id: String, - local_ip: Option, - pub(crate) readers: HashMap, - update_status: UpdateStatus, - staged_update_path: Option, - pub update_mode: rt_updater::UpdateMode, - /// Set to `true` when config is saved and the forwarder needs a restart to apply changes. - restart_needed: bool, - /// UPS status snapshot (None if UPS monitoring is not configured). - ups_status: Option, - /// Readers whose read counters changed since the last coalesced P2P - /// status broadcast (see [`spawn_read_count_broadcaster`]). - read_counts_dirty: HashSet, -} - -impl SubsystemStatus { - /// Create a fully-ready subsystem status. - pub fn ready() -> Self { - SubsystemStatus { - ready: true, - reason: None, - p2p_connected: false, - p2p_endpoint_id: None, - forwarder_id: String::new(), - local_ip: None, - readers: HashMap::new(), - update_status: UpdateStatus::UpToDate, - staged_update_path: None, - update_mode: rt_updater::UpdateMode::default(), - restart_needed: false, - ups_status: None, - read_counts_dirty: HashSet::new(), - } - } - - /// Create a not-ready subsystem status with a reason. - pub fn not_ready(reason: String) -> Self { - SubsystemStatus { - ready: false, - reason: Some(reason), - p2p_connected: false, - p2p_endpoint_id: None, - forwarder_id: String::new(), - local_ip: None, - readers: HashMap::new(), - update_status: UpdateStatus::UpToDate, - staged_update_path: None, - update_mode: rt_updater::UpdateMode::default(), - restart_needed: false, - ups_status: None, - read_counts_dirty: HashSet::new(), - } - } - - /// Set the P2P session state (does NOT affect `/readyz` result). - pub fn set_p2p_connected(&mut self, connected: bool) { - self.p2p_connected = connected; - } - - /// Return true if all local subsystems are ready. - pub fn is_ready(&self) -> bool { - self.ready - } - - /// Return the P2P session state. - pub fn p2p_connected(&self) -> bool { - self.p2p_connected - } - - pub fn set_p2p_endpoint_id(&mut self, endpoint_id: String) { - self.p2p_endpoint_id = Some(endpoint_id); - } - - /// Return whether a restart is needed to apply saved config changes. - pub fn restart_needed(&self) -> bool { - self.restart_needed - } - - /// Mark that a restart is needed to apply saved config changes. - pub fn set_restart_needed(&mut self) { - self.restart_needed = true; - } - - /// Return the connection state for a given reader IP, if tracked. - pub fn reader_connection_state(&self, reader_ip: &str) -> Option { - self.readers.get(reader_ip).map(|r| r.state.clone()) - } - - /// Return a reference to the readers map. - pub fn readers(&self) -> &HashMap { - &self.readers - } - - pub(crate) fn cached_reader_info( - &self, - reader_ip: &str, - ) -> Option { - self.readers - .get(reader_ip) - .and_then(|r| r.reader_info.clone()) - } - - pub(crate) fn update_cached_reader_info_unless_disconnected( - &mut self, - reader_ip: &str, - info: crate::reader_control::ReaderInfo, - ) -> bool { - let Some(reader) = self.readers.get_mut(reader_ip) else { - tracing::warn!(reader_ip = %reader_ip, "update_cached_reader_info: reader not found in status map, skipping broadcast"); - return false; - }; - if reader.state == ReaderConnectionState::Disconnected { - tracing::debug!( - reader_ip, - "dropping cached reader info update for disconnected reader" - ); - return false; - } - reader.reader_info = Some(info); - true - } - - /// Set the UPS status snapshot. - pub fn set_ups_status(&mut self, state: UpsStatusState) { - self.ups_status = Some(state); - } - - /// Return the current UPS status snapshot, if any. - pub fn ups_status(&self) -> Option<&UpsStatusState> { - self.ups_status.as_ref() - } -} - #[cfg(any(feature = "eink", feature = "lcd"))] fn subsystem_to_display_state( ss: &SubsystemStatus, @@ -400,18 +153,7 @@ fn subsystem_to_display_state( #[derive(Clone)] pub struct StatusServer { local_addr: SocketAddr, - subsystem: Arc>, - ui_tx: tokio::sync::broadcast::Sender, - status_event_tx: broadcast::Sender, - logger: Arc>, - control_clients: - Arc>>>, - download_trackers: Arc< - std::sync::RwLock< - HashMap>>, - >, - >, - reconnect_notifies: Arc>>>, + store: StatusStore, #[cfg(any(feature = "eink", feature = "lcd"))] display_tx: Option>, #[cfg(any(feature = "eink", feature = "lcd"))] @@ -479,236 +221,100 @@ impl Clone for AppState { } } -fn bridge_download_progress_events( - stream_id: String, - tracker: Arc>, - status_event_tx: broadcast::Sender, -) { - if let Ok(tracker) = tracker.try_lock() { - spawn_download_progress_bridge(stream_id, tracker.subscribe(), status_event_tx); - return; - } - - tokio::spawn(async move { - let rx = { - let tracker = tracker.lock().await; - tracker.subscribe() - }; - spawn_download_progress_bridge(stream_id, rx, status_event_tx); - }); -} - -fn spawn_download_progress_bridge( - stream_id: String, - mut rx: broadcast::Receiver, - status_event_tx: broadcast::Sender, -) { - tokio::spawn(async move { - loop { - match rx.recv().await { - Ok(event) => { - let _ = status_event_tx.send(ForwarderStatusEvent::DownloadProgress { - stream_id: stream_id.clone(), - event, - }); - } - Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => { - tracing::debug!(skipped = n, "download status bridge lagged"); - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => break, - } - } - }); -} - -/// Interval at which accumulated read-count changes are broadcast to P2P -/// control sessions as `ReaderStatus` deltas. -const READ_COUNT_BROADCAST_INTERVAL: Duration = Duration::from_secs(2); - -/// Broadcast a `ReaderStatus` delta for every reader whose read counters -/// changed since the last tick (see `record_read`). -/// -/// Sends while holding the `SubsystemStatus` lock so deltas stay ordered -/// against `subscribe_and_snapshot`, like every other status broadcast. -/// Broadcasting current state (not increments) makes a dropped or duplicated -/// delta harmless: the next tick carries the up-to-date counters. -async fn broadcast_dirty_read_counts( - subsystem: &Mutex, - status_event_tx: &broadcast::Sender, -) { - let mut ss = subsystem.lock().await; - if ss.read_counts_dirty.is_empty() { - return; - } - let dirty = std::mem::take(&mut ss.read_counts_dirty); - for reader_ip in dirty { - if let Some(status) = ss.readers.get(&reader_ip) { - let _ = status_event_tx.send(ForwarderStatusEvent::ReaderStatus { - stream_id: reader_ip, - status: status.clone(), - }); - } - } -} - -/// Spawn the coalescing task that pushes read-count updates to P2P peers at a -/// bounded rate. Quiet when no reads arrive; at most one delta per reader per -/// interval during bursts. -fn spawn_read_count_broadcaster( - subsystem: Arc>, - status_event_tx: broadcast::Sender, -) { - tokio::spawn(async move { - let mut ticker = tokio::time::interval(READ_COUNT_BROADCAST_INTERVAL); - ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - loop { - ticker.tick().await; - broadcast_dirty_read_counts(&subsystem, &status_event_tx).await; - } - }); -} - impl StatusServer { /// Return the bound listen address. pub fn local_addr(&self) -> SocketAddr { self.local_addr } + /// Return a clone of the pure status store. + pub fn store(&self) -> StatusStore { + self.store.clone() + } + /// Return a clone of the internal subsystem status Arc. pub fn subsystem_arc(&self) -> Arc> { - self.subsystem.clone() + self.store.subsystem_arc() } /// Return a clone of the UI event broadcast sender. pub fn ui_sender(&self) -> tokio::sync::broadcast::Sender { - self.ui_tx.clone() + self.store.ui_sender() } /// Return a read-only status feed for P2P control sessions. pub fn status_feed(&self) -> ForwarderStatusFeed { - ForwarderStatusFeed { - subsystem: self.subsystem.clone(), - status_event_tx: self.status_event_tx.clone(), - } + self.store.status_feed() } /// Return the shared reader-control service used by HTTP and P2P control paths. pub fn reader_control_service(&self) -> crate::reader_control_service::ReaderControlService { - crate::reader_control_service::ReaderControlService::new( - self.subsystem.clone(), - self.control_clients.clone(), - self.download_trackers.clone(), - self.reconnect_notifies.clone(), - self.ui_tx.clone(), - self.status_event_tx.clone(), - self.logger.clone(), - ) + self.store.reader_control_service() } /// Return a clone of the shared UI logger Arc. pub fn logger(&self) -> Arc> { - self.logger.clone() + self.store.logger() } /// Mark all local subsystems as ready. pub async fn set_ready(&self) { - { - let mut ss = self.subsystem.lock().await; - ss.ready = true; - ss.reason = None; - let _ = self - .ui_tx - .send(crate::ui_events::ForwarderUiEvent::StatusChanged { - ready: ss.is_ready(), - p2p_connected: ss.p2p_connected(), - restart_needed: ss.restart_needed(), - }); - } + self.store.set_ready().await; #[cfg(any(feature = "eink", feature = "lcd"))] self.publish_display_state().await; } /// Mark that a restart is needed to apply saved config changes. pub async fn set_restart_needed(&self) { - mark_restart_needed_and_emit(&self.subsystem, &self.ui_tx).await; + self.store.set_restart_needed().await; } /// Return whether a restart is needed to apply saved config changes. pub async fn restart_needed(&self) -> bool { - self.subsystem.lock().await.restart_needed() + self.store.restart_needed().await } pub async fn set_p2p_endpoint_id(&self, endpoint_id: String) { - let mut ss = self.subsystem.lock().await; - ss.set_p2p_endpoint_id(endpoint_id); + self.store.set_p2p_endpoint_id(endpoint_id).await; } /// Update the P2P session state (does not affect readiness). pub async fn set_p2p_connected(&self, connected: bool) { - { - let mut ss = self.subsystem.lock().await; - ss.set_p2p_connected(connected); - let _ = self - .ui_tx - .send(crate::ui_events::ForwarderUiEvent::StatusChanged { - ready: ss.is_ready(), - p2p_connected: connected, - restart_needed: ss.restart_needed(), - }); - } + self.store.set_p2p_connected(connected).await; #[cfg(any(feature = "eink", feature = "lcd"))] self.publish_display_state().await; } /// Set the forwarder ID (call once at startup). pub async fn set_forwarder_id(&self, id: &str) { - self.subsystem.lock().await.forwarder_id = id.to_owned(); + self.store.set_forwarder_id(id).await; } /// Set the detected local IP (at startup and on reader connect/disconnect). pub async fn set_local_ip(&self, ip: Option) { - self.subsystem.lock().await.local_ip = ip; + self.store.set_local_ip(ip).await; #[cfg(any(feature = "eink", feature = "lcd"))] self.publish_display_state().await; } /// Set the update mode (controls check-only vs check-and-download behavior). pub async fn set_update_mode(&self, mode: rt_updater::UpdateMode) { - self.subsystem.lock().await.update_mode = mode; + self.store.set_update_mode(mode).await; } /// Update the current rt-updater status (shown on `/update/status`). pub async fn set_update_status(&self, status: UpdateStatus) { - self.subsystem.lock().await.update_status = status.clone(); - let _ = self - .ui_tx - .send(crate::ui_events::ForwarderUiEvent::UpdateStatusChanged { status }); + self.store.set_update_status(status).await; } /// Record the filesystem path of a downloaded update artifact ready to apply. pub async fn set_staged_update_path(&self, path: std::path::PathBuf) { - self.subsystem.lock().await.staged_update_path = Some(path); + self.store.set_staged_update_path(path).await; } /// Update the UPS status snapshot in the subsystem state. - /// - /// The local HTTP/UI snapshot is always updated, but the P2P control-event - /// broadcast only fires when the UPS status actually changed from the stored - /// previous value. The UPS poller calls this every interval, so gating the - /// broadcast on a real change avoids spamming connected receivers with - /// identical `UpsStatus` frames. The send happens under the subsystem lock so - /// it is ordered against `subscribe_and_snapshot`. pub async fn set_ups_status(&self, state: UpsStatusState) { - { - let mut ss = self.subsystem.lock().await; - let changed = ss.ups_status() != Some(&state); - ss.set_ups_status(state.clone()); - if changed { - let _ = self - .status_event_tx - .send(ForwarderStatusEvent::UpsStatus(state)); - } - } + self.store.set_ups_status(state).await; #[cfg(any(feature = "eink", feature = "lcd"))] self.publish_display_state().await; } @@ -716,7 +322,7 @@ impl StatusServer { pub fn control_clients( &self, ) -> &Arc>>> { - &self.control_clients + self.store.control_clients() } #[allow(clippy::type_complexity)] @@ -727,7 +333,7 @@ impl StatusServer { HashMap>>, >, > { - &self.download_trackers + self.store.download_trackers() } pub fn register_download_tracker( @@ -735,46 +341,30 @@ impl StatusServer { reader_ip: &str, tracker: Arc>, ) { - self.download_trackers - .write() - .unwrap_or_else(|e| e.into_inner()) - .insert(reader_ip.to_owned(), tracker.clone()); - bridge_download_progress_events( - reader_ip.to_owned(), - tracker, - self.status_event_tx.clone(), - ); + self.store.register_download_tracker(reader_ip, tracker); } pub fn deregister_download_tracker(&self, reader_ip: &str) { - self.download_trackers - .write() - .unwrap_or_else(|e| e.into_inner()) - .remove(reader_ip); + self.store.deregister_download_tracker(reader_ip); } pub fn register_reconnect_notify(&self, reader_ip: &str, notify: Arc) { - self.reconnect_notifies - .write() - .unwrap_or_else(|e| e.into_inner()) - .insert(reader_ip.to_owned(), notify); + self.store.register_reconnect_notify(reader_ip, notify); } pub fn deregister_reconnect_notify(&self, reader_ip: &str) { - self.reconnect_notifies - .write() - .unwrap_or_else(|e| e.into_inner()) - .remove(reader_ip); + self.store.deregister_reconnect_notify(reader_ip); } pub fn reconnect_notifies(&self) -> &Arc>>> { - &self.reconnect_notifies + self.store.reconnect_notifies() } #[cfg(any(feature = "eink", feature = "lcd"))] async fn publish_display_state(&self) { if let Some(ref tx) = self.display_tx { - let ss = self.subsystem.lock().await; + let subsystem = self.store.subsystem_arc(); + let ss = subsystem.lock().await; let forwarder_name = self.display_name.lock().await.clone(); let cpu_temp = *self.cpu_temp.lock().await; let state = subsystem_to_display_state(&ss, forwarder_name, cpu_temp); @@ -788,6 +378,24 @@ impl StatusServer { tx: tokio::sync::watch::Sender, ) { self.display_tx = Some(tx); + self.spawn_display_change_bridge(); + } + + #[cfg(any(feature = "eink", feature = "lcd"))] + fn spawn_display_change_bridge(&self) { + let mut display_changes = self.store.subscribe_display_changes(); + let server = self.clone(); + tokio::spawn(async move { + loop { + match display_changes.recv().await { + Ok(()) => server.publish_display_state().await, + Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => { + server.publish_display_state().await; + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => break, + } + } + }); } #[cfg(any(feature = "eink", feature = "lcd"))] @@ -812,10 +420,7 @@ impl StatusServer { &self, reader_ip: &str, ) -> Option { - let ss = self.subsystem.lock().await; - ss.readers - .get(reader_ip) - .and_then(|r| r.reader_info.clone()) + self.store.get_reader_info(reader_ip).await } pub async fn update_reader_info( @@ -823,68 +428,20 @@ impl StatusServer { reader_ip: &str, info: crate::reader_control::ReaderInfo, ) { - { - let mut ss = self.subsystem.lock().await; - if let Some(r) = ss.readers.get_mut(reader_ip) { - r.reader_info = Some(info.clone()); - } - let _ = self - .ui_tx - .send(crate::ui_events::ForwarderUiEvent::ReaderInfoUpdated { - ip: reader_ip.to_owned(), - info: info.clone(), - }); - // Broadcast under the lock so it is ordered against - // `subscribe_and_snapshot`. - let _ = self.status_event_tx.send(ForwarderStatusEvent::ReaderInfo { - stream_id: reader_ip.to_owned(), - info, - }); - } + self.store.update_reader_info(reader_ip, info).await; #[cfg(any(feature = "eink", feature = "lcd"))] self.publish_display_state().await; } /// Update reader info only if the reader has not transitioned to Disconnected. - /// - /// This is used by the background poller so a late poll result cannot restore - /// stale info after the read loop has already marked the reader disconnected. pub async fn update_reader_info_unless_disconnected( &self, reader_ip: &str, info: crate::reader_control::ReaderInfo, ) { - { - let mut ss = self.subsystem.lock().await; - if let Some(r) = ss.readers.get_mut(reader_ip) { - if r.state == ReaderConnectionState::Disconnected { - tracing::debug!( - reader_ip, - "dropping reader info update for disconnected reader" - ); - return; - } - r.reader_info = Some(info.clone()); - } else { - tracing::debug!( - reader_ip, - "reader IP not found in map, skipping info update" - ); - return; - } - let _ = self - .ui_tx - .send(crate::ui_events::ForwarderUiEvent::ReaderInfoUpdated { - ip: reader_ip.to_owned(), - info: info.clone(), - }); - // Broadcast under the lock so it is ordered against - // `subscribe_and_snapshot`. - let _ = self.status_event_tx.send(ForwarderStatusEvent::ReaderInfo { - stream_id: reader_ip.to_owned(), - info, - }); - } + self.store + .update_reader_info_unless_disconnected(reader_ip, info) + .await; #[cfg(any(feature = "eink", feature = "lcd"))] self.publish_display_state().await; } @@ -894,141 +451,44 @@ impl StatusServer { reader_ip: &str, client: Arc, ) { - self.control_clients - .write() - .unwrap_or_else(|e| e.into_inner()) - .insert(reader_ip.to_owned(), client); + self.store.register_control_client(reader_ip, client); } pub fn deregister_control_client(&self, reader_ip: &str) { - self.control_clients - .write() - .unwrap_or_else(|e| e.into_inner()) - .remove(reader_ip); + self.store.deregister_control_client(reader_ip); } /// Pre-populate all configured reader IPs as Disconnected. - /// - /// Each entry is `(reader_addr, local_port)` where `reader_addr` is `"ip:port"` - /// and `local_port` is the port the forwarder listens on to re-expose reads. pub async fn init_readers(&self, readers: &[(String, u16)]) { - { - let mut ss = self.subsystem.lock().await; - for (addr, local_port) in readers { - ss.readers.entry(addr.clone()).or_insert(ReaderStatus { - state: ReaderConnectionState::Disconnected, - last_seen: None, - reads_since_restart: 0, - reads_total: 0, - local_port: *local_port, - current_epoch_name: None, - reader_info: None, - }); - } - } + self.store.init_readers(readers).await; #[cfg(any(feature = "eink", feature = "lcd"))] self.publish_display_state().await; } /// Seed a reader's total historical count from durable journal state. pub async fn set_reader_total(&self, reader_ip: &str, total: i64) { - { - let mut ss = self.subsystem.lock().await; - if let Some(r) = ss.readers.get_mut(reader_ip) { - r.reads_total = total; - } - } + self.store.set_reader_total(reader_ip, total).await; #[cfg(any(feature = "eink", feature = "lcd"))] self.publish_display_state().await; } /// Set the current epoch name for a reader and broadcast a ReaderUpdated SSE event. pub async fn set_reader_epoch_name(&self, reader_ip: &str, name: Option) { - { - let mut ss = self.subsystem.lock().await; - if let Some(r) = ss.readers.get_mut(reader_ip) { - r.current_epoch_name = name; - let _ = self - .ui_tx - .send(crate::ui_events::ForwarderUiEvent::ReaderUpdated { - ip: reader_ip.to_owned(), - state: (&r.state).into(), - reads_session: r.reads_since_restart, - reads_total: r.reads_total, - last_seen_secs: r.last_seen.map(|t| t.elapsed().as_secs()), - local_port: r.local_port, - current_epoch_name: r.current_epoch_name.clone(), - }); - } - } + self.store.set_reader_epoch_name(reader_ip, name).await; #[cfg(any(feature = "eink", feature = "lcd"))] self.publish_display_state().await; } /// Update a reader's connection state. pub async fn update_reader_state(&self, reader_ip: &str, state: ReaderConnectionState) { - { - let mut ss = self.subsystem.lock().await; - if let Some(r) = ss.readers.get_mut(reader_ip) { - let changed = r.state != state; - if state == ReaderConnectionState::Disconnected { - r.reader_info = None; - } - r.state = state; - let _ = self - .ui_tx - .send(crate::ui_events::ForwarderUiEvent::ReaderUpdated { - ip: reader_ip.to_owned(), - state: (&r.state).into(), - reads_session: r.reads_since_restart, - reads_total: r.reads_total, - last_seen_secs: r.last_seen.map(|t| t.elapsed().as_secs()), - local_port: r.local_port, - current_epoch_name: r.current_epoch_name.clone(), - }); - // Only broadcast a P2P control delta on an actual state - // transition, and do so under the lock so it is ordered against - // `subscribe_and_snapshot`. The local UI event above is always - // emitted. - if changed { - let _ = self - .status_event_tx - .send(ForwarderStatusEvent::ReaderStatus { - stream_id: reader_ip.to_owned(), - status: r.clone(), - }); - } - } - } + self.store.update_reader_state(reader_ip, state).await; #[cfg(any(feature = "eink", feature = "lcd"))] self.publish_display_state().await; } /// Record a successful chip read for a reader. pub async fn record_read(&self, reader_ip: &str) { - { - let mut ss = self.subsystem.lock().await; - if let Some(r) = ss.readers.get_mut(reader_ip) { - r.reads_since_restart += 1; - r.reads_total += 1; - r.last_seen = Some(Instant::now()); - let _ = self - .ui_tx - .send(crate::ui_events::ForwarderUiEvent::ReaderUpdated { - ip: reader_ip.to_owned(), - state: (&r.state).into(), - reads_session: r.reads_since_restart, - reads_total: r.reads_total, - last_seen_secs: r.last_seen.map(|t| t.elapsed().as_secs()), - local_port: r.local_port, - current_epoch_name: r.current_epoch_name.clone(), - }); - // Per-read P2P broadcasts would flood the control stream during - // bursts, so reads only mark the reader dirty; the coalescing - // task broadcasts the latest counters at a bounded rate. - ss.read_counts_dirty.insert(reader_ip.to_owned()); - } - } + self.store.record_read(reader_ip).await; #[cfg(any(feature = "eink", feature = "lcd"))] self.publish_display_state().await; } @@ -1050,33 +510,23 @@ impl StatusServer { let listener = TcpListener::bind(&cfg.bind).await?; let local_addr = listener.local_addr()?; - let (ui_tx, _) = tokio::sync::broadcast::channel(256); - let (status_event_tx, _) = broadcast::channel(256); - let logger = Arc::new(rt_ui_log::UiLogger::with_buffer( - ui_tx.clone(), - |entry| crate::ui_events::ForwarderUiEvent::LogEntry { entry }, - 500, - )); - let subsystem = Arc::new(Mutex::new(subsystem)); - let control_clients = Arc::new(std::sync::RwLock::new(HashMap::new())); - let download_trackers = Arc::new(std::sync::RwLock::new(HashMap::new())); - let reconnect_notifies = Arc::new(std::sync::RwLock::new(HashMap::new())); + let store = StatusStore::new(subsystem); #[cfg(any(feature = "eink", feature = "lcd"))] let display_name = Arc::new(Mutex::new(None)); #[cfg(any(feature = "eink", feature = "lcd"))] let cpu_temp = Arc::new(Mutex::new(None)); let state = AppState { - subsystem: subsystem.clone(), + subsystem: store.subsystem_arc(), journal, version: Arc::new(cfg.forwarder_version), config_state: None, restart_signal: None, - ui_tx: ui_tx.clone(), - status_event_tx: status_event_tx.clone(), - logger: logger.clone(), - control_clients: control_clients.clone(), - download_trackers: download_trackers.clone(), - reconnect_notifies: reconnect_notifies.clone(), + ui_tx: store.ui_sender(), + status_event_tx: store.status_event_sender(), + logger: store.logger(), + control_clients: store.control_clients().clone(), + download_trackers: store.download_trackers().clone(), + reconnect_notifies: store.reconnect_notifies().clone(), #[cfg(any(feature = "eink", feature = "lcd"))] display_name: display_name.clone(), #[cfg(any(feature = "eink", feature = "lcd"))] @@ -1089,17 +539,10 @@ impl StatusServer { tracing::error!(error = %err, "status HTTP server fatal error"); } }); - spawn_read_count_broadcaster(subsystem.clone(), status_event_tx.clone()); Ok(StatusServer { local_addr, - subsystem, - ui_tx, - status_event_tx, - logger, - control_clients, - download_trackers, - reconnect_notifies, + store, #[cfg(any(feature = "eink", feature = "lcd"))] display_tx: None, #[cfg(any(feature = "eink", feature = "lcd"))] @@ -1120,33 +563,23 @@ impl StatusServer { let listener = TcpListener::bind(&cfg.bind).await?; let local_addr = listener.local_addr()?; - let (ui_tx, _) = tokio::sync::broadcast::channel(256); - let (status_event_tx, _) = broadcast::channel(256); - let logger = Arc::new(rt_ui_log::UiLogger::with_buffer( - ui_tx.clone(), - |entry| crate::ui_events::ForwarderUiEvent::LogEntry { entry }, - 500, - )); - let subsystem = Arc::new(Mutex::new(subsystem)); - let control_clients = Arc::new(std::sync::RwLock::new(HashMap::new())); - let download_trackers = Arc::new(std::sync::RwLock::new(HashMap::new())); - let reconnect_notifies = Arc::new(std::sync::RwLock::new(HashMap::new())); + let store = StatusStore::new(subsystem); #[cfg(any(feature = "eink", feature = "lcd"))] let display_name = Arc::new(Mutex::new(None)); #[cfg(any(feature = "eink", feature = "lcd"))] let cpu_temp = Arc::new(Mutex::new(None)); let state = AppState { - subsystem: subsystem.clone(), + subsystem: store.subsystem_arc(), journal, version: Arc::new(cfg.forwarder_version), config_state: Some(config_state), restart_signal: Some(restart_signal), - ui_tx: ui_tx.clone(), - status_event_tx: status_event_tx.clone(), - logger: logger.clone(), - control_clients: control_clients.clone(), - download_trackers: download_trackers.clone(), - reconnect_notifies: reconnect_notifies.clone(), + ui_tx: store.ui_sender(), + status_event_tx: store.status_event_sender(), + logger: store.logger(), + control_clients: store.control_clients().clone(), + download_trackers: store.download_trackers().clone(), + reconnect_notifies: store.reconnect_notifies().clone(), #[cfg(any(feature = "eink", feature = "lcd"))] display_name: display_name.clone(), #[cfg(any(feature = "eink", feature = "lcd"))] @@ -1159,17 +592,10 @@ impl StatusServer { tracing::error!(error = %err, "status HTTP server fatal error"); } }); - spawn_read_count_broadcaster(subsystem.clone(), status_event_tx.clone()); Ok(StatusServer { local_addr, - subsystem, - ui_tx, - status_event_tx, - logger, - control_clients, - download_trackers, - reconnect_notifies, + store, #[cfg(any(feature = "eink", feature = "lcd"))] display_tx: None, #[cfg(any(feature = "eink", feature = "lcd"))] @@ -1241,19 +667,6 @@ impl JournalAccess for NoJournal { // Helpers // --------------------------------------------------------------------------- -pub(crate) async fn mark_restart_needed_and_emit( - subsystem: &Arc>, - ui_tx: &tokio::sync::broadcast::Sender, -) { - let mut ss = subsystem.lock().await; - ss.set_restart_needed(); - let _ = ui_tx.send(crate::ui_events::ForwarderUiEvent::StatusChanged { - ready: ss.is_ready(), - p2p_connected: ss.p2p_connected(), - restart_needed: true, - }); -} - fn text_response(status: StatusCode, body: impl Into) -> Response { (status, [(header::CONTENT_TYPE, "text/plain")], body.into()).into_response() } @@ -3386,7 +2799,7 @@ mod tests { .update_reader_state(reader_ip, ReaderConnectionState::Connected) .await; - let mut ui_rx = server.ui_tx.subscribe(); + let mut ui_rx = server.ui_sender().subscribe(); let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::channel::>(8); let (control_client, control_sink) = crate::reader_control::ControlClient::new(cmd_tx); @@ -3553,17 +2966,17 @@ mod tests { .await; let state = AppState { - subsystem: server.subsystem.clone(), + subsystem: server.subsystem_arc(), journal: Arc::new(Mutex::new(NoJournal)), version: Arc::new("0.2.0".to_owned()), config_state: None, restart_signal: None, - logger: server.logger.clone(), - ui_tx: server.ui_tx.clone(), - status_event_tx: server.status_event_tx.clone(), - control_clients: server.control_clients.clone(), - download_trackers: server.download_trackers.clone(), - reconnect_notifies: server.reconnect_notifies.clone(), + logger: server.logger(), + ui_tx: server.ui_sender(), + status_event_tx: server.store.status_event_sender(), + control_clients: server.control_clients().clone(), + download_trackers: server.download_trackers().clone(), + reconnect_notifies: server.reconnect_notifies().clone(), #[cfg(any(feature = "eink", feature = "lcd"))] display_name: server.display_name.clone(), #[cfg(any(feature = "eink", feature = "lcd"))] @@ -4516,7 +3929,7 @@ mod tests { .update_reader_info(reader_ip, crate::reader_control::ReaderInfo::default()) .await; - let mut rx = server.ui_tx.subscribe(); + let mut rx = server.ui_sender().subscribe(); let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::channel::>(16); let (control_client, control_sink) = crate::reader_control::ControlClient::new(cmd_tx); server.register_control_client(reader_ip, Arc::new(control_client)); @@ -4578,7 +3991,7 @@ mod tests { .await .expect("start status server"); - let mut rx = server.ui_tx.subscribe(); + let mut rx = server.ui_sender().subscribe(); server.set_ready().await; let evt = tokio::time::timeout(Duration::from_millis(250), rx.recv()) @@ -4605,7 +4018,7 @@ mod tests { .await .expect("start status server"); - let mut rx = server.ui_tx.subscribe(); + let mut rx = server.ui_sender().subscribe(); server .set_update_status(UpdateStatus::Downloaded { version: "1.2.3".to_owned(), @@ -4662,7 +4075,7 @@ target = "192.168.1.100:10000" .await .expect("start status server"); - let mut rx = server.ui_tx.subscribe(); + let mut rx = server.ui_sender().subscribe(); let client = reqwest::Client::new(); let resp = client .post(format!( @@ -4723,7 +4136,7 @@ target = "192.168.1.100:10000" .expect("start status server"); assert_eq!( - server.subsystem.lock().await.update_mode, + server.subsystem_arc().lock().await.update_mode, rt_updater::UpdateMode::CheckAndDownload ); @@ -4740,7 +4153,7 @@ target = "192.168.1.100:10000" .expect("post config update"); assert_eq!(resp.status(), StatusCode::OK); assert_eq!( - server.subsystem.lock().await.update_mode, + server.subsystem_arc().lock().await.update_mode, rt_updater::UpdateMode::CheckOnly ); assert!( @@ -4796,7 +4209,7 @@ target = "192.168.1.100:10000" .expect("post config update"); assert_eq!(resp.status(), StatusCode::BAD_REQUEST); assert_eq!( - server.subsystem.lock().await.update_mode, + server.subsystem_arc().lock().await.update_mode, rt_updater::UpdateMode::CheckAndDownload ); } @@ -5001,7 +4414,7 @@ target = "192.168.1.100:10000" }; let workflow_state = - ForwarderWorkflowAdapter::new(server.subsystem.clone(), server.ui_tx.clone()); + ForwarderWorkflowAdapter::new(server.subsystem_arc(), server.ui_sender()); let status = run_check(&workflow_state, &checker, rt_updater::UpdateMode::CheckOnly).await; assert_eq!( @@ -5034,7 +4447,7 @@ target = "192.168.1.100:10000" }; let workflow_state = - ForwarderWorkflowAdapter::new(server.subsystem.clone(), server.ui_tx.clone()); + ForwarderWorkflowAdapter::new(server.subsystem_arc(), server.ui_sender()); let status = run_check(&workflow_state, &checker, rt_updater::UpdateMode::Disabled).await; assert_eq!( @@ -5067,7 +4480,7 @@ target = "192.168.1.100:10000" }; let workflow_state = - ForwarderWorkflowAdapter::new(server.subsystem.clone(), server.ui_tx.clone()); + ForwarderWorkflowAdapter::new(server.subsystem_arc(), server.ui_sender()); let status = run_check( &workflow_state, &checker, @@ -5083,7 +4496,7 @@ target = "192.168.1.100:10000" ); assert_eq!(download_calls.load(Ordering::SeqCst), 1); assert_eq!( - server.subsystem.lock().await.staged_update_path, + server.subsystem_arc().lock().await.staged_update_path, Some(std::path::PathBuf::from("/tmp/staged-forwarder")) ); } @@ -5307,7 +4720,7 @@ target = "192.168.1.100:10000" }; let workflow_state = - ForwarderWorkflowAdapter::new(server.subsystem.clone(), server.ui_tx.clone()); + ForwarderWorkflowAdapter::new(server.subsystem_arc(), server.ui_sender()); let status = run_download(&workflow_state, &checker).await; assert_eq!( @@ -5318,7 +4731,7 @@ target = "192.168.1.100:10000" ); assert_eq!(download_calls.load(Ordering::SeqCst), 1); assert_eq!( - server.subsystem.lock().await.staged_update_path, + server.subsystem_arc().lock().await.staged_update_path, Some(std::path::PathBuf::from("/tmp/staged-forwarder")) ); } @@ -5340,7 +4753,7 @@ target = "192.168.1.100:10000" version: "2.0.0".to_owned(), }) .await; - let mut rx = server.ui_tx.subscribe(); + let mut rx = server.ui_sender().subscribe(); let checker = FakeChecker { check_result: Ok(UpdateStatus::UpToDate), @@ -5349,7 +4762,7 @@ target = "192.168.1.100:10000" }; let workflow_state = - ForwarderWorkflowAdapter::new(server.subsystem.clone(), server.ui_tx.clone()); + ForwarderWorkflowAdapter::new(server.subsystem_arc(), server.ui_sender()); let status = run_download(&workflow_state, &checker).await; assert_eq!( status, @@ -5390,7 +4803,7 @@ target = "192.168.1.100:10000" }; let workflow_state = - ForwarderWorkflowAdapter::new(server.subsystem.clone(), server.ui_tx.clone()); + ForwarderWorkflowAdapter::new(server.subsystem_arc(), server.ui_sender()); let status = run_download(&workflow_state, &checker).await; assert!(status.is_err()); } @@ -5420,7 +4833,7 @@ target = "192.168.1.100:10000" }; let workflow_state = - ForwarderWorkflowAdapter::new(server.subsystem.clone(), server.ui_tx.clone()); + ForwarderWorkflowAdapter::new(server.subsystem_arc(), server.ui_sender()); let status = run_download(&workflow_state, &checker).await; assert_eq!( status, @@ -5449,7 +4862,7 @@ target = "192.168.1.100:10000" .update_reader_state("192.168.1.10", ReaderConnectionState::Connected) .await; - let mut rx = server.ui_tx.subscribe(); + let mut rx = server.ui_sender().subscribe(); // Set epoch name server @@ -5492,7 +4905,7 @@ target = "192.168.1.100:10000" .set_reader_epoch_name("192.168.1.10", Some("Race Day".to_owned())) .await; - let mut rx = server.ui_tx.subscribe(); + let mut rx = server.ui_sender().subscribe(); // Clear epoch name server.set_reader_epoch_name("192.168.1.10", None).await; @@ -5890,7 +5303,8 @@ target = "192.168.1.100:10000" // have been consumed if the background broadcaster ticked, in which // case the delta must be on the feed instead.) let dirty = { - let ss = server.subsystem.lock().await; + let subsystem = server.subsystem_arc(); + let ss = subsystem.lock().await; ss.read_counts_dirty.contains("10.0.0.9:10000") }; if !dirty { diff --git a/services/forwarder/src/status_store.rs b/services/forwarder/src/status_store.rs new file mode 100644 index 00000000..8c93b7bf --- /dev/null +++ b/services/forwarder/src/status_store.rs @@ -0,0 +1,841 @@ +//! Pure runtime status state for the forwarder. +//! +//! This module owns the in-memory status store and status-event fanout used by +//! the data plane, UI event stream, and P2P control status feed. It intentionally +//! has no HTTP/axum dependencies. + +use rt_updater::UpdateStatus; +use serde::Serialize; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::{Mutex, Notify, broadcast}; + +// --------------------------------------------------------------------------- +// Subsystem readiness +// --------------------------------------------------------------------------- + +/// Connection state of a reader TCP socket. +#[derive(Debug, Clone, PartialEq)] +pub enum ReaderConnectionState { + Connecting, + Connected, + Disconnected, +} + +impl From<&ReaderConnectionState> for crate::ui_events::ReaderConnectionState { + fn from(state: &ReaderConnectionState) -> Self { + match state { + ReaderConnectionState::Connected => crate::ui_events::ReaderConnectionState::Connected, + ReaderConnectionState::Connecting => { + crate::ui_events::ReaderConnectionState::Connecting + } + ReaderConnectionState::Disconnected => { + crate::ui_events::ReaderConnectionState::Disconnected + } + } + } +} + +/// Per-reader status tracked in memory. +#[derive(Debug, Clone)] +pub struct ReaderStatus { + pub state: ReaderConnectionState, + pub last_seen: Option, + pub reads_since_restart: u64, + pub reads_total: i64, + /// The local port the forwarder listens on to re-expose reads from this reader. + pub local_port: u16, + /// The name of the current epoch, if any. + pub current_epoch_name: Option, + /// Control protocol info (firmware, clock, etc.) — populated on connect. + pub reader_info: Option, +} + +/// UPS daemon availability + latest readings snapshot. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct UpsStatusState { + pub available: bool, + pub status: Option, +} + +/// Forwarder-local status updates consumed by P2P control sessions. +#[derive(Debug, Clone)] +pub enum ForwarderStatusEvent { + ReaderStatus { + stream_id: String, + status: ReaderStatus, + }, + ReaderInfo { + stream_id: String, + info: crate::reader_control::ReaderInfo, + }, + DownloadProgress { + stream_id: String, + event: crate::reader_control::DownloadEvent, + }, + UpsStatus(UpsStatusState), +} + +/// Current forwarder status snapshot consumed by a newly connected P2P peer. +#[derive(Debug, Clone, Default)] +pub struct ForwarderStatusSnapshot { + pub readers: Vec<(String, ReaderStatus)>, + pub ups_status: Option, +} + +/// Read-only status feed for P2P control sessions. +#[derive(Clone)] +pub struct ForwarderStatusFeed { + subsystem: Arc>, + status_event_tx: broadcast::Sender, +} + +impl ForwarderStatusFeed { + /// Atomically subscribe to the status broadcast and capture the current + /// snapshot under the same `SubsystemStatus` lock. + /// + /// Holding the lock across both operations guarantees the returned snapshot + /// and the delta stream do not overlap: every status update either landed in + /// the snapshot (and was broadcast before this subscription existed) or is + /// delivered as a post-snapshot delta on the returned receiver. This relies + /// on all `ForwarderStatusEvent` broadcasts being emitted while holding the + /// same lock. No `.await` is held across the lock. + pub async fn subscribe_and_snapshot( + &self, + ) -> ( + broadcast::Receiver, + ForwarderStatusSnapshot, + ) { + let ss = self.subsystem.lock().await; + let receiver = self.status_event_tx.subscribe(); + let snapshot = ForwarderStatusSnapshot { + readers: ss + .readers() + .iter() + .map(|(stream_id, status)| (stream_id.clone(), status.clone())) + .collect(), + ups_status: ss.ups_status().cloned(), + }; + (receiver, snapshot) + } +} + +/// Tracks local subsystem readiness for the `/readyz` endpoint. +/// +/// Ready = config loaded + journal open + worker tasks started. +/// P2P session connectivity is explicitly excluded from readiness. +#[derive(Debug, Clone)] +pub struct SubsystemStatus { + pub(crate) ready: bool, + pub(crate) reason: Option, + /// P2P session state is tracked for the status page but does NOT affect readiness. + pub(crate) p2p_connected: bool, + pub(crate) p2p_endpoint_id: Option, + pub(crate) forwarder_id: String, + pub(crate) local_ip: Option, + pub(crate) readers: HashMap, + pub(crate) update_status: UpdateStatus, + pub(crate) staged_update_path: Option, + pub update_mode: rt_updater::UpdateMode, + /// Set to `true` when config is saved and the forwarder needs a restart to apply changes. + pub(crate) restart_needed: bool, + /// UPS status snapshot (None if UPS monitoring is not configured). + pub(crate) ups_status: Option, + /// Readers whose read counters changed since the last coalesced P2P + /// status broadcast (see [`spawn_read_count_broadcaster`]). + pub(crate) read_counts_dirty: HashSet, +} + +impl SubsystemStatus { + /// Create a fully-ready subsystem status. + pub fn ready() -> Self { + SubsystemStatus { + ready: true, + reason: None, + p2p_connected: false, + p2p_endpoint_id: None, + forwarder_id: String::new(), + local_ip: None, + readers: HashMap::new(), + update_status: UpdateStatus::UpToDate, + staged_update_path: None, + update_mode: rt_updater::UpdateMode::default(), + restart_needed: false, + ups_status: None, + read_counts_dirty: HashSet::new(), + } + } + + /// Create a not-ready subsystem status with a reason. + pub fn not_ready(reason: String) -> Self { + SubsystemStatus { + ready: false, + reason: Some(reason), + p2p_connected: false, + p2p_endpoint_id: None, + forwarder_id: String::new(), + local_ip: None, + readers: HashMap::new(), + update_status: UpdateStatus::UpToDate, + staged_update_path: None, + update_mode: rt_updater::UpdateMode::default(), + restart_needed: false, + ups_status: None, + read_counts_dirty: HashSet::new(), + } + } + + /// Set the P2P session state (does NOT affect `/readyz` result). + pub fn set_p2p_connected(&mut self, connected: bool) { + self.p2p_connected = connected; + } + + /// Return true if all local subsystems are ready. + pub fn is_ready(&self) -> bool { + self.ready + } + + /// Return the P2P session state. + pub fn p2p_connected(&self) -> bool { + self.p2p_connected + } + + pub fn set_p2p_endpoint_id(&mut self, endpoint_id: String) { + self.p2p_endpoint_id = Some(endpoint_id); + } + + /// Return whether a restart is needed to apply saved config changes. + pub fn restart_needed(&self) -> bool { + self.restart_needed + } + + /// Mark that a restart is needed to apply saved config changes. + pub fn set_restart_needed(&mut self) { + self.restart_needed = true; + } + + /// Return the connection state for a given reader IP, if tracked. + pub fn reader_connection_state(&self, reader_ip: &str) -> Option { + self.readers.get(reader_ip).map(|r| r.state.clone()) + } + + /// Return a reference to the readers map. + pub fn readers(&self) -> &HashMap { + &self.readers + } + + pub(crate) fn cached_reader_info( + &self, + reader_ip: &str, + ) -> Option { + self.readers + .get(reader_ip) + .and_then(|r| r.reader_info.clone()) + } + + pub(crate) fn update_cached_reader_info_unless_disconnected( + &mut self, + reader_ip: &str, + info: crate::reader_control::ReaderInfo, + ) -> bool { + let Some(reader) = self.readers.get_mut(reader_ip) else { + tracing::warn!(reader_ip = %reader_ip, "update_cached_reader_info: reader not found in status map, skipping broadcast"); + return false; + }; + if reader.state == ReaderConnectionState::Disconnected { + tracing::debug!( + reader_ip, + "dropping cached reader info update for disconnected reader" + ); + return false; + } + reader.reader_info = Some(info); + true + } + + /// Set the UPS status snapshot. + pub fn set_ups_status(&mut self, state: UpsStatusState) { + self.ups_status = Some(state); + } + + /// Return the current UPS status snapshot, if any. + pub fn ups_status(&self) -> Option<&UpsStatusState> { + self.ups_status.as_ref() + } +} + +// --------------------------------------------------------------------------- +// StatusStore handle +// --------------------------------------------------------------------------- + +/// Pure shared status state used by the data plane and HTTP adapter. +#[derive(Clone)] +pub struct StatusStore { + subsystem: Arc>, + ui_tx: tokio::sync::broadcast::Sender, + status_event_tx: broadcast::Sender, + display_change_tx: broadcast::Sender<()>, + logger: Arc>, + control_clients: + Arc>>>, + download_trackers: Arc< + std::sync::RwLock< + HashMap>>, + >, + >, + reconnect_notifies: Arc>>>, +} + +fn bridge_download_progress_events( + stream_id: String, + tracker: Arc>, + status_event_tx: broadcast::Sender, +) { + if let Ok(tracker) = tracker.try_lock() { + spawn_download_progress_bridge(stream_id, tracker.subscribe(), status_event_tx); + return; + } + + tokio::spawn(async move { + let rx = { + let tracker = tracker.lock().await; + tracker.subscribe() + }; + spawn_download_progress_bridge(stream_id, rx, status_event_tx); + }); +} + +fn spawn_download_progress_bridge( + stream_id: String, + mut rx: broadcast::Receiver, + status_event_tx: broadcast::Sender, +) { + tokio::spawn(async move { + loop { + match rx.recv().await { + Ok(event) => { + let _ = status_event_tx.send(ForwarderStatusEvent::DownloadProgress { + stream_id: stream_id.clone(), + event, + }); + } + Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => { + tracing::debug!(skipped = n, "download status bridge lagged"); + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => break, + } + } + }); +} + +/// Interval at which accumulated read-count changes are broadcast to P2P +/// control sessions as `ReaderStatus` deltas. +const READ_COUNT_BROADCAST_INTERVAL: Duration = Duration::from_secs(2); + +/// Broadcast a `ReaderStatus` delta for every reader whose read counters +/// changed since the last tick (see `record_read`). +/// +/// Sends while holding the `SubsystemStatus` lock so deltas stay ordered +/// against `subscribe_and_snapshot`, like every other status broadcast. +/// Broadcasting current state (not increments) makes a dropped or duplicated +/// delta harmless: the next tick carries the up-to-date counters. +pub(crate) async fn broadcast_dirty_read_counts( + subsystem: &Mutex, + status_event_tx: &broadcast::Sender, +) { + let mut ss = subsystem.lock().await; + if ss.read_counts_dirty.is_empty() { + return; + } + let dirty = std::mem::take(&mut ss.read_counts_dirty); + for reader_ip in dirty { + if let Some(status) = ss.readers.get(&reader_ip) { + let _ = status_event_tx.send(ForwarderStatusEvent::ReaderStatus { + stream_id: reader_ip, + status: status.clone(), + }); + } + } +} + +/// Spawn the coalescing task that pushes read-count updates to P2P peers at a +/// bounded rate. Quiet when no reads arrive; at most one delta per reader per +/// interval during bursts. +fn spawn_read_count_broadcaster( + subsystem: Arc>, + status_event_tx: broadcast::Sender, +) { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(READ_COUNT_BROADCAST_INTERVAL); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + ticker.tick().await; + broadcast_dirty_read_counts(&subsystem, &status_event_tx).await; + } + }); +} +impl StatusStore { + /// Create a status store with fresh UI and P2P status broadcast channels. + pub fn new(subsystem: SubsystemStatus) -> Self { + let (ui_tx, _) = tokio::sync::broadcast::channel(256); + let (status_event_tx, _) = broadcast::channel(256); + let (display_change_tx, _) = broadcast::channel(256); + let logger = Arc::new(rt_ui_log::UiLogger::with_buffer( + ui_tx.clone(), + |entry| crate::ui_events::ForwarderUiEvent::LogEntry { entry }, + 500, + )); + let subsystem = Arc::new(Mutex::new(subsystem)); + let store = Self { + subsystem: subsystem.clone(), + ui_tx, + status_event_tx: status_event_tx.clone(), + display_change_tx, + logger, + control_clients: Arc::new(std::sync::RwLock::new(HashMap::new())), + download_trackers: Arc::new(std::sync::RwLock::new(HashMap::new())), + reconnect_notifies: Arc::new(std::sync::RwLock::new(HashMap::new())), + }; + spawn_read_count_broadcaster(subsystem, status_event_tx); + store + } + + /// Return a clone of the P2P status event sender for HTTP adapter state. + pub(crate) fn status_event_sender(&self) -> broadcast::Sender { + self.status_event_tx.clone() + } + + /// Subscribe to state changes that should refresh the optional local display. + pub(crate) fn subscribe_display_changes(&self) -> broadcast::Receiver<()> { + self.display_change_tx.subscribe() + } + + fn notify_display_changed(&self) { + let _ = self.display_change_tx.send(()); + } + /// Return a clone of the internal subsystem status Arc. + pub fn subsystem_arc(&self) -> Arc> { + self.subsystem.clone() + } + + /// Return a clone of the UI event broadcast sender. + pub fn ui_sender(&self) -> tokio::sync::broadcast::Sender { + self.ui_tx.clone() + } + + /// Return a read-only status feed for P2P control sessions. + pub fn status_feed(&self) -> ForwarderStatusFeed { + ForwarderStatusFeed { + subsystem: self.subsystem.clone(), + status_event_tx: self.status_event_tx.clone(), + } + } + + /// Return the shared reader-control service used by HTTP and P2P control paths. + pub fn reader_control_service(&self) -> crate::reader_control_service::ReaderControlService { + crate::reader_control_service::ReaderControlService::new( + self.subsystem.clone(), + self.control_clients.clone(), + self.download_trackers.clone(), + self.reconnect_notifies.clone(), + self.ui_tx.clone(), + self.status_event_tx.clone(), + self.logger.clone(), + ) + } + + /// Return a clone of the shared UI logger Arc. + pub fn logger(&self) -> Arc> { + self.logger.clone() + } + + /// Mark all local subsystems as ready. + pub async fn set_ready(&self) { + { + let mut ss = self.subsystem.lock().await; + ss.ready = true; + ss.reason = None; + let _ = self + .ui_tx + .send(crate::ui_events::ForwarderUiEvent::StatusChanged { + ready: ss.is_ready(), + p2p_connected: ss.p2p_connected(), + restart_needed: ss.restart_needed(), + }); + } + self.notify_display_changed(); + } + + /// Mark that a restart is needed to apply saved config changes. + pub async fn set_restart_needed(&self) { + mark_restart_needed_and_emit(&self.subsystem, &self.ui_tx).await; + } + + /// Return whether a restart is needed to apply saved config changes. + pub async fn restart_needed(&self) -> bool { + self.subsystem.lock().await.restart_needed() + } + + pub async fn set_p2p_endpoint_id(&self, endpoint_id: String) { + let mut ss = self.subsystem.lock().await; + ss.set_p2p_endpoint_id(endpoint_id); + } + + /// Update the P2P session state (does not affect readiness). + pub async fn set_p2p_connected(&self, connected: bool) { + { + let mut ss = self.subsystem.lock().await; + ss.set_p2p_connected(connected); + let _ = self + .ui_tx + .send(crate::ui_events::ForwarderUiEvent::StatusChanged { + ready: ss.is_ready(), + p2p_connected: connected, + restart_needed: ss.restart_needed(), + }); + } + self.notify_display_changed(); + } + + /// Set the forwarder ID (call once at startup). + pub async fn set_forwarder_id(&self, id: &str) { + self.subsystem.lock().await.forwarder_id = id.to_owned(); + } + + /// Set the detected local IP (call once at startup). + pub async fn set_local_ip(&self, ip: Option) { + self.subsystem.lock().await.local_ip = ip; + self.notify_display_changed(); + } + + /// Set the update mode (controls check-only vs check-and-download behavior). + pub async fn set_update_mode(&self, mode: rt_updater::UpdateMode) { + self.subsystem.lock().await.update_mode = mode; + } + + /// Update the current rt-updater status (shown on `/update/status`). + pub async fn set_update_status(&self, status: UpdateStatus) { + self.subsystem.lock().await.update_status = status.clone(); + let _ = self + .ui_tx + .send(crate::ui_events::ForwarderUiEvent::UpdateStatusChanged { status }); + } + + /// Record the filesystem path of a downloaded update artifact ready to apply. + pub async fn set_staged_update_path(&self, path: std::path::PathBuf) { + self.subsystem.lock().await.staged_update_path = Some(path); + } + + /// Update the UPS status snapshot in the subsystem state. + /// + /// The local HTTP/UI snapshot is always updated, but the P2P control-event + /// broadcast only fires when the UPS status actually changed from the stored + /// previous value. The UPS poller calls this every interval, so gating the + /// broadcast on a real change avoids spamming connected receivers with + /// identical `UpsStatus` frames. The send happens under the subsystem lock so + /// it is ordered against `subscribe_and_snapshot`. + pub async fn set_ups_status(&self, state: UpsStatusState) { + { + let mut ss = self.subsystem.lock().await; + let changed = ss.ups_status() != Some(&state); + ss.set_ups_status(state.clone()); + if changed { + let _ = self + .status_event_tx + .send(ForwarderStatusEvent::UpsStatus(state)); + } + } + self.notify_display_changed(); + } + + pub fn control_clients( + &self, + ) -> &Arc>>> { + &self.control_clients + } + + #[allow(clippy::type_complexity)] + pub fn download_trackers( + &self, + ) -> &Arc< + std::sync::RwLock< + HashMap>>, + >, + > { + &self.download_trackers + } + + pub fn register_download_tracker( + &self, + reader_ip: &str, + tracker: Arc>, + ) { + self.download_trackers + .write() + .unwrap_or_else(|e| e.into_inner()) + .insert(reader_ip.to_owned(), tracker.clone()); + bridge_download_progress_events( + reader_ip.to_owned(), + tracker, + self.status_event_tx.clone(), + ); + } + + pub fn deregister_download_tracker(&self, reader_ip: &str) { + self.download_trackers + .write() + .unwrap_or_else(|e| e.into_inner()) + .remove(reader_ip); + } + + pub fn register_reconnect_notify(&self, reader_ip: &str, notify: Arc) { + self.reconnect_notifies + .write() + .unwrap_or_else(|e| e.into_inner()) + .insert(reader_ip.to_owned(), notify); + } + + pub fn deregister_reconnect_notify(&self, reader_ip: &str) { + self.reconnect_notifies + .write() + .unwrap_or_else(|e| e.into_inner()) + .remove(reader_ip); + } + + pub fn reconnect_notifies(&self) -> &Arc>>> { + &self.reconnect_notifies + } + + /// Retrieve a clone of the cached reader info for a given reader IP. + pub async fn get_reader_info( + &self, + reader_ip: &str, + ) -> Option { + let ss = self.subsystem.lock().await; + ss.readers + .get(reader_ip) + .and_then(|r| r.reader_info.clone()) + } + + pub async fn update_reader_info( + &self, + reader_ip: &str, + info: crate::reader_control::ReaderInfo, + ) { + { + let mut ss = self.subsystem.lock().await; + if let Some(r) = ss.readers.get_mut(reader_ip) { + r.reader_info = Some(info.clone()); + } + let _ = self + .ui_tx + .send(crate::ui_events::ForwarderUiEvent::ReaderInfoUpdated { + ip: reader_ip.to_owned(), + info: info.clone(), + }); + // Broadcast under the lock so it is ordered against + // `subscribe_and_snapshot`. + let _ = self.status_event_tx.send(ForwarderStatusEvent::ReaderInfo { + stream_id: reader_ip.to_owned(), + info, + }); + } + self.notify_display_changed(); + } + + /// Update reader info only if the reader has not transitioned to Disconnected. + /// + /// This is used by the background poller so a late poll result cannot restore + /// stale info after the read loop has already marked the reader disconnected. + pub async fn update_reader_info_unless_disconnected( + &self, + reader_ip: &str, + info: crate::reader_control::ReaderInfo, + ) { + { + let mut ss = self.subsystem.lock().await; + if let Some(r) = ss.readers.get_mut(reader_ip) { + if r.state == ReaderConnectionState::Disconnected { + tracing::debug!( + reader_ip, + "dropping reader info update for disconnected reader" + ); + return; + } + r.reader_info = Some(info.clone()); + } else { + tracing::debug!( + reader_ip, + "reader IP not found in map, skipping info update" + ); + return; + } + let _ = self + .ui_tx + .send(crate::ui_events::ForwarderUiEvent::ReaderInfoUpdated { + ip: reader_ip.to_owned(), + info: info.clone(), + }); + // Broadcast under the lock so it is ordered against + // `subscribe_and_snapshot`. + let _ = self.status_event_tx.send(ForwarderStatusEvent::ReaderInfo { + stream_id: reader_ip.to_owned(), + info, + }); + } + self.notify_display_changed(); + } + + pub fn register_control_client( + &self, + reader_ip: &str, + client: Arc, + ) { + self.control_clients + .write() + .unwrap_or_else(|e| e.into_inner()) + .insert(reader_ip.to_owned(), client); + } + + pub fn deregister_control_client(&self, reader_ip: &str) { + self.control_clients + .write() + .unwrap_or_else(|e| e.into_inner()) + .remove(reader_ip); + } + + /// Pre-populate all configured reader IPs as Disconnected. + /// + /// Each entry is `(reader_addr, local_port)` where `reader_addr` is `"ip:port"` + /// and `local_port` is the port the forwarder listens on to re-expose reads. + pub async fn init_readers(&self, readers: &[(String, u16)]) { + { + let mut ss = self.subsystem.lock().await; + for (addr, local_port) in readers { + ss.readers.entry(addr.clone()).or_insert(ReaderStatus { + state: ReaderConnectionState::Disconnected, + last_seen: None, + reads_since_restart: 0, + reads_total: 0, + local_port: *local_port, + current_epoch_name: None, + reader_info: None, + }); + } + } + self.notify_display_changed(); + } + + /// Seed a reader's total historical count from durable journal state. + pub async fn set_reader_total(&self, reader_ip: &str, total: i64) { + { + let mut ss = self.subsystem.lock().await; + if let Some(r) = ss.readers.get_mut(reader_ip) { + r.reads_total = total; + } + } + self.notify_display_changed(); + } + + /// Set the current epoch name for a reader and broadcast a ReaderUpdated SSE event. + pub async fn set_reader_epoch_name(&self, reader_ip: &str, name: Option) { + { + let mut ss = self.subsystem.lock().await; + if let Some(r) = ss.readers.get_mut(reader_ip) { + r.current_epoch_name = name; + let _ = self + .ui_tx + .send(crate::ui_events::ForwarderUiEvent::ReaderUpdated { + ip: reader_ip.to_owned(), + state: (&r.state).into(), + reads_session: r.reads_since_restart, + reads_total: r.reads_total, + last_seen_secs: r.last_seen.map(|t| t.elapsed().as_secs()), + local_port: r.local_port, + current_epoch_name: r.current_epoch_name.clone(), + }); + } + } + self.notify_display_changed(); + } + + /// Update a reader's connection state. + pub async fn update_reader_state(&self, reader_ip: &str, state: ReaderConnectionState) { + { + let mut ss = self.subsystem.lock().await; + if let Some(r) = ss.readers.get_mut(reader_ip) { + let changed = r.state != state; + if state == ReaderConnectionState::Disconnected { + r.reader_info = None; + } + r.state = state; + let _ = self + .ui_tx + .send(crate::ui_events::ForwarderUiEvent::ReaderUpdated { + ip: reader_ip.to_owned(), + state: (&r.state).into(), + reads_session: r.reads_since_restart, + reads_total: r.reads_total, + last_seen_secs: r.last_seen.map(|t| t.elapsed().as_secs()), + local_port: r.local_port, + current_epoch_name: r.current_epoch_name.clone(), + }); + // Only broadcast a P2P control delta on an actual state + // transition, and do so under the lock so it is ordered against + // `subscribe_and_snapshot`. The local UI event above is always + // emitted. + if changed { + let _ = self + .status_event_tx + .send(ForwarderStatusEvent::ReaderStatus { + stream_id: reader_ip.to_owned(), + status: r.clone(), + }); + } + } + } + self.notify_display_changed(); + } + + /// Record a successful chip read for a reader. + pub async fn record_read(&self, reader_ip: &str) { + { + let mut ss = self.subsystem.lock().await; + if let Some(r) = ss.readers.get_mut(reader_ip) { + r.reads_since_restart += 1; + r.reads_total += 1; + r.last_seen = Some(Instant::now()); + let _ = self + .ui_tx + .send(crate::ui_events::ForwarderUiEvent::ReaderUpdated { + ip: reader_ip.to_owned(), + state: (&r.state).into(), + reads_session: r.reads_since_restart, + reads_total: r.reads_total, + last_seen_secs: r.last_seen.map(|t| t.elapsed().as_secs()), + local_port: r.local_port, + current_epoch_name: r.current_epoch_name.clone(), + }); + // Per-read P2P broadcasts would flood the control stream during + // bursts, so reads only mark the reader dirty; the coalescing + // task broadcasts the latest counters at a bounded rate. + ss.read_counts_dirty.insert(reader_ip.to_owned()); + } + } + self.notify_display_changed(); + } +} + +pub(crate) async fn mark_restart_needed_and_emit( + subsystem: &Arc>, + ui_tx: &tokio::sync::broadcast::Sender, +) { + let mut ss = subsystem.lock().await; + ss.set_restart_needed(); + let _ = ui_tx.send(crate::ui_events::ForwarderUiEvent::StatusChanged { + ready: ss.is_ready(), + p2p_connected: ss.p2p_connected(), + restart_needed: true, + }); +} diff --git a/services/forwarder/src/ups_task.rs b/services/forwarder/src/ups_task.rs index 350521ff..90ec8d36 100644 --- a/services/forwarder/src/ups_task.rs +++ b/services/forwarder/src/ups_task.rs @@ -7,7 +7,7 @@ use tokio::sync::{mpsc, watch}; use tracing::{info, warn}; use crate::config::UpsConfig; -use crate::status_http::{StatusServer, UpsStatusState}; +use crate::status_store::{StatusStore, UpsStatusState}; use crate::ui_events::ForwarderUiEvent; /// Handle returned by [`spawn_ups_task`] carrying the upstream status channel. @@ -22,7 +22,7 @@ pub struct UpsTaskHandle { pub fn spawn_ups_task( config: UpsConfig, forwarder_id: String, - status: StatusServer, + status: StatusStore, mut shutdown_rx: watch::Receiver, ) -> UpsTaskHandle { let (ups_tx, ups_rx) = mpsc::unbounded_channel(); @@ -39,7 +39,7 @@ pub fn spawn_ups_task( async fn run_ups_loop( config: UpsConfig, forwarder_id: String, - status: StatusServer, + status: StatusStore, shutdown_rx: &mut watch::Receiver, ups_tx: mpsc::UnboundedSender, ) { @@ -234,19 +234,9 @@ mod tests { }; let (shutdown_tx, shutdown_rx) = watch::channel(false); + let status = StatusStore::new(crate::status_store::SubsystemStatus::ready()); - // Need a StatusServer — use a minimal one - let server = crate::status_http::StatusServer::start( - crate::status_http::StatusConfig { - bind: "127.0.0.1:0".to_owned(), - forwarder_version: "test".to_owned(), - }, - crate::status_http::SubsystemStatus::ready(), - ) - .await - .expect("start status server"); - - let mut handle = spawn_ups_task(config, "fwd-test".to_owned(), server, shutdown_rx); + let mut handle = spawn_ups_task(config, "fwd-test".to_owned(), status, shutdown_rx); // Should receive one status message let msg = tokio::time::timeout(Duration::from_secs(5), handle.ups_status_rx.recv()) @@ -274,18 +264,9 @@ mod tests { }; let (shutdown_tx, shutdown_rx) = watch::channel(false); + let status = StatusStore::new(crate::status_store::SubsystemStatus::ready()); - let server = crate::status_http::StatusServer::start( - crate::status_http::StatusConfig { - bind: "127.0.0.1:0".to_owned(), - forwarder_version: "test".to_owned(), - }, - crate::status_http::SubsystemStatus::ready(), - ) - .await - .expect("start status server"); - - let mut handle = spawn_ups_task(config, "fwd-test".to_owned(), server, shutdown_rx); + let mut handle = spawn_ups_task(config, "fwd-test".to_owned(), status, shutdown_rx); let msg = tokio::time::timeout(Duration::from_secs(10), handle.ups_status_rx.recv()) .await @@ -311,18 +292,9 @@ mod tests { }; let (shutdown_tx, shutdown_rx) = watch::channel(false); + let status = StatusStore::new(crate::status_store::SubsystemStatus::ready()); - let server = crate::status_http::StatusServer::start( - crate::status_http::StatusConfig { - bind: "127.0.0.1:0".to_owned(), - forwarder_version: "test".to_owned(), - }, - crate::status_http::SubsystemStatus::ready(), - ) - .await - .expect("start status server"); - - let mut handle = spawn_ups_task(config, "fwd-test".to_owned(), server, shutdown_rx); + let mut handle = spawn_ups_task(config, "fwd-test".to_owned(), status, shutdown_rx); let first = tokio::time::timeout(Duration::from_secs(5), handle.ups_status_rx.recv()) .await diff --git a/services/forwarder/tests/config_endpoints.rs b/services/forwarder/tests/config_endpoints.rs index afcc980d..7993b0e3 100644 --- a/services/forwarder/tests/config_endpoints.rs +++ b/services/forwarder/tests/config_endpoints.rs @@ -1,7 +1,8 @@ //! Integration tests for forwarder config editing endpoints. use forwarder::status_http::{EpochResetError, JournalAccess}; -use forwarder::status_http::{StatusConfig, StatusServer, SubsystemStatus}; +use forwarder::status_http::{StatusConfig, StatusServer}; +use forwarder::status_store::SubsystemStatus; use std::net::SocketAddr; use std::time::Duration; diff --git a/services/forwarder/tests/health_endpoints.rs b/services/forwarder/tests/health_endpoints.rs index 1c577587..3b2edfcf 100644 --- a/services/forwarder/tests/health_endpoints.rs +++ b/services/forwarder/tests/health_endpoints.rs @@ -9,7 +9,8 @@ //! 6. status page returns HTML with expected content //! 7. graceful shutdown handler registered -use forwarder::status_http::{StatusConfig, StatusServer, SubsystemStatus}; +use forwarder::status_http::{StatusConfig, StatusServer}; +use forwarder::status_store::SubsystemStatus; use serde_json::Value; use std::net::SocketAddr; use std::sync::Arc; @@ -675,7 +676,7 @@ async fn status_json_shows_forwarder_id() { #[tokio::test] async fn status_json_shows_reader_status() { - use forwarder::status_http::ReaderConnectionState; + use forwarder::status_store::ReaderConnectionState; let cfg = StatusConfig { bind: "127.0.0.1:0".to_owned(), diff --git a/services/forwarder/tests/ui_embed_fallback.rs b/services/forwarder/tests/ui_embed_fallback.rs index 75520d7f..61b52530 100644 --- a/services/forwarder/tests/ui_embed_fallback.rs +++ b/services/forwarder/tests/ui_embed_fallback.rs @@ -1,6 +1,7 @@ #![cfg(feature = "embed-ui")] -use forwarder::status_http::{StatusConfig, StatusServer, SubsystemStatus}; +use forwarder::status_http::{StatusConfig, StatusServer}; +use forwarder::status_store::SubsystemStatus; use std::time::Duration; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; From 2b172127f409ab9afed484109169e8a052d5b3ee Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 16:06:34 -0400 Subject: [PATCH 23/38] refactor(forwarder): move display projection and updater state out of status_http; dedupe clock-sync helper --- services/forwarder/src/config_service.rs | 355 +++++++- .../forwarder/src/reader_control_service.rs | 55 ++ services/forwarder/src/status_http.rs | 800 ++---------------- services/forwarder/src/status_store.rs | 230 ++++- 4 files changed, 675 insertions(+), 765 deletions(-) diff --git a/services/forwarder/src/config_service.rs b/services/forwarder/src/config_service.rs index 2aecdbc4..9cb832c3 100644 --- a/services/forwarder/src/config_service.rs +++ b/services/forwarder/src/config_service.rs @@ -5,15 +5,14 @@ //! local status HTTP surface and the P2P remote-config verbs, and the //! per-section update logic behind `POST /api/v1/config/{section}`. +use std::future::Future; use std::io::Write as _; use std::net::SocketAddrV4; +use std::pin::Pin; use std::sync::Arc; -use tokio::sync::Mutex; +use tokio::sync::{Mutex, Notify}; -use crate::status_http::{ - apply_control_action_from_config, bad_request_error, require_object_payload, -}; use crate::status_store::{SubsystemStatus, mark_restart_needed_and_emit}; /// Holds the config file path and a write lock for read-modify-write operations. @@ -31,6 +30,294 @@ impl ConfigState { } } +pub(crate) fn bad_request_error(message: impl Into) -> (u16, String) { + ( + 400u16, + serde_json::json!({"ok": false, "error": message.into()}).to_string(), + ) +} + +pub(crate) fn require_object_payload(payload: &serde_json::Value) -> Result<(), (u16, String)> { + if payload.is_object() { + Ok(()) + } else { + Err(bad_request_error("payload must be a JSON object")) + } +} + +async fn read_allow_power_actions(config_state: &ConfigState) -> Result { + let _lock = config_state.write_lock.lock().await; + let toml_str = std::fs::read_to_string(&config_state.path).map_err(|e| { + ( + 500u16, + serde_json::json!({"ok": false, "error": format!("File read error: {}", e)}) + .to_string(), + ) + })?; + let raw: crate::config::RawConfig = toml::from_str(&toml_str).map_err(|e| { + ( + 500u16, + serde_json::json!({"ok": false, "error": format!("TOML parse error: {}", e)}) + .to_string(), + ) + })?; + Ok(raw + .control + .and_then(|c| c.allow_power_actions) + .unwrap_or(false)) +} + +#[cfg(unix)] +pub(crate) fn map_power_action_command_result( + systemctl_action: &'static str, + result: std::io::Result, + logger: Option<&rt_ui_log::UiLogger>, +) -> Result<(), (u16, String)> { + match result { + Ok(output) if output.status.success() => Ok(()), + Ok(output) => { + let detail = power_action_command_detail(&output); + let status_code = if power_action_auth_failed(&detail) { + 403u16 + } else { + 500u16 + }; + tracing::error!( + action = systemctl_action, + exit_status = ?output.status.code(), + detail = %detail, + "control action command exited with failure" + ); + if let Some(logger) = logger { + logger.log_at( + rt_ui_log::UiLogLevel::Error, + format!( + "systemctl {} exited with failure (code {:?})", + systemctl_action, + output.status.code(), + ), + ); + } + Err(( + status_code, + serde_json::json!({ + "ok": false, + "error": format!( + "control action command exited with failure: systemctl {} ({})", + systemctl_action, + detail + ) + }) + .to_string(), + )) + } + Err(e) => { + tracing::error!(action = systemctl_action, error = %e, "control action command failed"); + if let Some(logger) = logger { + logger.log_at( + rt_ui_log::UiLogLevel::Error, + format!("systemctl {} failed: {}", systemctl_action, e), + ); + } + Err(( + 500u16, + serde_json::json!({ + "ok": false, + "error": format!("control action command failed: {}", e) + }) + .to_string(), + )) + } + } +} + +#[cfg(unix)] +fn map_power_action_join_error( + systemctl_action: &'static str, + e: tokio::task::JoinError, + logger: Option<&rt_ui_log::UiLogger>, +) -> (u16, String) { + tracing::error!(action = systemctl_action, error = %e, "control action task failed"); + if let Some(logger) = logger { + logger.log_at( + rt_ui_log::UiLogLevel::Error, + format!("systemctl {} task failed: {}", systemctl_action, e), + ); + } + ( + 500u16, + serde_json::json!({ + "ok": false, + "error": format!("control action task failed: {}", e) + }) + .to_string(), + ) +} + +#[cfg(unix)] +async fn run_device_power_action( + systemctl_action: &'static str, + logger: Option<&rt_ui_log::UiLogger>, +) -> Result<(), (u16, String)> { + match tokio::task::spawn_blocking(move || run_power_action_command(systemctl_action)).await { + Ok(result) => map_power_action_command_result(systemctl_action, result, logger), + Err(e) => Err(map_power_action_join_error(systemctl_action, e, logger)), + } +} + +#[cfg(unix)] +fn run_power_action_command( + systemctl_action: &'static str, +) -> std::io::Result { + std::process::Command::new("systemctl") + .arg("--no-ask-password") + .arg(systemctl_action) + .output() +} + +#[cfg(unix)] +fn power_action_command_detail(output: &std::process::Output) -> String { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); + if !stderr.is_empty() { + return stderr; + } + let stdout = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + if !stdout.is_empty() { + return stdout; + } + "no command output".to_owned() +} + +#[cfg(unix)] +fn power_action_auth_failed(detail: &str) -> bool { + let lower = detail.to_ascii_lowercase(); + lower.contains("interactive authentication required") + || lower.contains("authentication is required") + || lower.contains("not authorized") + || lower.contains("access denied") + || lower.contains("permission denied") + || lower.contains("a password is required") +} + +#[cfg(not(unix))] +async fn run_device_power_action( + _systemctl_action: &'static str, + _logger: Option<&rt_ui_log::UiLogger>, +) -> Result<(), (u16, String)> { + Err(( + 501u16, + serde_json::json!({ + "ok": false, + "error": "power actions not supported on non-unix platforms" + }) + .to_string(), + )) +} + +pub(crate) async fn apply_control_action_from_config( + action: &str, + config_state: &ConfigState, + logger: Option<&rt_ui_log::UiLogger>, +) -> Result<(), (u16, String)> { + apply_control_action_from_config_with( + action, + config_state, + logger, + |action, config_state, restart_signal, logger| { + Box::pin(async move { + apply_control_action(action, config_state, restart_signal, logger).await + }) + }, + ) + .await +} + +async fn apply_control_action_from_config_with( + action: &str, + config_state: &ConfigState, + logger: Option<&rt_ui_log::UiLogger>, + apply_fn: F, +) -> Result<(), (u16, String)> +where + F: for<'a> FnOnce( + &'a str, + Option<&'a ConfigState>, + Option<&'a Arc>, + Option<&'a rt_ui_log::UiLogger>, + ) -> Pin> + Send + 'a>>, +{ + apply_fn(action, Some(config_state), None, logger).await +} + +pub async fn apply_control_action( + action: &str, + config_state: Option<&ConfigState>, + restart_signal: Option<&Arc>, + logger: Option<&rt_ui_log::UiLogger>, +) -> Result<(), (u16, String)> { + match action { + "restart_service" => { + let signal = restart_signal.ok_or_else(|| { + ( + 404u16, + serde_json::json!({"ok": false, "error": "restart signal not available"}) + .to_string(), + ) + })?; + if cfg!(unix) { + signal.notify_one(); + Ok(()) + } else { + Err(( + 501u16, + serde_json::json!({ + "ok": false, + "error": "restart not supported on non-unix platforms" + }) + .to_string(), + )) + } + } + "restart_device" | "shutdown_device" => { + let cs = config_state.ok_or_else(|| { + ( + 404u16, + serde_json::json!({"ok": false, "error": "Config editing not available"}) + .to_string(), + ) + })?; + let allow_power_actions = read_allow_power_actions(cs).await?; + if !allow_power_actions { + return Err(( + 403u16, + serde_json::json!({"ok": false, "error": "power actions disabled"}).to_string(), + )); + } + if !cfg!(unix) { + return Err(( + 501u16, + serde_json::json!({ + "ok": false, + "error": format!("{} not supported on non-unix platforms", action) + }) + .to_string(), + )); + } + let systemctl_action = if action == "restart_device" { + "reboot" + } else { + "poweroff" + }; + run_device_power_action(systemctl_action, logger).await?; + Ok(()) + } + _ => Err(bad_request_error(format!( + "unknown control action: {}", + action + ))), + } +} + fn write_atomic(path: &std::path::Path, content: &str) -> std::io::Result<()> { let original_permissions = std::fs::metadata(path).map(|m| m.permissions()).ok(); @@ -733,3 +1020,63 @@ fn validate_server_url(url: &str) -> Result<(), String> { Err("server_url must start with http:// or https://".to_owned()) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicBool, Ordering}; + + #[cfg(unix)] + #[tokio::test] + async fn power_action_join_error_logs_to_ui_when_logger_present() { + let (tx, mut rx) = tokio::sync::broadcast::channel(16); + let logger = rt_ui_log::UiLogger::new(tx, |entry| { + crate::ui_events::ForwarderUiEvent::LogEntry { entry } + }); + + let join_err = tokio::task::spawn_blocking(|| -> () { + panic!("boom"); + }) + .await + .expect_err("task must panic"); + + let (_status, _body) = map_power_action_join_error("reboot", join_err, Some(&logger)); + + let evt = rx.try_recv().expect("expected UI log event"); + match evt { + crate::ui_events::ForwarderUiEvent::LogEntry { entry } => { + assert!(entry.contains("systemctl reboot task failed")); + } + other => panic!("unexpected event: {other:?}"), + } + } + + #[tokio::test] + async fn control_action_from_config_forwards_logger_to_apply_fn() { + let (tx, _) = tokio::sync::broadcast::channel(16); + let logger = rt_ui_log::UiLogger::new(tx, |entry| { + crate::ui_events::ForwarderUiEvent::LogEntry { entry } + }); + let config = ConfigState::new(std::path::PathBuf::from("/tmp/unused.toml")); + + let saw_logger = Arc::new(AtomicBool::new(false)); + let spy = Arc::clone(&saw_logger); + + let _ = apply_control_action_from_config_with( + "restart_device", + &config, + Some(&logger), + move |_action, _config_state, _restart_signal, logger| { + let spy = Arc::clone(&spy); + let has_logger = logger.is_some(); + Box::pin(async move { + spy.store(has_logger, Ordering::SeqCst); + Err((500u16, "{\"ok\":false}".to_owned())) + }) + }, + ) + .await; + + assert!(saw_logger.load(Ordering::SeqCst)); + } +} diff --git a/services/forwarder/src/reader_control_service.rs b/services/forwarder/src/reader_control_service.rs index 19590e0e..dd06b93a 100644 --- a/services/forwarder/src/reader_control_service.rs +++ b/services/forwarder/src/reader_control_service.rs @@ -575,4 +575,59 @@ mod tests { assert!(target > wall_now); assert_eq!(wait, std::time::Duration::from_millis(375)); } + + #[test] + fn sync_timing_rollover_frac_above_half_rounds_up() { + let wall_now = chrono::Local + .with_ymd_and_hms(2026, 3, 8, 12, 0, 0) + .unwrap() + .with_nanosecond(200_000_000) + .unwrap(); + let one_way = std::time::Duration::from_millis(50); + let (target, wait) = compute_sync_timing(wall_now, one_way, 500); + assert_eq!(target.second(), 1); + assert_eq!(target.nanosecond(), 0); + assert!(wait < std::time::Duration::from_secs(1)); + } + + #[test] + fn sync_timing_rollover_frac_below_half_stays_same_second() { + let wall_now = chrono::Local + .with_ymd_and_hms(2026, 3, 8, 12, 0, 0) + .unwrap() + .with_nanosecond(800_000_000) + .unwrap(); + let one_way = std::time::Duration::from_millis(50); + let (target, _wait) = compute_sync_timing(wall_now, one_way, 500); + assert_eq!(target.second(), 2); + assert_eq!(target.nanosecond(), 0); + } + + #[test] + fn sync_timing_ideal_send_past_bumps_target() { + let wall_now = chrono::Local + .with_ymd_and_hms(2026, 3, 8, 12, 0, 0) + .unwrap() + .with_nanosecond(500_000_000) + .unwrap(); + let one_way = std::time::Duration::from_millis(1); + let (target, wait) = compute_sync_timing(wall_now, one_way, 500); + assert_eq!(target.second(), 2); + assert_eq!(target.nanosecond(), 0); + assert!(wait > std::time::Duration::from_millis(900)); + assert!(wait < std::time::Duration::from_millis(1100)); + } + + #[test] + fn sync_timing_zero_latency() { + let wall_now = chrono::Local + .with_ymd_and_hms(2026, 3, 8, 12, 0, 0) + .unwrap() + .with_nanosecond(300_000_000) + .unwrap(); + let one_way = std::time::Duration::ZERO; + let (target, _wait) = compute_sync_timing(wall_now, one_way, 500); + assert_eq!(target.second(), 1); + assert_eq!(target.nanosecond(), 0); + } } diff --git a/services/forwarder/src/status_http.rs b/services/forwarder/src/status_http.rs index 3d8e34b7..93055942 100644 --- a/services/forwarder/src/status_http.rs +++ b/services/forwarder/src/status_http.rs @@ -46,7 +46,7 @@ //! # Security //! No authentication in v1. -use crate::config_service::ConfigState; +use crate::config_service::{ConfigState, apply_control_action, require_object_payload}; use crate::status_store::{ ForwarderStatusEvent, ForwarderStatusFeed, ReaderConnectionState, StatusStore, SubsystemStatus, UpsStatusState, @@ -62,13 +62,11 @@ use axum::response::sse::{Event as SseEvent, KeepAlive, Sse}; use axum::response::{IntoResponse, Response}; use axum::routing::{get, post, put}; use rt_updater::UpdateStatus; -use rt_updater::workflow::{RealChecker, WorkflowState, run_check, run_download}; +use rt_updater::workflow::{RealChecker, run_check, run_download}; use serde::de::DeserializeOwned; use std::collections::HashMap; use std::convert::Infallible; -use std::future::Future; use std::net::SocketAddr; -use std::pin::Pin; use std::sync::Arc; use std::time::Duration; #[cfg(test)] @@ -89,62 +87,6 @@ pub struct StatusConfig { pub forwarder_version: String, } -#[cfg(any(feature = "eink", feature = "lcd"))] -fn subsystem_to_display_state( - ss: &SubsystemStatus, - forwarder_name: Option, - cpu_temp: Option, -) -> rt_screen::state::DisplayState { - let readers = ss - .readers() - .iter() - .map(|(addr, r)| { - // Reader addresses are "ip:port" — extract just the IP. - let ip = addr - .rsplit_once(':') - .map_or(addr.as_str(), |(ip, _)| ip) - .to_owned(); - rt_screen::state::ReaderDisplayState { - ip, - state: match r.state { - ReaderConnectionState::Connected => { - rt_screen::state::ReaderConnectionState::Connected - } - ReaderConnectionState::Connecting => { - rt_screen::state::ReaderConnectionState::Connecting - } - ReaderConnectionState::Disconnected => { - rt_screen::state::ReaderConnectionState::Disconnected - } - }, - drift_ms: r - .reader_info - .as_ref() - .and_then(|info| info.clock.as_ref()) - .map(|c| c.drift_ms), - session_reads: r.reads_since_restart, - } - }) - .collect(); - - let total_reads: u64 = ss.readers().values().map(|r| r.reads_since_restart).sum(); - - rt_screen::state::DisplayState { - forwarder_name, - local_ip: ss.local_ip.clone(), - p2p_connected: ss.p2p_connected(), - readers, - total_reads, - cpu_temp_celsius: cpu_temp, - battery: ss.ups_status().and_then(|u| { - u.status.as_ref().map(|s| rt_screen::state::BatteryState { - percent: s.battery_percent, - charging: s.charging, - }) - }), - } -} - // --------------------------------------------------------------------------- // StatusServer handle // --------------------------------------------------------------------------- @@ -154,15 +96,10 @@ fn subsystem_to_display_state( pub struct StatusServer { local_addr: SocketAddr, store: StatusStore, - #[cfg(any(feature = "eink", feature = "lcd"))] - display_tx: Option>, - #[cfg(any(feature = "eink", feature = "lcd"))] - display_name: Arc>>, - #[cfg(any(feature = "eink", feature = "lcd"))] - cpu_temp: Arc>>, } struct AppState { + store: StatusStore, subsystem: Arc>, journal: Arc>, version: Arc, @@ -179,10 +116,6 @@ struct AppState { >, >, reconnect_notifies: Arc>>>, - #[cfg(any(feature = "eink", feature = "lcd"))] - display_name: Arc>>, - #[cfg(any(feature = "eink", feature = "lcd"))] - cpu_temp: Arc>>, } impl AppState { @@ -202,6 +135,7 @@ impl AppState { impl Clone for AppState { fn clone(&self) -> Self { Self { + store: self.store.clone(), subsystem: self.subsystem.clone(), journal: self.journal.clone(), version: self.version.clone(), @@ -213,10 +147,6 @@ impl Clone for AppState { control_clients: self.control_clients.clone(), download_trackers: self.download_trackers.clone(), reconnect_notifies: self.reconnect_notifies.clone(), - #[cfg(any(feature = "eink", feature = "lcd"))] - display_name: self.display_name.clone(), - #[cfg(any(feature = "eink", feature = "lcd"))] - cpu_temp: self.cpu_temp.clone(), } } } @@ -260,8 +190,6 @@ impl StatusServer { /// Mark all local subsystems as ready. pub async fn set_ready(&self) { self.store.set_ready().await; - #[cfg(any(feature = "eink", feature = "lcd"))] - self.publish_display_state().await; } /// Mark that a restart is needed to apply saved config changes. @@ -281,8 +209,6 @@ impl StatusServer { /// Update the P2P session state (does not affect readiness). pub async fn set_p2p_connected(&self, connected: bool) { self.store.set_p2p_connected(connected).await; - #[cfg(any(feature = "eink", feature = "lcd"))] - self.publish_display_state().await; } /// Set the forwarder ID (call once at startup). @@ -293,8 +219,6 @@ impl StatusServer { /// Set the detected local IP (at startup and on reader connect/disconnect). pub async fn set_local_ip(&self, ip: Option) { self.store.set_local_ip(ip).await; - #[cfg(any(feature = "eink", feature = "lcd"))] - self.publish_display_state().await; } /// Set the update mode (controls check-only vs check-and-download behavior). @@ -315,8 +239,6 @@ impl StatusServer { /// Update the UPS status snapshot in the subsystem state. pub async fn set_ups_status(&self, state: UpsStatusState) { self.store.set_ups_status(state).await; - #[cfg(any(feature = "eink", feature = "lcd"))] - self.publish_display_state().await; } pub fn control_clients( @@ -360,59 +282,27 @@ impl StatusServer { self.store.reconnect_notifies() } - #[cfg(any(feature = "eink", feature = "lcd"))] - async fn publish_display_state(&self) { - if let Some(ref tx) = self.display_tx { - let subsystem = self.store.subsystem_arc(); - let ss = subsystem.lock().await; - let forwarder_name = self.display_name.lock().await.clone(); - let cpu_temp = *self.cpu_temp.lock().await; - let state = subsystem_to_display_state(&ss, forwarder_name, cpu_temp); - tx.send_replace(state); - } - } - #[cfg(any(feature = "eink", feature = "lcd"))] pub fn set_display_sender( &mut self, tx: tokio::sync::watch::Sender, ) { - self.display_tx = Some(tx); - self.spawn_display_change_bridge(); - } - - #[cfg(any(feature = "eink", feature = "lcd"))] - fn spawn_display_change_bridge(&self) { - let mut display_changes = self.store.subscribe_display_changes(); - let server = self.clone(); - tokio::spawn(async move { - loop { - match display_changes.recv().await { - Ok(()) => server.publish_display_state().await, - Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => { - server.publish_display_state().await; - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => break, - } - } - }); + self.store.set_display_sender(tx); } #[cfg(any(feature = "eink", feature = "lcd"))] pub async fn set_display_name(&self, name: Option) { - *self.display_name.lock().await = name; - self.publish_display_state().await; + self.store.set_display_name(name).await; } #[cfg(any(feature = "eink", feature = "lcd"))] pub async fn set_cpu_temp(&self, temp: Option) { - self.set_cpu_temp_cached(temp).await; - self.publish_display_state().await; + self.store.set_cpu_temp(temp).await; } #[cfg(any(feature = "eink", feature = "lcd"))] pub async fn set_cpu_temp_cached(&self, temp: Option) { - *self.cpu_temp.lock().await = temp; + self.store.set_cpu_temp_cached(temp).await; } /// Retrieve a clone of the cached reader info for a given reader IP. @@ -429,8 +319,6 @@ impl StatusServer { info: crate::reader_control::ReaderInfo, ) { self.store.update_reader_info(reader_ip, info).await; - #[cfg(any(feature = "eink", feature = "lcd"))] - self.publish_display_state().await; } /// Update reader info only if the reader has not transitioned to Disconnected. @@ -442,8 +330,6 @@ impl StatusServer { self.store .update_reader_info_unless_disconnected(reader_ip, info) .await; - #[cfg(any(feature = "eink", feature = "lcd"))] - self.publish_display_state().await; } pub fn register_control_client( @@ -461,36 +347,26 @@ impl StatusServer { /// Pre-populate all configured reader IPs as Disconnected. pub async fn init_readers(&self, readers: &[(String, u16)]) { self.store.init_readers(readers).await; - #[cfg(any(feature = "eink", feature = "lcd"))] - self.publish_display_state().await; } /// Seed a reader's total historical count from durable journal state. pub async fn set_reader_total(&self, reader_ip: &str, total: i64) { self.store.set_reader_total(reader_ip, total).await; - #[cfg(any(feature = "eink", feature = "lcd"))] - self.publish_display_state().await; } /// Set the current epoch name for a reader and broadcast a ReaderUpdated SSE event. pub async fn set_reader_epoch_name(&self, reader_ip: &str, name: Option) { self.store.set_reader_epoch_name(reader_ip, name).await; - #[cfg(any(feature = "eink", feature = "lcd"))] - self.publish_display_state().await; } /// Update a reader's connection state. pub async fn update_reader_state(&self, reader_ip: &str, state: ReaderConnectionState) { self.store.update_reader_state(reader_ip, state).await; - #[cfg(any(feature = "eink", feature = "lcd"))] - self.publish_display_state().await; } /// Record a successful chip read for a reader. pub async fn record_read(&self, reader_ip: &str) { self.store.record_read(reader_ip).await; - #[cfg(any(feature = "eink", feature = "lcd"))] - self.publish_display_state().await; } /// Start the status HTTP server without a journal (epoch reset returns 404). @@ -511,11 +387,8 @@ impl StatusServer { let local_addr = listener.local_addr()?; let store = StatusStore::new(subsystem); - #[cfg(any(feature = "eink", feature = "lcd"))] - let display_name = Arc::new(Mutex::new(None)); - #[cfg(any(feature = "eink", feature = "lcd"))] - let cpu_temp = Arc::new(Mutex::new(None)); let state = AppState { + store: store.clone(), subsystem: store.subsystem_arc(), journal, version: Arc::new(cfg.forwarder_version), @@ -527,10 +400,6 @@ impl StatusServer { control_clients: store.control_clients().clone(), download_trackers: store.download_trackers().clone(), reconnect_notifies: store.reconnect_notifies().clone(), - #[cfg(any(feature = "eink", feature = "lcd"))] - display_name: display_name.clone(), - #[cfg(any(feature = "eink", feature = "lcd"))] - cpu_temp: cpu_temp.clone(), }; let app = build_router(state); @@ -540,16 +409,7 @@ impl StatusServer { } }); - Ok(StatusServer { - local_addr, - store, - #[cfg(any(feature = "eink", feature = "lcd"))] - display_tx: None, - #[cfg(any(feature = "eink", feature = "lcd"))] - display_name, - #[cfg(any(feature = "eink", feature = "lcd"))] - cpu_temp, - }) + Ok(StatusServer { local_addr, store }) } /// Start the status HTTP server with config editing support. @@ -564,11 +424,8 @@ impl StatusServer { let local_addr = listener.local_addr()?; let store = StatusStore::new(subsystem); - #[cfg(any(feature = "eink", feature = "lcd"))] - let display_name = Arc::new(Mutex::new(None)); - #[cfg(any(feature = "eink", feature = "lcd"))] - let cpu_temp = Arc::new(Mutex::new(None)); let state = AppState { + store: store.clone(), subsystem: store.subsystem_arc(), journal, version: Arc::new(cfg.forwarder_version), @@ -580,10 +437,6 @@ impl StatusServer { control_clients: store.control_clients().clone(), download_trackers: store.download_trackers().clone(), reconnect_notifies: store.reconnect_notifies().clone(), - #[cfg(any(feature = "eink", feature = "lcd"))] - display_name: display_name.clone(), - #[cfg(any(feature = "eink", feature = "lcd"))] - cpu_temp: cpu_temp.clone(), }; let app = build_router(state); @@ -593,16 +446,7 @@ impl StatusServer { } }); - Ok(StatusServer { - local_addr, - store, - #[cfg(any(feature = "eink", feature = "lcd"))] - display_tx: None, - #[cfg(any(feature = "eink", feature = "lcd"))] - display_name, - #[cfg(any(feature = "eink", feature = "lcd"))] - cpu_temp, - }) + Ok(StatusServer { local_addr, store }) } } @@ -679,21 +523,6 @@ fn parse_json_body(body: &Bytes) -> Result { serde_json::from_slice::(body).map_err(|e| format!("Invalid JSON: {}", e)) } -pub(crate) fn bad_request_error(message: impl Into) -> (u16, String) { - ( - 400u16, - serde_json::json!({"ok": false, "error": message.into()}).to_string(), - ) -} - -pub(crate) fn require_object_payload(payload: &serde_json::Value) -> Result<(), (u16, String)> { - if payload.is_object() { - Ok(()) - } else { - Err(bad_request_error("payload must be a JSON object")) - } -} - fn config_not_available() -> Response { text_response(StatusCode::NOT_FOUND, "Config editing not available") } @@ -721,279 +550,6 @@ async fn restart_handler( } } -async fn read_allow_power_actions(config_state: &ConfigState) -> Result { - let _lock = config_state.write_lock.lock().await; - let toml_str = std::fs::read_to_string(&config_state.path).map_err(|e| { - ( - 500u16, - serde_json::json!({"ok": false, "error": format!("File read error: {}", e)}) - .to_string(), - ) - })?; - let raw: crate::config::RawConfig = toml::from_str(&toml_str).map_err(|e| { - ( - 500u16, - serde_json::json!({"ok": false, "error": format!("TOML parse error: {}", e)}) - .to_string(), - ) - })?; - Ok(raw - .control - .and_then(|c| c.allow_power_actions) - .unwrap_or(false)) -} - -#[cfg(unix)] -fn map_power_action_command_result( - systemctl_action: &'static str, - result: std::io::Result, - logger: Option<&rt_ui_log::UiLogger>, -) -> Result<(), (u16, String)> { - match result { - Ok(output) if output.status.success() => Ok(()), - Ok(output) => { - let detail = power_action_command_detail(&output); - let status_code = if power_action_auth_failed(&detail) { - 403u16 - } else { - 500u16 - }; - tracing::error!( - action = systemctl_action, - exit_status = ?output.status.code(), - detail = %detail, - "control action command exited with failure" - ); - if let Some(logger) = logger { - logger.log_at( - rt_ui_log::UiLogLevel::Error, - format!( - "systemctl {} exited with failure (code {:?})", - systemctl_action, - output.status.code(), - ), - ); - } - Err(( - status_code, - serde_json::json!({ - "ok": false, - "error": format!( - "control action command exited with failure: systemctl {} ({})", - systemctl_action, - detail - ) - }) - .to_string(), - )) - } - Err(e) => { - tracing::error!(action = systemctl_action, error = %e, "control action command failed"); - if let Some(logger) = logger { - logger.log_at( - rt_ui_log::UiLogLevel::Error, - format!("systemctl {} failed: {}", systemctl_action, e), - ); - } - Err(( - 500u16, - serde_json::json!({ - "ok": false, - "error": format!("control action command failed: {}", e) - }) - .to_string(), - )) - } - } -} - -#[cfg(unix)] -fn map_power_action_join_error( - systemctl_action: &'static str, - e: tokio::task::JoinError, - logger: Option<&rt_ui_log::UiLogger>, -) -> (u16, String) { - tracing::error!(action = systemctl_action, error = %e, "control action task failed"); - if let Some(logger) = logger { - logger.log_at( - rt_ui_log::UiLogLevel::Error, - format!("systemctl {} task failed: {}", systemctl_action, e), - ); - } - ( - 500u16, - serde_json::json!({ - "ok": false, - "error": format!("control action task failed: {}", e) - }) - .to_string(), - ) -} - -#[cfg(unix)] -async fn run_device_power_action( - systemctl_action: &'static str, - logger: Option<&rt_ui_log::UiLogger>, -) -> Result<(), (u16, String)> { - match tokio::task::spawn_blocking(move || run_power_action_command(systemctl_action)).await { - Ok(result) => map_power_action_command_result(systemctl_action, result, logger), - Err(e) => Err(map_power_action_join_error(systemctl_action, e, logger)), - } -} - -#[cfg(unix)] -fn run_power_action_command( - systemctl_action: &'static str, -) -> std::io::Result { - std::process::Command::new("systemctl") - .arg("--no-ask-password") - .arg(systemctl_action) - .output() -} - -#[cfg(unix)] -fn power_action_command_detail(output: &std::process::Output) -> String { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); - if !stderr.is_empty() { - return stderr; - } - let stdout = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - if !stdout.is_empty() { - return stdout; - } - "no command output".to_owned() -} - -#[cfg(unix)] -fn power_action_auth_failed(detail: &str) -> bool { - let lower = detail.to_ascii_lowercase(); - lower.contains("interactive authentication required") - || lower.contains("authentication is required") - || lower.contains("not authorized") - || lower.contains("access denied") - || lower.contains("permission denied") - || lower.contains("a password is required") -} - -#[cfg(not(unix))] -async fn run_device_power_action( - _systemctl_action: &'static str, - _logger: Option<&rt_ui_log::UiLogger>, -) -> Result<(), (u16, String)> { - Err(( - 501u16, - serde_json::json!({ - "ok": false, - "error": "power actions not supported on non-unix platforms" - }) - .to_string(), - )) -} - -pub(crate) async fn apply_control_action_from_config( - action: &str, - config_state: &ConfigState, - logger: Option<&rt_ui_log::UiLogger>, -) -> Result<(), (u16, String)> { - apply_control_action_from_config_with( - action, - config_state, - logger, - |action, config_state, restart_signal, logger| { - Box::pin(async move { - apply_control_action(action, config_state, restart_signal, logger).await - }) - }, - ) - .await -} - -async fn apply_control_action_from_config_with( - action: &str, - config_state: &ConfigState, - logger: Option<&rt_ui_log::UiLogger>, - apply_fn: F, -) -> Result<(), (u16, String)> -where - F: for<'a> FnOnce( - &'a str, - Option<&'a ConfigState>, - Option<&'a Arc>, - Option<&'a rt_ui_log::UiLogger>, - ) -> Pin> + Send + 'a>>, -{ - apply_fn(action, Some(config_state), None, logger).await -} - -pub async fn apply_control_action( - action: &str, - config_state: Option<&ConfigState>, - restart_signal: Option<&Arc>, - logger: Option<&rt_ui_log::UiLogger>, -) -> Result<(), (u16, String)> { - match action { - "restart_service" => { - let signal = restart_signal.ok_or_else(|| { - ( - 404u16, - serde_json::json!({"ok": false, "error": "restart signal not available"}) - .to_string(), - ) - })?; - if cfg!(unix) { - signal.notify_one(); - Ok(()) - } else { - Err(( - 501u16, - serde_json::json!({ - "ok": false, - "error": "restart not supported on non-unix platforms" - }) - .to_string(), - )) - } - } - "restart_device" | "shutdown_device" => { - let cs = config_state.ok_or_else(|| { - ( - 404u16, - serde_json::json!({"ok": false, "error": "Config editing not available"}) - .to_string(), - ) - })?; - let allow_power_actions = read_allow_power_actions(cs).await?; - if !allow_power_actions { - return Err(( - 403u16, - serde_json::json!({"ok": false, "error": "power actions disabled"}).to_string(), - )); - } - if !cfg!(unix) { - return Err(( - 501u16, - serde_json::json!({ - "ok": false, - "error": format!("{} not supported on non-unix platforms", action) - }) - .to_string(), - )); - } - let systemctl_action = if action == "restart_device" { - "reboot" - } else { - "poweroff" - }; - run_device_power_action(systemctl_action, logger).await?; - Ok(()) - } - _ => Err(bad_request_error(format!( - "unknown control action: {}", - action - ))), - } -} - fn control_action_error_message(body: &str) -> String { serde_json::from_str::(body) .ok() @@ -1351,19 +907,17 @@ async fn status_json_handler( async fn display_state_handler( State(state): State>, ) -> axum::Json { - let ss = state.subsystem.lock().await; #[cfg(any(feature = "eink", feature = "lcd"))] { - let forwarder_name = state.display_name.lock().await.clone(); - let cpu_temp = *state.cpu_temp.lock().await; return axum::Json( - serde_json::to_value(subsystem_to_display_state(&ss, forwarder_name, cpu_temp)) + serde_json::to_value(state.store.display_state().await) .expect("display state serializes"), ); } #[cfg(not(any(feature = "eink", feature = "lcd")))] { + let ss = state.subsystem.lock().await; let readers: Vec = ss .readers .iter() @@ -1480,53 +1034,6 @@ async fn reader_info_handler( } } -/// Compute the target second boundary and pre-SET wait duration for clock sync. -/// -/// Given the current wall time, one-way latency estimate, and the fixed sync delay, -/// returns `(target_boundary, pre_set_wait)` where: -/// - `target_boundary` is the `DateTime` whole-second that the rollover should align with -/// - `pre_set_wait` is how long to sleep before sending SET_DATE_TIME -pub fn compute_sync_timing( - wall_now: chrono::DateTime, - one_way: std::time::Duration, - sync_delay_ms: u64, -) -> (chrono::DateTime, std::time::Duration) { - use chrono::Timelike; - - let arrival_offset = chrono::Duration::from_std(one_way).unwrap_or_else(|_| { - tracing::warn!( - one_way_ms = one_way.as_millis(), - "one-way latency exceeds chrono Duration range, falling back to zero" - ); - chrono::Duration::zero() - }); - let sync_delay = chrono::Duration::milliseconds(sync_delay_ms as i64); - let wall_at_rollover_if_now = wall_now + arrival_offset + sync_delay; - let rollover_frac = wall_at_rollover_if_now.nanosecond() as f64 / 1_000_000_000.0; - - let target = if rollover_frac >= 0.5 { - wall_at_rollover_if_now + chrono::Duration::seconds(1) - } else { - wall_at_rollover_if_now - }; - let target_boundary_initial = target - .with_nanosecond(0) - .expect("nanosecond 0 is always valid"); - - let mut target_boundary = target_boundary_initial; - let mut ideal_send = target_boundary - arrival_offset - sync_delay; - if ideal_send < wall_now { - target_boundary += chrono::Duration::seconds(1); - ideal_send = target_boundary - arrival_offset - sync_delay; - } - let pre_set_wait = ideal_send - .signed_duration_since(wall_now) - .to_std() - .unwrap_or(std::time::Duration::ZERO); - - (target_boundary, pre_set_wait) -} - /// POST /api/v1/readers/{ip}/sync-clock /// /// Minimizes clock drift by: @@ -2132,10 +1639,7 @@ fn status_from_u16_or_internal(status: u16) -> StatusCode { async fn update_status_handler( State(state): State>, ) -> Response { - let update_status = { - let ss = state.subsystem.lock().await; - ss.update_status.clone() - }; + let update_status = state.store.update_status().await; let body = serde_json::to_string(&update_status) .unwrap_or_else(|_| r#"{"status":"failed","error":"serialization error"}"#.to_owned()); json_response(StatusCode::OK, body) @@ -2144,16 +1648,13 @@ async fn update_status_handler( async fn update_apply_handler( State(state): State>, ) -> Response { - let ss = state.subsystem.lock().await; - match &ss.staged_update_path { + match state.store.staged_update_path().await { Some(path) => { - let path = path.clone(); - drop(ss); if apply_via_restart_enabled() { schedule_process_restart(); json_response(StatusCode::OK, r#"{"status":"restarting"}"#.to_owned()) } else { - let sub = state.subsystem.clone(); + let store = state.store.clone(); let restart = state.restart_signal.clone(); tokio::spawn(async move { match tokio::task::spawn_blocking(move || { @@ -2168,38 +1669,36 @@ async fn update_apply_handler( } Ok(Err(e)) => { tracing::error!(error = %e, "update apply failed"); - sub.lock().await.update_status = rt_updater::UpdateStatus::Failed { - error: e.to_string(), - }; + store + .set_update_status(rt_updater::UpdateStatus::Failed { + error: e.to_string(), + }) + .await; } Err(e) => { tracing::error!(error = %e, "update apply task failed"); - sub.lock().await.update_status = rt_updater::UpdateStatus::Failed { - error: e.to_string(), - }; + store + .set_update_status(rt_updater::UpdateStatus::Failed { + error: e.to_string(), + }) + .await; } } }); json_response(StatusCode::OK, r#"{"status":"applying"}"#.to_owned()) } } - None => { - drop(ss); - json_response( - StatusCode::NOT_FOUND, - r#"{"error":"no update staged"}"#.to_owned(), - ) - } + None => json_response( + StatusCode::NOT_FOUND, + r#"{"error":"no update staged"}"#.to_owned(), + ), } } async fn update_check_handler( State(state): State>, ) -> Response { - let update_mode = { - let ss = state.subsystem.lock().await; - ss.update_mode - }; + let update_mode = state.store.update_mode().await; let checker = match rt_updater::UpdateChecker::new( "iwismer", @@ -2212,7 +1711,7 @@ async fn update_check_handler( let status = rt_updater::UpdateStatus::Failed { error: e.to_string(), }; - state.subsystem.lock().await.update_status = status.clone(); + state.store.set_update_status(status.clone()).await; let body = serde_json::to_string(&status).unwrap_or_else(|_| { r#"{"status":"failed","error":"serialization error"}"#.to_owned() }); @@ -2220,68 +1719,13 @@ async fn update_check_handler( } }; - let workflow_state = - ForwarderWorkflowAdapter::new(state.subsystem.clone(), state.ui_tx.clone()); + let workflow_state = state.store.workflow_state(); let status = run_check(&workflow_state, &checker, update_mode).await; let body = serde_json::to_string(&status) .unwrap_or_else(|_| r#"{"status":"failed","error":"serialization error"}"#.to_owned()); json_response(StatusCode::OK, body) } -struct ForwarderWorkflowAdapter { - subsystem: Arc>, - ui_tx: tokio::sync::broadcast::Sender, -} - -impl ForwarderWorkflowAdapter { - fn new( - subsystem: Arc>, - ui_tx: tokio::sync::broadcast::Sender, - ) -> Self { - Self { subsystem, ui_tx } - } -} - -impl WorkflowState for ForwarderWorkflowAdapter { - fn current_status<'a>( - &'a self, - ) -> std::pin::Pin + Send + 'a>> { - Box::pin(async move { self.subsystem.lock().await.update_status.clone() }) - } - - fn set_status<'a>( - &'a self, - status: UpdateStatus, - ) -> std::pin::Pin + Send + 'a>> { - Box::pin(async move { - self.subsystem.lock().await.update_status = status; - }) - } - - fn set_downloaded<'a>( - &'a self, - status: UpdateStatus, - path: std::path::PathBuf, - ) -> std::pin::Pin + Send + 'a>> { - Box::pin(async move { - let mut ss = self.subsystem.lock().await; - ss.update_status = status; - ss.staged_update_path = Some(path); - }) - } - - fn emit_status_changed<'a>( - &'a self, - status: UpdateStatus, - ) -> std::pin::Pin + Send + 'a>> { - Box::pin(async move { - let _ = self - .ui_tx - .send(crate::ui_events::ForwarderUiEvent::UpdateStatusChanged { status }); - }) - } -} - async fn update_download_handler( State(state): State>, ) -> Response { @@ -2296,7 +1740,7 @@ async fn update_download_handler( let status = rt_updater::UpdateStatus::Failed { error: e.to_string(), }; - state.subsystem.lock().await.update_status = status.clone(); + state.store.set_update_status(status.clone()).await; let body = serde_json::to_string(&status).unwrap_or_else(|_| { r#"{"status":"failed","error":"serialization error"}"#.to_owned() }); @@ -2304,8 +1748,7 @@ async fn update_download_handler( } }; - let workflow_state = - ForwarderWorkflowAdapter::new(state.subsystem.clone(), state.ui_tx.clone()); + let workflow_state = state.store.workflow_state(); match run_download(&workflow_state, &checker).await { Ok(status) => { let body = serde_json::to_string(&status).unwrap_or_default(); @@ -2478,11 +1921,13 @@ fn schedule_process_restart() {} #[cfg(test)] mod tests { use super::*; + #[cfg(unix)] + use crate::config_service::map_power_action_command_result; use ipico_core::control::{self, Command, TagMessageFormat}; use rt_updater::workflow::{Checker, run_check, run_download}; use std::future::Future; use std::pin::Pin; - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::atomic::{AtomicUsize, Ordering}; use tokio::time::{Duration, sleep}; struct FakeChecker { @@ -2966,6 +2411,7 @@ mod tests { .await; let state = AppState { + store: server.store(), subsystem: server.subsystem_arc(), journal: Arc::new(Mutex::new(NoJournal)), version: Arc::new("0.2.0".to_owned()), @@ -2977,10 +2423,6 @@ mod tests { control_clients: server.control_clients().clone(), download_trackers: server.download_trackers().clone(), reconnect_notifies: server.reconnect_notifies().clone(), - #[cfg(any(feature = "eink", feature = "lcd"))] - display_name: server.display_name.clone(), - #[cfg(any(feature = "eink", feature = "lcd"))] - cpu_temp: server.cpu_temp.clone(), }; update_cached_reader_info( &state, @@ -4413,8 +3855,7 @@ target = "192.168.1.100:10000" download_calls: Arc::clone(&download_calls), }; - let workflow_state = - ForwarderWorkflowAdapter::new(server.subsystem_arc(), server.ui_sender()); + let workflow_state = server.store().workflow_state(); let status = run_check(&workflow_state, &checker, rt_updater::UpdateMode::CheckOnly).await; assert_eq!( @@ -4446,8 +3887,7 @@ target = "192.168.1.100:10000" download_calls: Arc::clone(&download_calls), }; - let workflow_state = - ForwarderWorkflowAdapter::new(server.subsystem_arc(), server.ui_sender()); + let workflow_state = server.store().workflow_state(); let status = run_check(&workflow_state, &checker, rt_updater::UpdateMode::Disabled).await; assert_eq!( @@ -4479,8 +3919,7 @@ target = "192.168.1.100:10000" download_calls: Arc::clone(&download_calls), }; - let workflow_state = - ForwarderWorkflowAdapter::new(server.subsystem_arc(), server.ui_sender()); + let workflow_state = server.store().workflow_state(); let status = run_check( &workflow_state, &checker, @@ -4504,7 +3943,7 @@ target = "192.168.1.100:10000" #[cfg(unix)] #[test] fn power_action_execution_does_not_use_sudo_fallback() { - let source = include_str!("status_http.rs"); + let source = include_str!("config_service.rs"); assert!( !source.contains("Command::new(\"sudo\")"), "power actions must not invoke sudo fallback" @@ -4629,60 +4068,6 @@ target = "192.168.1.100:10000" assert!(result.is_ok()); } - #[cfg(unix)] - #[tokio::test] - async fn power_action_join_error_logs_to_ui_when_logger_present() { - let (tx, mut rx) = tokio::sync::broadcast::channel(16); - let logger = rt_ui_log::UiLogger::new(tx, |entry| { - crate::ui_events::ForwarderUiEvent::LogEntry { entry } - }); - - let join_err = tokio::task::spawn_blocking(|| -> () { - panic!("boom"); - }) - .await - .expect_err("task must panic"); - - let (_status, _body) = map_power_action_join_error("reboot", join_err, Some(&logger)); - - let evt = rx.try_recv().expect("expected UI log event"); - match evt { - crate::ui_events::ForwarderUiEvent::LogEntry { entry } => { - assert!(entry.contains("systemctl reboot task failed")); - } - other => panic!("unexpected event: {other:?}"), - } - } - - #[tokio::test] - async fn control_action_from_config_forwards_logger_to_apply_fn() { - let (tx, _) = tokio::sync::broadcast::channel(16); - let logger = rt_ui_log::UiLogger::new(tx, |entry| { - crate::ui_events::ForwarderUiEvent::LogEntry { entry } - }); - let config = ConfigState::new(std::path::PathBuf::from("/tmp/unused.toml")); - - let saw_logger = Arc::new(AtomicBool::new(false)); - let spy = Arc::clone(&saw_logger); - - let _ = apply_control_action_from_config_with( - "restart_device", - &config, - Some(&logger), - move |_action, _config_state, _restart_signal, logger| { - let spy = Arc::clone(&spy); - let has_logger = logger.is_some(); - Box::pin(async move { - spy.store(has_logger, Ordering::SeqCst); - Err((500u16, "{\"ok\":false}".to_owned())) - }) - }, - ) - .await; - - assert!(saw_logger.load(Ordering::SeqCst)); - } - #[test] fn apply_via_restart_env_parsing() { assert!(apply_via_restart_from_env(Some("1".to_owned()))); @@ -4719,8 +4104,7 @@ target = "192.168.1.100:10000" download_calls: Arc::clone(&download_calls), }; - let workflow_state = - ForwarderWorkflowAdapter::new(server.subsystem_arc(), server.ui_sender()); + let workflow_state = server.store().workflow_state(); let status = run_download(&workflow_state, &checker).await; assert_eq!( @@ -4761,8 +4145,7 @@ target = "192.168.1.100:10000" download_calls: Arc::new(AtomicUsize::new(0)), }; - let workflow_state = - ForwarderWorkflowAdapter::new(server.subsystem_arc(), server.ui_sender()); + let workflow_state = server.store().workflow_state(); let status = run_download(&workflow_state, &checker).await; assert_eq!( status, @@ -4802,8 +4185,7 @@ target = "192.168.1.100:10000" download_calls: Arc::new(AtomicUsize::new(0)), }; - let workflow_state = - ForwarderWorkflowAdapter::new(server.subsystem_arc(), server.ui_sender()); + let workflow_state = server.store().workflow_state(); let status = run_download(&workflow_state, &checker).await; assert!(status.is_err()); } @@ -4832,8 +4214,7 @@ target = "192.168.1.100:10000" download_calls: Arc::new(AtomicUsize::new(0)), }; - let workflow_state = - ForwarderWorkflowAdapter::new(server.subsystem_arc(), server.ui_sender()); + let workflow_state = server.store().workflow_state(); let status = run_download(&workflow_state, &checker).await; assert_eq!( status, @@ -4988,7 +4369,7 @@ target = "192.168.1.100:10000" #[cfg(any(feature = "eink", feature = "lcd"))] #[tokio::test] - async fn set_ready_with_display_sender_does_not_deadlock() { + async fn set_ready_with_display_sender_publishes_once() { let mut server = StatusServer::start( StatusConfig { bind: "127.0.0.1:0".to_owned(), @@ -5003,13 +4384,25 @@ target = "192.168.1.100:10000" tokio::sync::watch::channel(rt_screen::state::DisplayState::initial()); server.set_display_sender(display_tx); + let (seen_tx, mut seen_rx) = tokio::sync::mpsc::channel(4); + let watcher = tokio::spawn(async move { + while display_rx.changed().await.is_ok() { + let _ = seen_tx.send(()).await; + } + }); + tokio::task::yield_now().await; + tokio::time::timeout(Duration::from_millis(100), server.set_ready()) .await .expect("set_ready timed out"); - tokio::time::timeout(Duration::from_millis(100), display_rx.changed()) + tokio::time::timeout(Duration::from_millis(100), seen_rx.recv()) .await .expect("display state publish timed out") .expect("display state sender dropped"); + tokio::time::timeout(Duration::from_millis(100), seen_rx.recv()) + .await + .expect_err("display state should publish exactly once"); + watcher.abort(); } #[cfg(any(feature = "eink", feature = "lcd"))] @@ -5761,75 +5154,6 @@ target = "192.168.1.100:10000" assert_eq!(resp.status(), 503); } - #[test] - fn sync_timing_rollover_frac_above_half_rounds_up() { - use chrono::TimeZone; - use chrono::Timelike; - // wall_now=.200, one_way=50ms, sync=500ms → rollover_if_now=.750, frac=0.75 → rounds UP - let wall_now = chrono::Local - .with_ymd_and_hms(2026, 3, 8, 12, 0, 0) - .unwrap() - .with_nanosecond(200_000_000) - .unwrap(); - let one_way = std::time::Duration::from_millis(50); - let (target, wait) = super::compute_sync_timing(wall_now, one_way, 500); - assert_eq!(target.second(), 1); - assert_eq!(target.nanosecond(), 0); - assert!(wait < std::time::Duration::from_secs(1)); - } - - #[test] - fn sync_timing_rollover_frac_below_half_stays_same_second() { - use chrono::TimeZone; - use chrono::Timelike; - // wall_now=.800, one_way=50ms, sync=500ms → rollover_if_now=1.350, frac=0.35 → truncates to 1.000 - // ideal_send=1.000-0.050-0.500=0.450 < 0.800 → BUMP → target=2.000 - let wall_now = chrono::Local - .with_ymd_and_hms(2026, 3, 8, 12, 0, 0) - .unwrap() - .with_nanosecond(800_000_000) - .unwrap(); - let one_way = std::time::Duration::from_millis(50); - let (target, _wait) = super::compute_sync_timing(wall_now, one_way, 500); - assert_eq!(target.second(), 2); - assert_eq!(target.nanosecond(), 0); - } - - #[test] - fn sync_timing_ideal_send_past_bumps_target() { - use chrono::TimeZone; - use chrono::Timelike; - // wall_now=.500, one_way=1ms, sync=500ms → rollover_if_now=1.001, frac=0.001 → target=1.000 - // ideal_send=1.000-0.001-0.500=0.499 < 0.500 → BUMP → target=2.000 - let wall_now = chrono::Local - .with_ymd_and_hms(2026, 3, 8, 12, 0, 0) - .unwrap() - .with_nanosecond(500_000_000) - .unwrap(); - let one_way = std::time::Duration::from_millis(1); - let (target, wait) = super::compute_sync_timing(wall_now, one_way, 500); - assert_eq!(target.second(), 2); - assert_eq!(target.nanosecond(), 0); - assert!(wait > std::time::Duration::from_millis(900)); - assert!(wait < std::time::Duration::from_millis(1100)); - } - - #[test] - fn sync_timing_zero_latency() { - use chrono::TimeZone; - use chrono::Timelike; - // wall_now=.300, one_way=0, sync=500ms → rollover_if_now=.800, frac=0.8 → rounds UP - let wall_now = chrono::Local - .with_ymd_and_hms(2026, 3, 8, 12, 0, 0) - .unwrap() - .with_nanosecond(300_000_000) - .unwrap(); - let one_way = std::time::Duration::ZERO; - let (target, _wait) = super::compute_sync_timing(wall_now, one_way, 500); - assert_eq!(target.second(), 1); - assert_eq!(target.nanosecond(), 0); - } - #[tokio::test] async fn config_ups_section_accepts_valid_values() { use std::io::Write; diff --git a/services/forwarder/src/status_store.rs b/services/forwarder/src/status_store.rs index 8c93b7bf..f8f602d2 100644 --- a/services/forwarder/src/status_store.rs +++ b/services/forwarder/src/status_store.rs @@ -5,6 +5,7 @@ //! has no HTTP/axum dependencies. use rt_updater::UpdateStatus; +use rt_updater::workflow::WorkflowState; use serde::Serialize; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -275,8 +276,14 @@ pub struct StatusStore { subsystem: Arc>, ui_tx: tokio::sync::broadcast::Sender, status_event_tx: broadcast::Sender, - display_change_tx: broadcast::Sender<()>, logger: Arc>, + #[cfg(any(feature = "eink", feature = "lcd"))] + display_tx: + Arc>>>, + #[cfg(any(feature = "eink", feature = "lcd"))] + display_name: Arc>>, + #[cfg(any(feature = "eink", feature = "lcd"))] + cpu_temp: Arc>>, control_clients: Arc>>>, download_trackers: Arc< @@ -375,12 +382,109 @@ fn spawn_read_count_broadcaster( } }); } + +#[cfg(any(feature = "eink", feature = "lcd"))] +fn subsystem_to_display_state( + ss: &SubsystemStatus, + forwarder_name: Option, + cpu_temp: Option, +) -> rt_screen::state::DisplayState { + let readers = ss + .readers() + .iter() + .map(|(addr, r)| { + let ip = addr + .rsplit_once(':') + .map_or(addr.as_str(), |(ip, _)| ip) + .to_owned(); + rt_screen::state::ReaderDisplayState { + ip, + state: match r.state { + ReaderConnectionState::Connected => { + rt_screen::state::ReaderConnectionState::Connected + } + ReaderConnectionState::Connecting => { + rt_screen::state::ReaderConnectionState::Connecting + } + ReaderConnectionState::Disconnected => { + rt_screen::state::ReaderConnectionState::Disconnected + } + }, + drift_ms: r + .reader_info + .as_ref() + .and_then(|info| info.clock.as_ref()) + .map(|c| c.drift_ms), + session_reads: r.reads_since_restart, + } + }) + .collect(); + + let total_reads: u64 = ss.readers().values().map(|r| r.reads_since_restart).sum(); + + rt_screen::state::DisplayState { + forwarder_name, + local_ip: ss.local_ip.clone(), + p2p_connected: ss.p2p_connected(), + readers, + total_reads, + cpu_temp_celsius: cpu_temp, + battery: ss.ups_status().and_then(|u| { + u.status.as_ref().map(|s| rt_screen::state::BatteryState { + percent: s.battery_percent, + charging: s.charging, + }) + }), + } +} + +/// Workflow state adapter over the forwarder's shared status store. +#[derive(Clone)] +pub(crate) struct ForwarderWorkflowAdapter { + store: StatusStore, +} + +impl WorkflowState for ForwarderWorkflowAdapter { + fn current_status<'a>( + &'a self, + ) -> std::pin::Pin + Send + 'a>> { + Box::pin(async move { self.store.update_status().await }) + } + + fn set_status<'a>( + &'a self, + status: UpdateStatus, + ) -> std::pin::Pin + Send + 'a>> { + Box::pin(async move { + self.store.set_update_status_without_emit(status).await; + }) + } + + fn set_downloaded<'a>( + &'a self, + status: UpdateStatus, + path: std::path::PathBuf, + ) -> std::pin::Pin + Send + 'a>> { + Box::pin(async move { + self.store.set_downloaded_update(status, path).await; + }) + } + + fn emit_status_changed<'a>( + &'a self, + status: UpdateStatus, + ) -> std::pin::Pin + Send + 'a>> { + Box::pin(async move { + self.store.emit_update_status_changed(status); + }) + } +} + impl StatusStore { /// Create a status store with fresh UI and P2P status broadcast channels. pub fn new(subsystem: SubsystemStatus) -> Self { let (ui_tx, _) = tokio::sync::broadcast::channel(256); let (status_event_tx, _) = broadcast::channel(256); - let (display_change_tx, _) = broadcast::channel(256); let logger = Arc::new(rt_ui_log::UiLogger::with_buffer( ui_tx.clone(), |entry| crate::ui_events::ForwarderUiEvent::LogEntry { entry }, @@ -391,8 +495,13 @@ impl StatusStore { subsystem: subsystem.clone(), ui_tx, status_event_tx: status_event_tx.clone(), - display_change_tx, logger, + #[cfg(any(feature = "eink", feature = "lcd"))] + display_tx: Arc::new(std::sync::Mutex::new(None)), + #[cfg(any(feature = "eink", feature = "lcd"))] + display_name: Arc::new(Mutex::new(None)), + #[cfg(any(feature = "eink", feature = "lcd"))] + cpu_temp: Arc::new(Mutex::new(None)), control_clients: Arc::new(std::sync::RwLock::new(HashMap::new())), download_trackers: Arc::new(std::sync::RwLock::new(HashMap::new())), reconnect_notifies: Arc::new(std::sync::RwLock::new(HashMap::new())), @@ -406,14 +515,6 @@ impl StatusStore { self.status_event_tx.clone() } - /// Subscribe to state changes that should refresh the optional local display. - pub(crate) fn subscribe_display_changes(&self) -> broadcast::Receiver<()> { - self.display_change_tx.subscribe() - } - - fn notify_display_changed(&self) { - let _ = self.display_change_tx.send(()); - } /// Return a clone of the internal subsystem status Arc. pub fn subsystem_arc(&self) -> Arc> { self.subsystem.clone() @@ -464,7 +565,7 @@ impl StatusStore { restart_needed: ss.restart_needed(), }); } - self.notify_display_changed(); + self.publish_display_state().await; } /// Mark that a restart is needed to apply saved config changes. @@ -495,7 +596,7 @@ impl StatusStore { restart_needed: ss.restart_needed(), }); } - self.notify_display_changed(); + self.publish_display_state().await; } /// Set the forwarder ID (call once at startup). @@ -506,7 +607,7 @@ impl StatusStore { /// Set the detected local IP (call once at startup). pub async fn set_local_ip(&self, ip: Option) { self.subsystem.lock().await.local_ip = ip; - self.notify_display_changed(); + self.publish_display_state().await; } /// Set the update mode (controls check-only vs check-and-download behavior). @@ -514,19 +615,102 @@ impl StatusStore { self.subsystem.lock().await.update_mode = mode; } + /// Return the configured update mode. + pub async fn update_mode(&self) -> rt_updater::UpdateMode { + self.subsystem.lock().await.update_mode + } + + /// Return the current rt-updater status. + pub async fn update_status(&self) -> UpdateStatus { + self.subsystem.lock().await.update_status.clone() + } + + /// Return the staged update artifact path, if one exists. + pub async fn staged_update_path(&self) -> Option { + self.subsystem.lock().await.staged_update_path.clone() + } + /// Update the current rt-updater status (shown on `/update/status`). pub async fn set_update_status(&self, status: UpdateStatus) { - self.subsystem.lock().await.update_status = status.clone(); + self.set_update_status_without_emit(status.clone()).await; + self.emit_update_status_changed(status); + } + + async fn set_update_status_without_emit(&self, status: UpdateStatus) { + self.subsystem.lock().await.update_status = status; + } + + async fn set_downloaded_update(&self, status: UpdateStatus, path: std::path::PathBuf) { + let mut ss = self.subsystem.lock().await; + ss.update_status = status; + ss.staged_update_path = Some(path); + } + + fn emit_update_status_changed(&self, status: UpdateStatus) { let _ = self .ui_tx .send(crate::ui_events::ForwarderUiEvent::UpdateStatusChanged { status }); } + pub(crate) fn workflow_state(&self) -> ForwarderWorkflowAdapter { + ForwarderWorkflowAdapter { + store: self.clone(), + } + } + /// Record the filesystem path of a downloaded update artifact ready to apply. pub async fn set_staged_update_path(&self, path: std::path::PathBuf) { self.subsystem.lock().await.staged_update_path = Some(path); } + #[cfg(any(feature = "eink", feature = "lcd"))] + pub fn set_display_sender( + &self, + tx: tokio::sync::watch::Sender, + ) { + *self.display_tx.lock().unwrap_or_else(|e| e.into_inner()) = Some(tx); + } + + #[cfg(any(feature = "eink", feature = "lcd"))] + pub async fn set_display_name(&self, name: Option) { + *self.display_name.lock().await = name; + self.publish_display_state().await; + } + + #[cfg(any(feature = "eink", feature = "lcd"))] + pub async fn set_cpu_temp(&self, temp: Option) { + self.set_cpu_temp_cached(temp).await; + self.publish_display_state().await; + } + + #[cfg(any(feature = "eink", feature = "lcd"))] + pub async fn set_cpu_temp_cached(&self, temp: Option) { + *self.cpu_temp.lock().await = temp; + } + + #[cfg(any(feature = "eink", feature = "lcd"))] + pub async fn display_state(&self) -> rt_screen::state::DisplayState { + let ss = self.subsystem.lock().await; + let forwarder_name = self.display_name.lock().await.clone(); + let cpu_temp = *self.cpu_temp.lock().await; + subsystem_to_display_state(&ss, forwarder_name, cpu_temp) + } + + #[cfg(any(feature = "eink", feature = "lcd"))] + async fn publish_display_state(&self) { + let tx = self + .display_tx + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(); + if let Some(tx) = tx { + tx.send_replace(self.display_state().await); + } + } + + #[cfg(not(any(feature = "eink", feature = "lcd")))] + async fn publish_display_state(&self) {} + /// Update the UPS status snapshot in the subsystem state. /// /// The local HTTP/UI snapshot is always updated, but the P2P control-event @@ -546,7 +730,7 @@ impl StatusStore { .send(ForwarderStatusEvent::UpsStatus(state)); } } - self.notify_display_changed(); + self.publish_display_state().await; } pub fn control_clients( @@ -641,7 +825,7 @@ impl StatusStore { info, }); } - self.notify_display_changed(); + self.publish_display_state().await; } /// Update reader info only if the reader has not transitioned to Disconnected. @@ -684,7 +868,7 @@ impl StatusStore { info, }); } - self.notify_display_changed(); + self.publish_display_state().await; } pub fn register_control_client( @@ -724,7 +908,7 @@ impl StatusStore { }); } } - self.notify_display_changed(); + self.publish_display_state().await; } /// Seed a reader's total historical count from durable journal state. @@ -735,7 +919,7 @@ impl StatusStore { r.reads_total = total; } } - self.notify_display_changed(); + self.publish_display_state().await; } /// Set the current epoch name for a reader and broadcast a ReaderUpdated SSE event. @@ -757,7 +941,7 @@ impl StatusStore { }); } } - self.notify_display_changed(); + self.publish_display_state().await; } /// Update a reader's connection state. @@ -795,7 +979,7 @@ impl StatusStore { } } } - self.notify_display_changed(); + self.publish_display_state().await; } /// Record a successful chip read for a reader. @@ -823,7 +1007,7 @@ impl StatusStore { ss.read_counts_dirty.insert(reader_ip.to_owned()); } } - self.notify_display_changed(); + self.publish_display_state().await; } } From 1e1666b347d2908304fb57a244839b2a4804066b Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 16:10:26 -0400 Subject: [PATCH 24/38] test(forwarder): keep control action tests with config service --- services/forwarder/src/config_service.rs | 130 ++++++++++++++++++++++- services/forwarder/src/status_http.rs | 130 ----------------------- 2 files changed, 129 insertions(+), 131 deletions(-) diff --git a/services/forwarder/src/config_service.rs b/services/forwarder/src/config_service.rs index 9cb832c3..8676cb0c 100644 --- a/services/forwarder/src/config_service.rs +++ b/services/forwarder/src/config_service.rs @@ -68,7 +68,7 @@ async fn read_allow_power_actions(config_state: &ConfigState) -> Result, logger: Option<&rt_ui_log::UiLogger>, @@ -1026,6 +1026,134 @@ mod tests { use super::*; use std::sync::atomic::{AtomicBool, Ordering}; + #[cfg(unix)] + #[test] + fn power_action_execution_does_not_use_sudo_fallback() { + let source = include_str!("config_service.rs"); + assert!( + !source.contains("Command::new(\"sudo\")"), + "power actions must not invoke sudo fallback" + ); + } + + #[cfg(unix)] + #[test] + fn power_action_command_result_returns_500_on_spawn_error() { + let result = map_power_action_command_result( + "reboot", + Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "systemctl not found", + )), + None, + ); + + let (status, body) = result.expect_err("spawn errors must return an HTTP error"); + assert_eq!(status, 500); + assert!(body.contains("control action command failed")); + } + + #[cfg(unix)] + #[test] + fn power_action_command_result_returns_500_on_non_zero_exit() { + use std::os::unix::process::ExitStatusExt; + + let result = map_power_action_command_result( + "poweroff", + Ok(std::process::Output { + status: std::process::ExitStatus::from_raw(1 << 8), + stdout: vec![], + stderr: vec![], + }), + None, + ); + + let (http_status, body) = result.expect_err("non-zero exit must return an HTTP error"); + assert_eq!(http_status, 500); + assert!(body.contains("control action command exited with failure")); + } + + #[cfg(unix)] + #[test] + fn power_action_command_result_returns_403_on_auth_failure() { + use std::os::unix::process::ExitStatusExt; + + let result = map_power_action_command_result( + "poweroff", + Ok(std::process::Output { + status: std::process::ExitStatus::from_raw(1 << 8), + stdout: vec![], + stderr: b"Call to PowerOff failed: Interactive authentication required.\n".to_vec(), + }), + None, + ); + + let (http_status, body) = result.expect_err("auth failures must return an HTTP error"); + assert_eq!(http_status, 403); + assert!( + body.to_ascii_lowercase() + .contains("interactive authentication required") + ); + } + + #[cfg(unix)] + #[test] + fn power_action_command_result_returns_500_on_non_auth_polkit_error() { + use std::os::unix::process::ExitStatusExt; + + let result = map_power_action_command_result( + "poweroff", + Ok(std::process::Output { + status: std::process::ExitStatus::from_raw(1 << 8), + stdout: vec![], + stderr: b"polkit daemon unavailable".to_vec(), + }), + None, + ); + + let (http_status, body) = + result.expect_err("non-auth polkit failures must return an HTTP error"); + assert_eq!(http_status, 500); + assert!(body.contains("polkit daemon unavailable")); + } + + #[cfg(unix)] + #[test] + fn power_action_command_result_includes_stderr_in_error_body() { + use std::os::unix::process::ExitStatusExt; + + let result = map_power_action_command_result( + "reboot", + Ok(std::process::Output { + status: std::process::ExitStatus::from_raw(1 << 8), + stdout: vec![], + stderr: b"sudo: a password is required".to_vec(), + }), + None, + ); + + let (http_status, body) = result.expect_err("non-zero exit must return an HTTP error"); + assert_eq!(http_status, 403); + assert!(body.contains("sudo: a password is required")); + } + + #[cfg(unix)] + #[test] + fn power_action_command_result_returns_ok_on_success_exit() { + use std::os::unix::process::ExitStatusExt; + + let result = map_power_action_command_result( + "reboot", + Ok(std::process::Output { + status: std::process::ExitStatus::from_raw(0), + stdout: vec![], + stderr: vec![], + }), + None, + ); + assert!(result.is_ok()); + } + #[cfg(unix)] #[tokio::test] async fn power_action_join_error_logs_to_ui_when_logger_present() { diff --git a/services/forwarder/src/status_http.rs b/services/forwarder/src/status_http.rs index 93055942..767db92c 100644 --- a/services/forwarder/src/status_http.rs +++ b/services/forwarder/src/status_http.rs @@ -1921,8 +1921,6 @@ fn schedule_process_restart() {} #[cfg(test)] mod tests { use super::*; - #[cfg(unix)] - use crate::config_service::map_power_action_command_result; use ipico_core::control::{self, Command, TagMessageFormat}; use rt_updater::workflow::{Checker, run_check, run_download}; use std::future::Future; @@ -3940,134 +3938,6 @@ target = "192.168.1.100:10000" ); } - #[cfg(unix)] - #[test] - fn power_action_execution_does_not_use_sudo_fallback() { - let source = include_str!("config_service.rs"); - assert!( - !source.contains("Command::new(\"sudo\")"), - "power actions must not invoke sudo fallback" - ); - } - - #[cfg(unix)] - #[test] - fn power_action_command_result_returns_500_on_spawn_error() { - let result = map_power_action_command_result( - "reboot", - Err(std::io::Error::new( - std::io::ErrorKind::NotFound, - "systemctl not found", - )), - None, - ); - - let (status, body) = result.expect_err("spawn errors must return an HTTP error"); - assert_eq!(status, 500); - assert!(body.contains("control action command failed")); - } - - #[cfg(unix)] - #[test] - fn power_action_command_result_returns_500_on_non_zero_exit() { - use std::os::unix::process::ExitStatusExt; - - let result = map_power_action_command_result( - "poweroff", - Ok(std::process::Output { - status: std::process::ExitStatus::from_raw(1 << 8), - stdout: vec![], - stderr: vec![], - }), - None, - ); - - let (http_status, body) = result.expect_err("non-zero exit must return an HTTP error"); - assert_eq!(http_status, 500); - assert!(body.contains("control action command exited with failure")); - } - - #[cfg(unix)] - #[test] - fn power_action_command_result_returns_403_on_auth_failure() { - use std::os::unix::process::ExitStatusExt; - - let result = map_power_action_command_result( - "poweroff", - Ok(std::process::Output { - status: std::process::ExitStatus::from_raw(1 << 8), - stdout: vec![], - stderr: b"Call to PowerOff failed: Interactive authentication required.\n".to_vec(), - }), - None, - ); - - let (http_status, body) = result.expect_err("auth failures must return an HTTP error"); - assert_eq!(http_status, 403); - assert!( - body.to_ascii_lowercase() - .contains("interactive authentication required") - ); - } - - #[cfg(unix)] - #[test] - fn power_action_command_result_returns_500_on_non_auth_polkit_error() { - use std::os::unix::process::ExitStatusExt; - - let result = map_power_action_command_result( - "poweroff", - Ok(std::process::Output { - status: std::process::ExitStatus::from_raw(1 << 8), - stdout: vec![], - stderr: b"polkit daemon unavailable".to_vec(), - }), - None, - ); - - let (http_status, body) = - result.expect_err("non-auth polkit failures must return an HTTP error"); - assert_eq!(http_status, 500); - assert!(body.contains("polkit daemon unavailable")); - } - - #[cfg(unix)] - #[test] - fn power_action_command_result_includes_stderr_in_error_body() { - use std::os::unix::process::ExitStatusExt; - - let result = map_power_action_command_result( - "reboot", - Ok(std::process::Output { - status: std::process::ExitStatus::from_raw(1 << 8), - stdout: vec![], - stderr: b"sudo: a password is required".to_vec(), - }), - None, - ); - - let (http_status, body) = result.expect_err("non-zero exit must return an HTTP error"); - assert_eq!(http_status, 403); - assert!(body.contains("sudo: a password is required")); - } - - #[cfg(unix)] - #[test] - fn power_action_command_result_returns_ok_on_success_exit() { - use std::os::unix::process::ExitStatusExt; - - let result = map_power_action_command_result( - "reboot", - Ok(std::process::Output { - status: std::process::ExitStatus::from_raw(0), - stdout: vec![], - stderr: vec![], - }), - None, - ); - assert!(result.is_ok()); - } - #[test] fn apply_via_restart_env_parsing() { assert!(apply_via_restart_from_env(Some("1".to_owned()))); From 0a915d370f94c6a58390b396b5e310e7ff193fd4 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 16:33:41 -0400 Subject: [PATCH 25/38] refactor(rt-p2p-protocol): own length-prefixed frame decode in the protocol crate --- crates/rt-p2p-protocol/src/codec.rs | 28 ++++++++++++--- crates/rt-p2p-protocol/src/error.rs | 12 ++++++- crates/rt-p2p-protocol/src/lib.rs | 5 ++- .../rt-p2p-protocol/tests/codec_negotiate.rs | 35 ++++++++++++++++++- crates/rt-test-utils/src/p2p/mod.rs | 10 ++---- services/forwarder/src/p2p/control.rs | 12 +++---- services/receiver/src/p2p_session.rs | 21 ++++++----- 7 files changed, 93 insertions(+), 30 deletions(-) diff --git a/crates/rt-p2p-protocol/src/codec.rs b/crates/rt-p2p-protocol/src/codec.rs index 2132b93d..93ed67fb 100644 --- a/crates/rt-p2p-protocol/src/codec.rs +++ b/crates/rt-p2p-protocol/src/codec.rs @@ -23,16 +23,23 @@ pub fn encode_frame(message: &impl Message) -> Bytes { buf.freeze() } +/// Decodes and validates a little-endian frame payload length prefix. +pub fn decode_frame_len(len_buf: [u8; 4]) -> Result { + let len = u32::from_le_bytes(len_buf) as usize; + if len > MAX_FRAME_BYTES { + return Err(ProtocolError::frame_too_large(len)); + } + + Ok(len) +} + /// Decodes one complete length-prefixed frame, if enough bytes are buffered. pub fn decode_frame(buf: &mut BytesMut) -> Result, ProtocolError> { if buf.len() < 4 { return Ok(None); } - let len = u32::from_le_bytes(buf[..4].try_into().expect("slice has 4 bytes")) as usize; - if len > MAX_FRAME_BYTES { - return Err(ProtocolError::frame_too_large()); - } + let len = decode_frame_len(buf[..4].try_into().expect("slice has 4 bytes"))?; if buf.len() < 4 + len { return Ok(None); @@ -42,6 +49,19 @@ pub fn decode_frame(buf: &mut BytesMut) -> Result, ProtocolError> Ok(Some(buf.split_to(len).freeze())) } +/// Decodes a protobuf message from a previously read length prefix and payload. +pub fn decode_frame_payload(len_buf: [u8; 4], payload: &[u8]) -> Result +where + M: Message + Default, +{ + let len = decode_frame_len(len_buf)?; + if payload.len() != len { + return Err(ProtocolError::protocol_violation()); + } + + M::decode(payload).map_err(ProtocolError::decode_error) +} + /// Decodes one complete length-prefixed protobuf message, if enough bytes are buffered. pub fn decode_message_frame(buf: &mut BytesMut) -> Result, ProtocolError> where diff --git a/crates/rt-p2p-protocol/src/error.rs b/crates/rt-p2p-protocol/src/error.rs index 75788360..1865b1fe 100644 --- a/crates/rt-p2p-protocol/src/error.rs +++ b/crates/rt-p2p-protocol/src/error.rs @@ -103,6 +103,8 @@ pub enum ProtocolError { retryable: bool, /// Stream the error pertains to, if any. stream_id: Option>, + /// Advertised payload length. + length: usize, }, /// A protobuf payload could not be decoded. #[error("decode error: {source}")] @@ -188,10 +190,18 @@ impl ProtocolError { } } - pub(crate) fn frame_too_large() -> Self { + pub(crate) fn frame_too_large(length: usize) -> Self { Self::FrameTooLarge { retryable: false, stream_id: None, + length, + } + } + + pub(crate) fn protocol_violation() -> Self { + Self::ProtocolViolation { + retryable: false, + stream_id: None, } } diff --git a/crates/rt-p2p-protocol/src/lib.rs b/crates/rt-p2p-protocol/src/lib.rs index 9b043539..8dd6fae4 100644 --- a/crates/rt-p2p-protocol/src/lib.rs +++ b/crates/rt-p2p-protocol/src/lib.rs @@ -45,7 +45,10 @@ pub mod proto { } pub use capabilities::{CAP_CONTROL_EVENTS, CAP_READER_CONTROL, CAP_REMOTE_CONFIG, has_capability}; -pub use codec::{Frame, MAX_FRAME_BYTES, decode_frame, decode_message_frame, encode_frame}; +pub use codec::{ + Frame, MAX_FRAME_BYTES, decode_frame, decode_frame_len, decode_frame_payload, + decode_message_frame, encode_frame, +}; pub use error::{ProtocolError, ProtocolErrorCode}; pub use generated::{ Ack, CaughtUp, ConfigGetRequest, ConfigGetResponse, ConfigSetRequest, ConfigSetResponse, diff --git a/crates/rt-p2p-protocol/tests/codec_negotiate.rs b/crates/rt-p2p-protocol/tests/codec_negotiate.rs index 3dbf521f..86b61260 100644 --- a/crates/rt-p2p-protocol/tests/codec_negotiate.rs +++ b/crates/rt-p2p-protocol/tests/codec_negotiate.rs @@ -2,7 +2,7 @@ use bytes::{BufMut, BytesMut}; use prost::Message; use rt_p2p_protocol::{ CAP_CONTROL_EVENTS, CAP_REMOTE_CONFIG, Hello, Ping, ProtocolError, decode_frame, - decode_message_frame, encode_frame, has_capability, negotiate, + decode_frame_payload, decode_message_frame, encode_frame, has_capability, negotiate, }; #[test] @@ -26,6 +26,39 @@ fn roundtrip_frame() { assert_eq!(decoded, original); } +#[test] +fn decode_frame_payload_roundtrips_encoded_message() { + let original = Ping { nonce: 42 }; + let encoded = encode_frame(&original); + let len_buf = encoded[..4].try_into().expect("length prefix"); + + let decoded = decode_frame_payload::(len_buf, &encoded[4..]).expect("decode payload"); + + assert_eq!(decoded, original); +} + +#[test] +fn decode_frame_payload_rejects_oversize_prefix() { + let len_buf = u32::try_from(8 * 1024 * 1024 + 1) + .expect("fits u32") + .to_le_bytes(); + + let err = decode_frame_payload::(len_buf, &[]).expect_err("oversize frame"); + + assert!(matches!(err, ProtocolError::FrameTooLarge { .. })); +} + +#[test] +fn decode_frame_payload_rejects_payload_shorter_than_prefix() { + let encoded = encode_frame(&Ping { nonce: 42 }); + let len_buf = encoded[..4].try_into().expect("length prefix"); + + let err = decode_frame_payload::(len_buf, &encoded[4..encoded.len() - 1]) + .expect_err("truncated payload"); + + assert!(matches!(err, ProtocolError::ProtocolViolation { .. })); +} + #[test] fn partial_read_returns_none() { let encoded = encode_frame(&Ping { nonce: 7 }); diff --git a/crates/rt-test-utils/src/p2p/mod.rs b/crates/rt-test-utils/src/p2p/mod.rs index 0f9b9ac7..e717fe85 100644 --- a/crates/rt-test-utils/src/p2p/mod.rs +++ b/crates/rt-test-utils/src/p2p/mod.rs @@ -26,7 +26,7 @@ use std::time::Duration; use prost::Message; use rt_iroh::{RecvStream, SendStream}; -use rt_p2p_protocol::{MAX_FRAME_BYTES, encode_frame}; +use rt_p2p_protocol::{decode_frame_len, decode_frame_payload, encode_frame}; pub use mock_forwarder::{ForwarderScript, MockForwarderPeer}; pub use mock_receiver::{DataSubscription, MockReceiverPeer, ReceiverSession}; @@ -47,14 +47,10 @@ where { let mut len_buf = [0u8; 4]; recv.read_exact(&mut len_buf).await?; - let len = u32::from_le_bytes(len_buf) as usize; - if len > MAX_FRAME_BYTES { - return Err(format!("frame length {len} exceeds MAX_FRAME_BYTES {MAX_FRAME_BYTES}").into()); - } - + let len = decode_frame_len(len_buf)?; let mut payload = vec![0u8; len]; recv.read_exact(&mut payload).await?; - Ok(M::decode(payload.as_slice())?) + Ok(decode_frame_payload(len_buf, payload.as_slice())?) } /// Test-only connectivity fault shim. diff --git a/services/forwarder/src/p2p/control.rs b/services/forwarder/src/p2p/control.rs index 2d8ce239..435d4577 100644 --- a/services/forwarder/src/p2p/control.rs +++ b/services/forwarder/src/p2p/control.rs @@ -29,8 +29,8 @@ use rt_p2p_protocol::{ ConfigSetRequest, ConfigSetResponse, ControlC2F, ControlF2C, DownloadProgress, Hello, MAX_FRAME_BYTES, Ping, Pong, ProtocolError, ProtocolErrorCode, ReaderControlRequest, ReaderControlResponse, ReaderInfo, ReaderStatus, RestartRequest, RestartResponse, - StreamCatalog, SyncClock, UpsStatus, WireProtocolError, control_c2f, control_f2c, encode_frame, - has_capability, negotiate, + StreamCatalog, SyncClock, UpsStatus, WireProtocolError, control_c2f, control_f2c, + decode_frame_len, decode_frame_payload, encode_frame, has_capability, negotiate, }; use tokio::sync::mpsc; use tokio::time::MissedTickBehavior; @@ -949,14 +949,10 @@ where { let mut len_buf = [0u8; 4]; recv.read_exact(&mut len_buf).await?; - let len = u32::from_le_bytes(len_buf) as usize; - if len > MAX_FRAME_BYTES { - return Err(format!("frame length {len} exceeds MAX_FRAME_BYTES {MAX_FRAME_BYTES}").into()); - } - + let len = decode_frame_len(len_buf)?; let mut payload = vec![0u8; len]; recv.read_exact(&mut payload).await?; - Ok(M::decode(payload.as_slice())?) + Ok(decode_frame_payload(len_buf, payload.as_slice())?) } #[cfg(test)] diff --git a/services/receiver/src/p2p_session.rs b/services/receiver/src/p2p_session.rs index a91aa2ca..d63d00b6 100644 --- a/services/receiver/src/p2p_session.rs +++ b/services/receiver/src/p2p_session.rs @@ -33,8 +33,8 @@ use prost::Message; use rt_iroh::{Connection, Endpoint, EndpointAddr, RecvStream, SendStream}; use rt_p2p_protocol::{ Ack, ControlC2F, ControlF2C, DataC2F, DataF2C, DataSubscribe, EventBatch, GapNotice, Hello, - HelloOk, MAX_FRAME_BYTES, StreamCatalog, SubscribeMode, control_c2f, control_f2c, data_c2f, - data_f2c, encode_frame, + HelloOk, ProtocolError, StreamCatalog, SubscribeMode, control_c2f, control_f2c, data_c2f, + data_f2c, decode_frame_len, decode_frame_payload, encode_frame, }; use tokio::sync::broadcast; @@ -343,15 +343,20 @@ where recv.read_exact(&mut len_buf) .await .map_err(|e| P2pSessionError::Read(e.to_string()))?; - let len = u32::from_le_bytes(len_buf) as usize; - if len > MAX_FRAME_BYTES { - return Err(P2pSessionError::FrameTooLarge(len)); - } + let len = decode_frame_len(len_buf).map_err(map_frame_decode_error)?; let mut payload = vec![0u8; len]; recv.read_exact(&mut payload) .await .map_err(|e| P2pSessionError::Read(e.to_string()))?; - M::decode(payload.as_slice()).map_err(|e| P2pSessionError::Decode(e.to_string())) + decode_frame_payload(len_buf, payload.as_slice()).map_err(map_frame_decode_error) +} + +fn map_frame_decode_error(error: ProtocolError) -> P2pSessionError { + match error { + ProtocolError::FrameTooLarge { length, .. } => P2pSessionError::FrameTooLarge(length), + ProtocolError::DecodeError { source, .. } => P2pSessionError::Decode(source.to_string()), + other => P2pSessionError::Decode(other.to_string()), + } } /// Dials `forwarder_addr`, opens the control stream, sends `client_hello`, and @@ -642,7 +647,7 @@ mod tests { use super::*; use crate::control_api::ConnectionState; use rt_iroh::EndpointBuilder; - use rt_p2p_protocol::{ReadRecord, StreamCatalog, StreamEntry, SubscribeOk}; + use rt_p2p_protocol::{MAX_FRAME_BYTES, ReadRecord, StreamCatalog, StreamEntry, SubscribeOk}; use rt_test_utils::p2p::{ConnectivityFault, ForwarderScript, MockForwarderPeer}; use rt_test_utils::poll_until; From 190fb7e3c1c2e74c246015a2ba9d2d709c0df259 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 16:39:04 -0400 Subject: [PATCH 26/38] refactor(forwarder): split server HTTP clients out of allowlist module --- services/forwarder/src/p2p/allowlist.rs | 1014 +----------------- services/forwarder/src/p2p/mod.rs | 14 +- services/forwarder/src/p2p/server_client.rs | 1020 +++++++++++++++++++ 3 files changed, 1032 insertions(+), 1016 deletions(-) create mode 100644 services/forwarder/src/p2p/server_client.rs diff --git a/services/forwarder/src/p2p/allowlist.rs b/services/forwarder/src/p2p/allowlist.rs index 329d6cbd..adb33626 100644 --- a/services/forwarder/src/p2p/allowlist.rs +++ b/services/forwarder/src/p2p/allowlist.rs @@ -18,11 +18,9 @@ //! endpoint id ([`AllowList::try_register_connection`]); when an update removes a //! peer, its open connections are force-closed immediately. //! -//! Updates are sourced from the server: [`ServerAllowListClient`] fetches -//! the active receiver set over bearer-authenticated HTTP, and -//! [`run_allowlist_distribution`] keeps the list fresh from a startup fetch, -//! pushed snapshots, and periodic polling. Reader control/status event mapping -//! remains out of scope for this allow-list module. +//! Server refresh and catalog HTTP clients live in the sibling +//! `server_client` module; this module owns only the admission state machine and +//! last-known cache. use std::collections::{HashMap, HashSet}; use std::fs::{self, File, OpenOptions}; @@ -30,41 +28,13 @@ use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; -use std::time::Duration; use rt_iroh::{Connection, EndpointId}; -pub use rt_server_api::allowlist::ReceiverAllowListResponse as ReceiverAllowListUpdate; -use rt_server_api::catalog::ForwarderOwnCatalogResponse; -pub use rt_server_api::catalog::{ - ForwarderCatalogRequest as ForwarderCatalog, ForwarderCatalogStream, -}; -use rt_server_api::register::{RegisterRequest, RegisterResponse}; -use tokio::sync::mpsc; /// QUIC application error code used when force-closing a revoked peer's /// connection after it is removed from the allow-list. const REVOKED_ERROR_CODE: u32 = 3; -/// Production poll cadence for refreshing receiver authorization from the server. -pub const DEFAULT_ALLOWLIST_POLL_INTERVAL: Duration = Duration::from_secs(60); - -/// How long each long-poll request asks the server to hold open while waiting -/// for an allow-list change. Bounds how long a forwarder waits between re-arming -/// the push subscription, and is kept under typical proxy idle timeouts. -pub const ALLOWLIST_PUSH_HOLD: Duration = Duration::from_secs(25); - -/// Initial reconnect backoff for the long-poll push subscription after a server -/// error, doubled up to [`ALLOWLIST_PUSH_MAX_BACKOFF`]. -const ALLOWLIST_PUSH_INITIAL_BACKOFF: Duration = Duration::from_secs(1); - -/// Cap on the long-poll push subscription reconnect backoff. -const ALLOWLIST_PUSH_MAX_BACKOFF: Duration = Duration::from_secs(30); - -/// Conservative bound on a single server allow-list HTTP request. Without a -/// timeout a hung server would stall the distribution loop's initial refresh -/// and polling indefinitely; this caps how long any one fetch can block. -pub const DEFAULT_ALLOWLIST_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); - /// Shared, mutable allow-list state guarded by a single mutex. #[derive(Debug, Default)] struct State { @@ -239,428 +209,6 @@ impl AllowList { } } -/// HTTP client for fetching receiver allow-list snapshots from the server. -#[derive(Clone)] -pub struct ServerAllowListClient { - http: reqwest::Client, - base_url: String, - bearer_token: Arc, -} - -// Manual `Debug` so the bearer token can never leak through formatting (e.g. -// when a struct holding the client is logged or asserted on). -impl std::fmt::Debug for ServerAllowListClient { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ServerAllowListClient") - .field("base_url", &self.base_url) - .field("bearer_token", &"") - .finish() - } -} - -impl ServerAllowListClient { - /// Builds a client with the [`DEFAULT_ALLOWLIST_REQUEST_TIMEOUT`] applied to - /// every allow-list request. - #[must_use] - pub fn new(base_url: impl Into, bearer_token: impl Into) -> Self { - Self::with_timeout(base_url, bearer_token, DEFAULT_ALLOWLIST_REQUEST_TIMEOUT) - } - - /// Builds a client that bounds every allow-list request to `request_timeout`. - #[must_use] - pub fn with_timeout( - base_url: impl Into, - bearer_token: impl Into, - request_timeout: Duration, - ) -> Self { - let http = reqwest::Client::builder() - .timeout(request_timeout) - .build() - // A builder failure here only means the platform TLS/transport - // backend could not initialise. Never fall back to an untimed - // client: that would silently drop the request timeout and let a - // hung server stall the distribution loop indefinitely. Such a - // failure is a fatal environment problem, so surface it loudly. - .expect( - "reqwest client builder with request timeout must initialise; \ - a failure here indicates the platform TLS/transport backend is unavailable", - ); - Self { - http, - base_url: base_url.into().trim_end_matches('/').to_owned(), - bearer_token: Arc::from(bearer_token.into()), - } - } - - async fn fetch(&self) -> Result { - let url = format!("{}/allowlist/receivers", self.base_url); - Ok(self - .http - .get(url) - .bearer_auth(self.bearer_token.as_ref()) - .send() - .await? - .error_for_status()? - .json() - .await?) - } - - /// Long-polls the server allow-list: sends `since` (the last applied - /// version) and a `wait` budget, so the server holds the request until the - /// allow-list changes and returns the fresh snapshot — giving near-instant - /// approval propagation. The per-request timeout is overridden to exceed - /// `wait` so the held request is not aborted by the client's short default - /// timeout. A `since` of `None` requests an immediate snapshot (initial - /// fetch / resync). - async fn fetch_waiting( - &self, - since: Option, - wait: Duration, - ) -> Result { - // Built by hand (rather than reqwest's `query` builder) so no extra - // reqwest feature is required; both values are plain integers, so no - // percent-encoding is needed. - let url = match since { - Some(since) => format!( - "{}/allowlist/receivers?wait={}&since={since}", - self.base_url, - wait.as_secs() - ), - None => format!( - "{}/allowlist/receivers?wait={}", - self.base_url, - wait.as_secs() - ), - }; - Ok(self - .http - .get(url) - .bearer_auth(self.bearer_token.as_ref()) - // Allow the response to arrive any time within the server hold plus - // a transport margin, regardless of the client's short default. - .timeout(wait + Duration::from_secs(10)) - .send() - .await? - .error_for_status()? - .json() - .await?) - } -} - -/// HTTP client for registering the forwarder and pushing its stream catalog. -#[derive(Clone)] -pub struct ServerCatalogClient { - http: reqwest::Client, - base_url: String, - bearer_token: Arc, -} - -impl std::fmt::Debug for ServerCatalogClient { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ServerCatalogClient") - .field("base_url", &self.base_url) - .field("bearer_token", &"") - .finish() - } -} - -impl ServerCatalogClient { - /// Builds a client with the default server request timeout. - #[must_use] - pub fn new(base_url: impl Into, bearer_token: impl Into) -> Self { - Self::with_timeout(base_url, bearer_token, DEFAULT_ALLOWLIST_REQUEST_TIMEOUT) - } - - /// Builds a client that bounds every registration/catalog request. - #[must_use] - pub fn with_timeout( - base_url: impl Into, - bearer_token: impl Into, - request_timeout: Duration, - ) -> Self { - let http = reqwest::Client::builder() - .timeout(request_timeout) - .build() - .expect( - "reqwest client builder with request timeout must initialise; \ - a failure here indicates the platform TLS/transport backend is unavailable", - ); - Self { - http, - base_url: base_url.into().trim_end_matches('/').to_owned(), - bearer_token: Arc::from(bearer_token.into()), - } - } - - /// Bootstrap this forwarder against the server `/register` endpoint using the - /// configured bearer (an enrollment voucher on first boot, or the device's - /// own token for same-endpoint recovery), returning the server-minted - /// per-device token. The caller persists it and uses it for all later calls. - pub async fn bootstrap(&self, endpoint_id: &str) -> Result { - let url = format!("{}/register", self.base_url); - let response = self - .http - .post(url) - .bearer_auth(self.bearer_token.as_ref()) - .json(&RegisterRequest { - endpoint_id: endpoint_id.to_owned(), - device_kind: "forwarder".to_owned(), - display_name: None, - }) - .send() - .await? - .error_for_status()? - .json::() - .await?; - response - .device_token - .ok_or(CatalogPushError::MissingMintedToken) - } - - /// Fetches this forwarder's own stored stream catalog from the server - /// (`GET /forwarder/catalog`, self-scoped by the device token). - /// - /// Used at startup to restore per-stream epoch/next_seq high-water after - /// local journal loss. An older server without this route returns 404, - /// which surfaces as an `Err` here — callers treat any error as - /// "registry unavailable". - pub async fn fetch_own_catalog(&self) -> Result, CatalogPushError> { - let url = format!("{}/forwarder/catalog", self.base_url); - let response = self - .http - .get(url) - .bearer_auth(self.bearer_token.as_ref()) - .send() - .await? - .error_for_status()? - .json::() - .await?; - Ok(response.streams) - } - - /// Pushes this forwarder's latest identity and stream catalog. - pub async fn push_catalog(&self, catalog: &ForwarderCatalog) -> Result<(), CatalogPushError> { - let url = format!("{}/forwarder/catalog", self.base_url); - self.http - .post(url) - .bearer_auth(self.bearer_token.as_ref()) - .json(catalog) - .send() - .await? - .error_for_status()?; - Ok(()) - } -} - -#[derive(Debug, thiserror::Error)] -pub enum CatalogPushError { - // `reqwest::Error`'s `Display` covers URL/status/transport but never - // request headers, so the bearer token cannot leak here. - #[error("server catalog request failed: {0}")] - Http(#[from] reqwest::Error), - #[error("server /register did not return a minted device token")] - MissingMintedToken, -} - -#[derive(Debug, thiserror::Error)] -pub enum AllowListRefreshError { - // `reqwest::Error`'s `Display` covers the failing URL and transport cause - // but never request headers, so the bearer token cannot leak here. - #[error("server allow-list request failed: {0}")] - Http(#[from] reqwest::Error), - #[error("failed to apply receiver allow-list update: {0}")] - Apply(#[from] io::Error), -} - -/// Fetches the current server allow-list and applies it to `allow_list`. -pub async fn fetch_and_apply_once( - client: &ServerAllowListClient, - allow_list: &AllowList, -) -> Result, AllowListRefreshError> { - let update = client.fetch().await?; - apply_receiver_update(allow_list, update) -} - -/// Applies a pushed or fetched wire-format receiver allow-list update. -pub fn apply_receiver_update( - allow_list: &AllowList, - update: ReceiverAllowListUpdate, -) -> Result, AllowListRefreshError> { - let allowed = parse_update(update); - Ok(allow_list.apply_update(allowed)?) -} - -/// In-flight poll-fetch future held across `select!` iterations so pushed -/// updates stay responsive while a slow poll request is outstanding. -/// -/// The poll *fetches only*; it does not apply. Applying is deferred to the -/// distribution loop so a snapshot captured before a newer pushed update can be -/// discarded instead of clobbering the newer state (see the generation guard in -/// [`run_allowlist_distribution`]). -type PollFetch = std::pin::Pin< - Box< - dyn std::future::Future> - + Send, - >, ->; - -/// Keeps `allow_list` fresh from startup fetches, pushed snapshots, and polling. -pub async fn run_allowlist_distribution( - allow_list: AllowList, - client: ServerAllowListClient, - mut pushed_updates: mpsc::Receiver, - poll_interval: Duration, -) { - if let Err(err) = fetch_and_apply_once(&client, &allow_list).await { - tracing::warn!(error = %err, "initial receiver allow-list refresh failed"); - } - - let poll_interval = if poll_interval.is_zero() { - DEFAULT_ALLOWLIST_POLL_INTERVAL - } else { - poll_interval - }; - let mut ticker = tokio::time::interval(poll_interval); - ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - ticker.tick().await; - let mut push_closed = false; - // An in-flight poll fetch is held here rather than awaited inline so pushed - // updates stay responsive even while a slow (or timing-out) poll request is - // outstanding. At most one poll fetch runs at a time; a tick that lands - // while one is still in flight is dropped (the next tick re-polls). - let mut poll_fetch: Option = None; - // Monotonic count of applied updates. Bumped on every successful apply - // (pushed or polled). A poll captures this when it begins fetching; if the - // count has moved by the time the fetch completes, a newer update applied - // while the poll was in flight, so the (now stale) poll snapshot is - // discarded rather than overwriting the newer state. Without this guard a - // slow poll holding a pre-revocation snapshot could re-authorize a receiver - // that a pushed revocation had already removed. - let mut generation: u64 = 0; - let mut poll_generation: u64 = 0; - - loop { - tokio::select! { - update = pushed_updates.recv(), if !push_closed => { - match update { - Some(update) => { - match apply_receiver_update(&allow_list, update) { - Ok(_) => generation = generation.wrapping_add(1), - Err(err) => tracing::warn!( - error = %err, - "pushed receiver allow-list update failed", - ), - } - } - None => push_closed = true, - } - } - _ = ticker.tick() => { - if poll_fetch.is_none() { - let client = client.clone(); - poll_generation = generation; - poll_fetch = Some(Box::pin(async move { client.fetch().await })); - } - } - result = async { poll_fetch.as_mut().expect("poll fetch present").await }, if poll_fetch.is_some() => { - poll_fetch = None; - match result { - Ok(update) => { - if generation == poll_generation { - match apply_receiver_update(&allow_list, update) { - Ok(_) => generation = generation.wrapping_add(1), - Err(err) => tracing::warn!( - error = %err, - "polled receiver allow-list refresh failed", - ), - } - } else { - tracing::debug!( - "discarding stale polled receiver allow-list snapshot \ - superseded by a newer update", - ); - } - } - Err(err) => tracing::warn!( - error = %err, - "polled receiver allow-list refresh failed", - ), - } - } - } - } -} - -/// Long-poll the server for receiver allow-list changes and forward each fresh -/// snapshot into `push_tx`, which [`run_allowlist_distribution`] applies. -/// -/// This is the push transport that complements the periodic poll backstop: an -/// approval on the server releases the held request within milliseconds, so a -/// newly approved receiver is admitted almost immediately instead of waiting -/// up to a full poll interval. Loops until the receiving distribution loop is -/// gone (the channel send fails). Server errors back off and retry; the -/// distribution loop's polling keeps the allow-list correct meanwhile. -pub async fn run_allowlist_push_subscription( - client: ServerAllowListClient, - push_tx: mpsc::Sender, - hold: Duration, -) { - // `None` requests an immediate snapshot; afterwards we echo the server's - // version as `since` to long-poll for the *next* change. - let mut since: Option = None; - let mut backoff = ALLOWLIST_PUSH_INITIAL_BACKOFF; - - loop { - let started = tokio::time::Instant::now(); - match client.fetch_waiting(since, hold).await { - Ok(update) => { - backoff = ALLOWLIST_PUSH_INITIAL_BACKOFF; - let version = update.version; - let changed = since != Some(version); - since = Some(version); - if changed { - // A real change (or the initial snapshot): hand it to the - // distribution loop. A closed channel means that loop is - // gone, so this subscription has no consumer left. - if push_tx.send(update).await.is_err() { - return; - } - } else { - // No change within the hold. Pace the next request to the - // intended hold window so a non-holding or older server - // (which returns immediately and never sets a moving - // version) is not hammered; a compliant server already - // consumed ~`hold`, so this adds nothing. - tokio::time::sleep_until(started + hold).await; - } - } - Err(err) => { - tracing::warn!(error = %err, "receiver allow-list push subscription failed; retrying"); - tokio::time::sleep(backoff).await; - backoff = (backoff * 2).min(ALLOWLIST_PUSH_MAX_BACKOFF); - } - } - } -} - -fn parse_update(update: ReceiverAllowListUpdate) -> Vec { - update - .receiver_endpoint_ids - .into_iter() - .filter_map(|endpoint_id| match EndpointId::from_str(&endpoint_id) { - Ok(endpoint_id) => Some(endpoint_id), - Err(error) => { - tracing::warn!( - endpoint_id = %endpoint_id, - error = %error, - "skipping malformed receiver endpoint id in allow-list update", - ); - None - } - }) - .collect() -} - /// Deregisters a tracked connection when dropped, keeping the connection /// registry bounded to currently-open admitted connections. #[must_use = "dropping the guard immediately deregisters the tracked connection"] @@ -722,18 +270,9 @@ fn persist(path: &Path, allowed: &HashSet) -> io::Result<()> { mod tests { use super::*; - use std::sync::Arc; use std::time::Duration; - use axum::{ - Json, Router, - extract::State, - http::{HeaderMap, StatusCode, header::AUTHORIZATION}, - response::{IntoResponse, Response}, - routing::{get, post}, - }; - use rt_iroh::{Endpoint, EndpointAddr, EndpointBuilder}; - use tokio::sync::{Mutex as TokioMutex, mpsc, watch}; + use rt_iroh::{Endpoint, EndpointBuilder, EndpointAddr}; type BoxError = Box; type TestResult = Result<(), BoxError>; @@ -789,56 +328,6 @@ mod tests { Ok((connection, buf)) } - #[derive(Clone)] - struct TestAllowListServerState { - bearer_token: &'static str, - receiver_endpoint_ids: Arc>>, - // Retained count of completed authorized fetches. Unlike a `Notify`, - // a `watch` holds its latest value, so a fetch that completes before a - // waiter starts observing is not lost — letting tests deterministically - // sequence "initial fetch happened" against later poll fetches. - fetches: watch::Sender, - } - - async fn test_allowlist_handler( - State(state): State, - headers: HeaderMap, - ) -> Response { - let authorized = headers - .get(AUTHORIZATION) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value == format!("Bearer {}", state.bearer_token)); - if !authorized { - return StatusCode::UNAUTHORIZED.into_response(); - } - - let receiver_endpoint_ids = state.receiver_endpoint_ids.lock().await.clone(); - // Count the fetch only after the current id set has been read, so an - // observed count of N guarantees N fetches each saw the id set in force - // at their read. - state.fetches.send_modify(|count| *count += 1); - Json(serde_json::json!({ "receiver_endpoint_ids": receiver_endpoint_ids })).into_response() - } - - async fn spawn_test_allowlist_server( - receiver_endpoint_ids: Arc>>, - ) -> Result<(String, watch::Receiver, tokio::task::JoinHandle<()>), BoxError> { - let (fetches, fetches_rx) = watch::channel(0u64); - let app = Router::new() - .route("/allowlist/receivers", get(test_allowlist_handler)) - .with_state(TestAllowListServerState { - bearer_token: "thin-secret", - receiver_endpoint_ids, - fetches, - }); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; - let url = format!("http://{}", listener.local_addr()?); - let server = tokio::spawn(async move { - let _ = axum::serve(listener, app).await; - }); - Ok((url, fetches_rx, server)) - } - #[tokio::test] async fn denied_peer_cannot_subscribe() -> TestResult { let receiver = EndpointBuilder::test([40; 32]).bind().await?; @@ -1071,499 +560,4 @@ mod tests { server_only.close().await; Ok(()) } - - #[derive(Clone)] - struct TestCatalogServerState { - bearer_token: &'static str, - received: Arc>>, - } - - async fn test_catalog_handler( - State(state): State, - headers: HeaderMap, - Json(body): Json, - ) -> Response { - let authorized = headers - .get(AUTHORIZATION) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value == format!("Bearer {}", state.bearer_token)); - if !authorized { - return StatusCode::UNAUTHORIZED.into_response(); - } - *state.received.lock().await = Some((headers.clone(), body)); - StatusCode::OK.into_response() - } - - /// `/register` mock: records the request and returns a minted device token. - async fn test_register_handler( - State(state): State, - headers: HeaderMap, - Json(body): Json, - ) -> Response { - let authorized = headers - .get(AUTHORIZATION) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value == format!("Bearer {}", state.bearer_token)); - if !authorized { - return StatusCode::UNAUTHORIZED.into_response(); - } - *state.received.lock().await = Some((headers.clone(), body)); - Json(serde_json::json!({ - "endpoint_id": "fwd-node-1", - "device_kind": "forwarder", - "approval_state": "pending", - "device_token": "rtk_minted_secret" - })) - .into_response() - } - - #[tokio::test] - async fn catalog_client_posts_registration_and_catalog_with_bearer() -> TestResult { - let received = Arc::new(TokioMutex::new(None)); - let app = Router::new() - .route("/register", post(test_register_handler)) - .route("/forwarder/catalog", post(test_catalog_handler)) - .with_state(TestCatalogServerState { - bearer_token: "thin-secret", - received: received.clone(), - }); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; - let base_url = format!("http://{}", listener.local_addr()?); - let server = tokio::spawn(async move { - let _ = axum::serve(listener, app).await; - }); - - let client = ServerCatalogClient::new(base_url, "thin-secret"); - let minted = client - .bootstrap("fwd-node-1") - .await - .expect("bootstrap request succeeds"); - assert_eq!(minted, "rtk_minted_secret"); - let (headers, register_body) = received - .lock() - .await - .take() - .expect("server captured registration request"); - assert_eq!( - headers.get(AUTHORIZATION).unwrap().to_str().unwrap(), - "Bearer thin-secret" - ); - assert_eq!(register_body["endpoint_id"], "fwd-node-1"); - assert_eq!(register_body["device_kind"], "forwarder"); - - client - .push_catalog(&ForwarderCatalog { - endpoint_id: "fwd-node-1".to_owned(), - display_name: Some("Start Line".to_owned()), - direct_addrs: vec!["127.0.0.1:12345".to_owned()], - streams: vec![ForwarderCatalogStream { - stream_id: "reader-a".to_owned(), - epoch: 3, - next_seq: 42, - }], - }) - .await - .expect("catalog request succeeds"); - let (headers, catalog_body) = received - .lock() - .await - .take() - .expect("server captured catalog request"); - assert_eq!( - headers.get(AUTHORIZATION).unwrap().to_str().unwrap(), - "Bearer thin-secret" - ); - assert_eq!(catalog_body["endpoint_id"], "fwd-node-1"); - assert_eq!(catalog_body["display_name"], "Start Line"); - assert_eq!( - catalog_body["direct_addrs"], - serde_json::json!(["127.0.0.1:12345"]) - ); - assert_eq!(catalog_body["streams"][0]["stream_id"], "reader-a"); - assert_eq!(catalog_body["streams"][0]["epoch"], 3); - assert_eq!(catalog_body["streams"][0]["next_seq"], 42); - - server.abort(); - Ok(()) - } - - #[tokio::test] - async fn approved_receiver_appears_in_forwarder_list() -> TestResult { - let receiver = EndpointBuilder::test([50; 32]).bind().await?; - let receiver_endpoint_ids = - Arc::new(TokioMutex::new(vec![receiver.endpoint_id().to_string()])); - let (base_url, _fetches, server) = - spawn_test_allowlist_server(receiver_endpoint_ids).await?; - let client = ServerAllowListClient::new(base_url, "thin-secret"); - let allow = AllowList::default(); - - fetch_and_apply_once(&client, &allow).await?; - - assert!(allow.contains(&receiver.endpoint_id())); - server.abort(); - receiver.close().await; - Ok(()) - } - - #[tokio::test] - async fn revoke_propagates() -> TestResult { - let receiver = EndpointBuilder::test([51; 32]).bind().await?; - let allow = AllowList::new([receiver.endpoint_id()]); - let receiver_endpoint_ids = - Arc::new(TokioMutex::new(vec![receiver.endpoint_id().to_string()])); - let (base_url, _fetches, server) = - spawn_test_allowlist_server(receiver_endpoint_ids).await?; - let client = ServerAllowListClient::new(base_url, "thin-secret"); - let (tx, rx) = mpsc::channel(1); - let sync = tokio::spawn(run_allowlist_distribution( - allow.clone(), - client, - rx, - Duration::from_secs(60), - )); - - tx.send(ReceiverAllowListUpdate::replace(Vec::new())) - .await?; - tokio::time::timeout(Duration::from_secs(5), async { - while allow.contains(&receiver.endpoint_id()) { - tokio::task::yield_now().await; - } - }) - .await?; - - assert!(!allow.contains(&receiver.endpoint_id())); - sync.abort(); - server.abort(); - receiver.close().await; - Ok(()) - } - - #[tokio::test] - async fn poll_backstop_refreshes() -> TestResult { - let receiver = EndpointBuilder::test([52; 32]).bind().await?; - let receiver_endpoint_ids = Arc::new(TokioMutex::new(Vec::new())); - let (base_url, mut fetches, server) = - spawn_test_allowlist_server(receiver_endpoint_ids.clone()).await?; - let client = ServerAllowListClient::new(base_url, "thin-secret"); - let allow = AllowList::default(); - let (_tx, rx) = mpsc::channel(1); - // Short poll interval with real time: the backstop must re-fetch on its - // own without any pushed update. - let sync = tokio::spawn(run_allowlist_distribution( - allow.clone(), - client, - rx, - Duration::from_millis(50), - )); - - // Wait (on retained state) until the initial fetch has read the empty - // set, so the receiver can only be admitted by a later poll fetch. - tokio::time::timeout( - Duration::from_secs(5), - fetches.wait_for(|&count| count >= 1), - ) - .await??; - assert!(!allow.contains(&receiver.endpoint_id())); - - // Publish the receiver; only the polling backstop can pick this up. - *receiver_endpoint_ids.lock().await = vec![receiver.endpoint_id().to_string()]; - - tokio::time::timeout(Duration::from_secs(5), async { - while !allow.contains(&receiver.endpoint_id()) { - tokio::task::yield_now().await; - } - }) - .await?; - - assert!(allow.contains(&receiver.endpoint_id())); - sync.abort(); - server.abort(); - receiver.close().await; - Ok(()) - } - - #[tokio::test] - async fn stale_poll_does_not_override_newer_revocation() -> TestResult { - use std::sync::atomic::{AtomicU64, Ordering}; - - let receiver = EndpointBuilder::test([55; 32]).bind().await?; - // Start empty; the initial fetch admits the receiver from the old list. - let allow = AllowList::default(); - - #[derive(Clone)] - struct StallState { - // Old, pre-revocation list returned by every fetch: it still - // authorizes the receiver. - old_list: Vec, - // 1-based count of requests received by the server. - requests: Arc, - // Broadcasts the latest request count so the test can sequence on it. - observed: watch::Sender, - // Releases exactly one gated (poll) request when notified once. - release: Arc, - } - - async fn stall_handler(State(state): State) -> Response { - let n = state.requests.fetch_add(1, Ordering::SeqCst) + 1; - state.observed.send_modify(|count| *count = (*count).max(n)); - // The initial fetch (request 1) returns immediately so the receiver - // is admitted. Every later (poll) request blocks until explicitly - // released, so the test controls exactly when a poll completes. - if n >= 2 { - state.release.notified().await; - } - Json(serde_json::json!({ "receiver_endpoint_ids": state.old_list })).into_response() - } - - let (observed_tx, mut observed_rx) = watch::channel(0u64); - let release = Arc::new(tokio::sync::Notify::new()); - let app = Router::new() - .route("/allowlist/receivers", get(stall_handler)) - .with_state(StallState { - old_list: vec![receiver.endpoint_id().to_string()], - requests: Arc::new(AtomicU64::new(0)), - observed: observed_tx, - release: release.clone(), - }); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; - let base_url = format!("http://{}", listener.local_addr()?); - let server = tokio::spawn(async move { - let _ = axum::serve(listener, app).await; - }); - - let client = ServerAllowListClient::new(base_url, "thin-secret"); - let (tx, rx) = mpsc::channel(1); - // Short poll interval so a poll fetch starts promptly after the initial - // fetch admits the receiver. - let sync = tokio::spawn(run_allowlist_distribution( - allow.clone(), - client, - rx, - Duration::from_millis(50), - )); - - // Initial fetch (request 1) admits the receiver from the old list. - tokio::time::timeout(Duration::from_secs(5), async { - while !allow.contains(&receiver.endpoint_id()) { - tokio::task::yield_now().await; - } - }) - .await?; - - // A poll (request 2) is now in flight and blocked on the release gate. - tokio::time::timeout( - Duration::from_secs(5), - observed_rx.wait_for(|&count| count >= 2), - ) - .await??; - - // Apply a pushed revocation while the poll fetch is still in flight. - tx.send(ReceiverAllowListUpdate::replace(Vec::new())) - .await?; - tokio::time::timeout(Duration::from_secs(5), async { - while allow.contains(&receiver.endpoint_id()) { - tokio::task::yield_now().await; - } - }) - .await?; - - // Let the stale poll (request 2) complete with the old, pre-revocation - // list. notify_one releases exactly one waiter, so request 3 stays - // blocked and cannot itself re-admit the receiver. - release.notify_one(); - - // Request 3 starting proves the stale poll result has been fully - // processed by the distribution loop (the poll slot freed, a new tick - // fired). Request 3 then blocks on the gate without responding. - tokio::time::timeout( - Duration::from_secs(5), - observed_rx.wait_for(|&count| count >= 3), - ) - .await??; - - assert!( - !allow.contains(&receiver.endpoint_id()), - "a stale in-flight poll must not re-authorize a receiver revoked while it was outstanding" - ); - - sync.abort(); - server.abort(); - receiver.close().await; - Ok(()) - } - - #[derive(Clone)] - struct PushServerState { - bearer_token: &'static str, - ids: Arc>>, - version: Arc, - changed: Arc, - } - - #[derive(serde::Deserialize)] - struct PushQuery { - since: Option, - wait: Option, - } - - /// Mock long-poll allow-list endpoint mirroring the real server: holds the - /// request when the caller's `since` matches the current version, releasing - /// on a version bump, and always echoes the current `version`. - async fn push_allowlist_handler( - State(state): State, - headers: HeaderMap, - axum::extract::Query(query): axum::extract::Query, - ) -> Response { - let authorized = headers - .get(AUTHORIZATION) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value == format!("Bearer {}", state.bearer_token)); - if !authorized { - return StatusCode::UNAUTHORIZED.into_response(); - } - let current = state.version.load(std::sync::atomic::Ordering::SeqCst); - let wait = query.wait.unwrap_or(0); - if wait > 0 && query.since == Some(current) { - let _ = tokio::time::timeout(Duration::from_secs(wait), state.changed.notified()).await; - } - let version = state.version.load(std::sync::atomic::Ordering::SeqCst); - let ids = state.ids.lock().await.clone(); - Json(serde_json::json!({ "receiver_endpoint_ids": ids, "version": version })) - .into_response() - } - - #[tokio::test] - async fn push_subscription_delivers_initial_snapshot_then_releases_on_bump() -> TestResult { - let ids = Arc::new(TokioMutex::new(Vec::::new())); - let version = Arc::new(std::sync::atomic::AtomicU64::new(0)); - let changed = Arc::new(tokio::sync::Notify::new()); - let app = Router::new() - .route("/allowlist/receivers", get(push_allowlist_handler)) - .with_state(PushServerState { - bearer_token: "thin-secret", - ids: ids.clone(), - version: version.clone(), - changed: changed.clone(), - }); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; - let base_url = format!("http://{}", listener.local_addr()?); - let server = tokio::spawn(async move { - let _ = axum::serve(listener, app).await; - }); - - let client = ServerAllowListClient::new(base_url, "thin-secret"); - let (tx, mut rx) = mpsc::channel(8); - let sub = tokio::spawn(run_allowlist_push_subscription( - client, - tx, - Duration::from_secs(5), - )); - - // The initial request omits `since`, so the server returns immediately - // with the empty snapshot at version 0. - let first = tokio::time::timeout(Duration::from_secs(2), rx.recv()) - .await? - .expect("initial snapshot delivered"); - assert_eq!(first.version, 0); - assert!(first.receiver_endpoint_ids.is_empty()); - - // The subscription is now held at version 0. An approval bumps the - // version and must release it within milliseconds with the new set. - *ids.lock().await = vec!["receiver-1".to_owned()]; - version.store(1, std::sync::atomic::Ordering::SeqCst); - changed.notify_waiters(); - - let second = tokio::time::timeout(Duration::from_secs(2), rx.recv()) - .await? - .expect("approval releases the held long-poll"); - assert_eq!(second.version, 1); - assert_eq!(second.receiver_endpoint_ids, vec!["receiver-1".to_owned()]); - - sub.abort(); - server.abort(); - Ok(()) - } - - #[tokio::test] - async fn debug_redacts_bearer_token() { - let client = ServerAllowListClient::new("http://thin.example", "super-secret-token"); - let rendered = format!("{client:?}"); - assert!( - !rendered.contains("super-secret-token"), - "bearer token must never appear in Debug output: {rendered}" - ); - assert!(rendered.contains("")); - } - - #[tokio::test] - async fn hung_server_request_times_out() -> TestResult { - // A handler that never responds: only the client request timeout can - // unblock the fetch. - async fn hang() -> Response { - std::future::pending::<()>().await; - StatusCode::OK.into_response() - } - let app = Router::new().route("/allowlist/receivers", get(hang)); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; - let base_url = format!("http://{}", listener.local_addr()?); - let server = tokio::spawn(async move { - let _ = axum::serve(listener, app).await; - }); - - let client = ServerAllowListClient::with_timeout( - base_url, - "thin-secret", - Duration::from_millis(200), - ); - let allow = AllowList::default(); - - // Outer guard far exceeds the request timeout: if the request timeout - // works, the fetch returns an error well before this elapses. - let result = tokio::time::timeout( - Duration::from_secs(5), - fetch_and_apply_once(&client, &allow), - ) - .await?; - assert!( - matches!(result, Err(AllowListRefreshError::Http(_))), - "hung request must surface as a bounded HTTP error, got {result:?}" - ); - - server.abort(); - Ok(()) - } - - #[tokio::test] - async fn malformed_active_id_does_not_block_valid_revocation() -> TestResult { - let retained = EndpointBuilder::test([53; 32]).bind().await?; - let revoked = EndpointBuilder::test([54; 32]).bind().await?; - let allow = AllowList::new([retained.endpoint_id(), revoked.endpoint_id()]); - - // The server can contain arbitrary endpoint_id text from receiver - // registration. An invalid active id must be ignored, while the valid - // omission of `revoked` still removes its authorization. - let receiver_endpoint_ids = Arc::new(TokioMutex::new(vec![ - retained.endpoint_id().to_string(), - "not-a-valid-endpoint-id".to_owned(), - ])); - let (base_url, _fetches, server) = - spawn_test_allowlist_server(receiver_endpoint_ids).await?; - let client = ServerAllowListClient::new(base_url, "thin-secret"); - - let result = fetch_and_apply_once(&client, &allow).await?; - assert_eq!(result, vec![revoked.endpoint_id()]); - assert!( - allow.contains(&retained.endpoint_id()), - "valid active receiver must remain authorized" - ); - assert!( - !allow.contains(&revoked.endpoint_id()), - "valid revocation must apply even when the response also contains a malformed id" - ); - - server.abort(); - retained.close().await; - revoked.close().await; - Ok(()) - } } diff --git a/services/forwarder/src/p2p/mod.rs b/services/forwarder/src/p2p/mod.rs index f3da48ac..91db44ef 100644 --- a/services/forwarder/src/p2p/mod.rs +++ b/services/forwarder/src/p2p/mod.rs @@ -21,6 +21,7 @@ mod data; mod endpoint; mod reader_control; mod remote_config; +mod server_client; use std::net::SocketAddrV4; use std::num::TryFromIntError; @@ -37,12 +38,7 @@ use rt_p2p_protocol::{StreamCatalog, StreamEntry}; use tokio::sync::{Mutex, mpsc}; use tokio::task::JoinHandle; -pub use allowlist::{ - ALLOWLIST_PUSH_HOLD, AllowList, AllowListRefreshError, CatalogPushError, - DEFAULT_ALLOWLIST_POLL_INTERVAL, ForwarderCatalog, ForwarderCatalogStream, - ReceiverAllowListUpdate, ServerAllowListClient, ServerCatalogClient, apply_receiver_update, - fetch_and_apply_once, run_allowlist_distribution, run_allowlist_push_subscription, -}; +pub use allowlist::AllowList; pub use control::{ CatalogProvider, ConfigGetFuture, ConfigSetFuture, ControlEvent, ControlEventReceiver, ControlEventSender, HeartbeatConfig, NoopReaderControlHandler, NoopRemoteConfigHandler, @@ -54,6 +50,12 @@ pub use data::{DataConfig, serve_data_streams}; pub use endpoint::P2pEndpoint; pub use reader_control::ForwarderReaderControlHandler; pub use remote_config::ForwarderRemoteConfigHandler; +pub use server_client::{ + ALLOWLIST_PUSH_HOLD, AllowListRefreshError, CatalogPushError, DEFAULT_ALLOWLIST_POLL_INTERVAL, + ForwarderCatalog, ForwarderCatalogStream, ReceiverAllowListUpdate, ServerAllowListClient, + ServerCatalogClient, apply_receiver_update, fetch_and_apply_once, run_allowlist_distribution, + run_allowlist_push_subscription, +}; const DEFAULT_P2P_SECRET_KEY_PATH: &str = "/var/lib/rusty-timer/p2p-secret.key"; const DEFAULT_FORWARDER_CATALOG_PUSH_INTERVAL: Duration = Duration::from_secs(30); diff --git a/services/forwarder/src/p2p/server_client.rs b/services/forwarder/src/p2p/server_client.rs new file mode 100644 index 00000000..5881d2fe --- /dev/null +++ b/services/forwarder/src/p2p/server_client.rs @@ -0,0 +1,1020 @@ +//! Server HTTP clients and refresh loops for forwarder P2P allow-list/catalog state. + +use std::io; +use std::str::FromStr; +use std::sync::Arc; +use std::time::Duration; + +use rt_iroh::EndpointId; +pub use rt_server_api::allowlist::ReceiverAllowListResponse as ReceiverAllowListUpdate; +use rt_server_api::catalog::ForwarderOwnCatalogResponse; +pub use rt_server_api::catalog::{ + ForwarderCatalogRequest as ForwarderCatalog, ForwarderCatalogStream, +}; +use rt_server_api::register::{RegisterRequest, RegisterResponse}; +use tokio::sync::mpsc; + +use super::allowlist::AllowList; + +/// Production poll cadence for refreshing receiver authorization from the server. +pub const DEFAULT_ALLOWLIST_POLL_INTERVAL: Duration = Duration::from_secs(60); + +/// How long each long-poll request asks the server to hold open while waiting +/// for an allow-list change. Bounds how long a forwarder waits between re-arming +/// the push subscription, and is kept under typical proxy idle timeouts. +pub const ALLOWLIST_PUSH_HOLD: Duration = Duration::from_secs(25); + +/// Initial reconnect backoff for the long-poll push subscription after a server +/// error, doubled up to [`ALLOWLIST_PUSH_MAX_BACKOFF`]. +const ALLOWLIST_PUSH_INITIAL_BACKOFF: Duration = Duration::from_secs(1); + +/// Cap on the long-poll push subscription reconnect backoff. +const ALLOWLIST_PUSH_MAX_BACKOFF: Duration = Duration::from_secs(30); + +/// Conservative bound on a single server allow-list HTTP request. Without a +/// timeout a hung server would stall the distribution loop's initial refresh +/// and polling indefinitely; this caps how long any one fetch can block. +pub const DEFAULT_ALLOWLIST_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); + +/// HTTP client for fetching receiver allow-list snapshots from the server. +#[derive(Clone)] +pub struct ServerAllowListClient { + http: reqwest::Client, + base_url: String, + bearer_token: Arc, +} + +// Manual `Debug` so the bearer token can never leak through formatting (e.g. +// when a struct holding the client is logged or asserted on). +impl std::fmt::Debug for ServerAllowListClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ServerAllowListClient") + .field("base_url", &self.base_url) + .field("bearer_token", &"") + .finish() + } +} + +impl ServerAllowListClient { + /// Builds a client with the [`DEFAULT_ALLOWLIST_REQUEST_TIMEOUT`] applied to + /// every allow-list request. + #[must_use] + pub fn new(base_url: impl Into, bearer_token: impl Into) -> Self { + Self::with_timeout(base_url, bearer_token, DEFAULT_ALLOWLIST_REQUEST_TIMEOUT) + } + + /// Builds a client that bounds every allow-list request to `request_timeout`. + #[must_use] + pub fn with_timeout( + base_url: impl Into, + bearer_token: impl Into, + request_timeout: Duration, + ) -> Self { + let http = reqwest::Client::builder() + .timeout(request_timeout) + .build() + // A builder failure here only means the platform TLS/transport + // backend could not initialise. Never fall back to an untimed + // client: that would silently drop the request timeout and let a + // hung server stall the distribution loop indefinitely. Such a + // failure is a fatal environment problem, so surface it loudly. + .expect( + "reqwest client builder with request timeout must initialise; \ + a failure here indicates the platform TLS/transport backend is unavailable", + ); + Self { + http, + base_url: base_url.into().trim_end_matches('/').to_owned(), + bearer_token: Arc::from(bearer_token.into()), + } + } + + async fn fetch(&self) -> Result { + let url = format!("{}/allowlist/receivers", self.base_url); + Ok(self + .http + .get(url) + .bearer_auth(self.bearer_token.as_ref()) + .send() + .await? + .error_for_status()? + .json() + .await?) + } + + /// Long-polls the server allow-list: sends `since` (the last applied + /// version) and a `wait` budget, so the server holds the request until the + /// allow-list changes and returns the fresh snapshot — giving near-instant + /// approval propagation. The per-request timeout is overridden to exceed + /// `wait` so the held request is not aborted by the client's short default + /// timeout. A `since` of `None` requests an immediate snapshot (initial + /// fetch / resync). + async fn fetch_waiting( + &self, + since: Option, + wait: Duration, + ) -> Result { + // Built by hand (rather than reqwest's `query` builder) so no extra + // reqwest feature is required; both values are plain integers, so no + // percent-encoding is needed. + let url = match since { + Some(since) => format!( + "{}/allowlist/receivers?wait={}&since={since}", + self.base_url, + wait.as_secs() + ), + None => format!( + "{}/allowlist/receivers?wait={}", + self.base_url, + wait.as_secs() + ), + }; + Ok(self + .http + .get(url) + .bearer_auth(self.bearer_token.as_ref()) + // Allow the response to arrive any time within the server hold plus + // a transport margin, regardless of the client's short default. + .timeout(wait + Duration::from_secs(10)) + .send() + .await? + .error_for_status()? + .json() + .await?) + } +} + +/// HTTP client for registering the forwarder and pushing its stream catalog. +#[derive(Clone)] +pub struct ServerCatalogClient { + http: reqwest::Client, + base_url: String, + bearer_token: Arc, +} + +impl std::fmt::Debug for ServerCatalogClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ServerCatalogClient") + .field("base_url", &self.base_url) + .field("bearer_token", &"") + .finish() + } +} + +impl ServerCatalogClient { + /// Builds a client with the default server request timeout. + #[must_use] + pub fn new(base_url: impl Into, bearer_token: impl Into) -> Self { + Self::with_timeout(base_url, bearer_token, DEFAULT_ALLOWLIST_REQUEST_TIMEOUT) + } + + /// Builds a client that bounds every registration/catalog request. + #[must_use] + pub fn with_timeout( + base_url: impl Into, + bearer_token: impl Into, + request_timeout: Duration, + ) -> Self { + let http = reqwest::Client::builder() + .timeout(request_timeout) + .build() + .expect( + "reqwest client builder with request timeout must initialise; \ + a failure here indicates the platform TLS/transport backend is unavailable", + ); + Self { + http, + base_url: base_url.into().trim_end_matches('/').to_owned(), + bearer_token: Arc::from(bearer_token.into()), + } + } + + /// Bootstrap this forwarder against the server `/register` endpoint using the + /// configured bearer (an enrollment voucher on first boot, or the device's + /// own token for same-endpoint recovery), returning the server-minted + /// per-device token. The caller persists it and uses it for all later calls. + pub async fn bootstrap(&self, endpoint_id: &str) -> Result { + let url = format!("{}/register", self.base_url); + let response = self + .http + .post(url) + .bearer_auth(self.bearer_token.as_ref()) + .json(&RegisterRequest { + endpoint_id: endpoint_id.to_owned(), + device_kind: "forwarder".to_owned(), + display_name: None, + }) + .send() + .await? + .error_for_status()? + .json::() + .await?; + response + .device_token + .ok_or(CatalogPushError::MissingMintedToken) + } + + /// Fetches this forwarder's own stored stream catalog from the server + /// (`GET /forwarder/catalog`, self-scoped by the device token). + /// + /// Used at startup to restore per-stream epoch/next_seq high-water after + /// local journal loss. An older server without this route returns 404, + /// which surfaces as an `Err` here — callers treat any error as + /// "registry unavailable". + pub async fn fetch_own_catalog(&self) -> Result, CatalogPushError> { + let url = format!("{}/forwarder/catalog", self.base_url); + let response = self + .http + .get(url) + .bearer_auth(self.bearer_token.as_ref()) + .send() + .await? + .error_for_status()? + .json::() + .await?; + Ok(response.streams) + } + + /// Pushes this forwarder's latest identity and stream catalog. + pub async fn push_catalog(&self, catalog: &ForwarderCatalog) -> Result<(), CatalogPushError> { + let url = format!("{}/forwarder/catalog", self.base_url); + self.http + .post(url) + .bearer_auth(self.bearer_token.as_ref()) + .json(catalog) + .send() + .await? + .error_for_status()?; + Ok(()) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum CatalogPushError { + // `reqwest::Error`'s `Display` covers URL/status/transport but never + // request headers, so the bearer token cannot leak here. + #[error("server catalog request failed: {0}")] + Http(#[from] reqwest::Error), + #[error("server /register did not return a minted device token")] + MissingMintedToken, +} + +#[derive(Debug, thiserror::Error)] +pub enum AllowListRefreshError { + // `reqwest::Error`'s `Display` covers the failing URL and transport cause + // but never request headers, so the bearer token cannot leak here. + #[error("server allow-list request failed: {0}")] + Http(#[from] reqwest::Error), + #[error("failed to apply receiver allow-list update: {0}")] + Apply(#[from] io::Error), +} + +/// Fetches the current server allow-list and applies it to `allow_list`. +pub async fn fetch_and_apply_once( + client: &ServerAllowListClient, + allow_list: &AllowList, +) -> Result, AllowListRefreshError> { + let update = client.fetch().await?; + apply_receiver_update(allow_list, update) +} + +/// Applies a pushed or fetched wire-format receiver allow-list update. +pub fn apply_receiver_update( + allow_list: &AllowList, + update: ReceiverAllowListUpdate, +) -> Result, AllowListRefreshError> { + let allowed = parse_update(update); + Ok(allow_list.apply_update(allowed)?) +} + +/// In-flight poll-fetch future held across `select!` iterations so pushed +/// updates stay responsive while a slow poll request is outstanding. +/// +/// The poll *fetches only*; it does not apply. Applying is deferred to the +/// distribution loop so a snapshot captured before a newer pushed update can be +/// discarded instead of clobbering the newer state (see the generation guard in +/// [`run_allowlist_distribution`]). +type PollFetch = std::pin::Pin< + Box< + dyn std::future::Future> + + Send, + >, +>; + +/// Keeps `allow_list` fresh from startup fetches, pushed snapshots, and polling. +pub async fn run_allowlist_distribution( + allow_list: AllowList, + client: ServerAllowListClient, + mut pushed_updates: mpsc::Receiver, + poll_interval: Duration, +) { + if let Err(err) = fetch_and_apply_once(&client, &allow_list).await { + tracing::warn!(error = %err, "initial receiver allow-list refresh failed"); + } + + let poll_interval = if poll_interval.is_zero() { + DEFAULT_ALLOWLIST_POLL_INTERVAL + } else { + poll_interval + }; + let mut ticker = tokio::time::interval(poll_interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + ticker.tick().await; + let mut push_closed = false; + // An in-flight poll fetch is held here rather than awaited inline so pushed + // updates stay responsive even while a slow (or timing-out) poll request is + // outstanding. At most one poll fetch runs at a time; a tick that lands + // while one is still in flight is dropped (the next tick re-polls). + let mut poll_fetch: Option = None; + // Monotonic count of applied updates. Bumped on every successful apply + // (pushed or polled). A poll captures this when it begins fetching; if the + // count has moved by the time the fetch completes, a newer update applied + // while the poll was in flight, so the (now stale) poll snapshot is + // discarded rather than overwriting the newer state. Without this guard a + // slow poll holding a pre-revocation snapshot could re-authorize a receiver + // that a pushed revocation had already removed. + let mut generation: u64 = 0; + let mut poll_generation: u64 = 0; + + loop { + tokio::select! { + update = pushed_updates.recv(), if !push_closed => { + match update { + Some(update) => { + match apply_receiver_update(&allow_list, update) { + Ok(_) => generation = generation.wrapping_add(1), + Err(err) => tracing::warn!( + error = %err, + "pushed receiver allow-list update failed", + ), + } + } + None => push_closed = true, + } + } + _ = ticker.tick() => { + if poll_fetch.is_none() { + let client = client.clone(); + poll_generation = generation; + poll_fetch = Some(Box::pin(async move { client.fetch().await })); + } + } + result = async { poll_fetch.as_mut().expect("poll fetch present").await }, if poll_fetch.is_some() => { + poll_fetch = None; + match result { + Ok(update) => { + if generation == poll_generation { + match apply_receiver_update(&allow_list, update) { + Ok(_) => generation = generation.wrapping_add(1), + Err(err) => tracing::warn!( + error = %err, + "polled receiver allow-list refresh failed", + ), + } + } else { + tracing::debug!( + "discarding stale polled receiver allow-list snapshot \ + superseded by a newer update", + ); + } + } + Err(err) => tracing::warn!( + error = %err, + "polled receiver allow-list refresh failed", + ), + } + } + } + } +} + +/// Long-poll the server for receiver allow-list changes and forward each fresh +/// snapshot into `push_tx`, which [`run_allowlist_distribution`] applies. +/// +/// This is the push transport that complements the periodic poll backstop: an +/// approval on the server releases the held request within milliseconds, so a +/// newly approved receiver is admitted almost immediately instead of waiting +/// up to a full poll interval. Loops until the receiving distribution loop is +/// gone (the channel send fails). Server errors back off and retry; the +/// distribution loop's polling keeps the allow-list correct meanwhile. +pub async fn run_allowlist_push_subscription( + client: ServerAllowListClient, + push_tx: mpsc::Sender, + hold: Duration, +) { + // `None` requests an immediate snapshot; afterwards we echo the server's + // version as `since` to long-poll for the *next* change. + let mut since: Option = None; + let mut backoff = ALLOWLIST_PUSH_INITIAL_BACKOFF; + + loop { + let started = tokio::time::Instant::now(); + match client.fetch_waiting(since, hold).await { + Ok(update) => { + backoff = ALLOWLIST_PUSH_INITIAL_BACKOFF; + let version = update.version; + let changed = since != Some(version); + since = Some(version); + if changed { + // A real change (or the initial snapshot): hand it to the + // distribution loop. A closed channel means that loop is + // gone, so this subscription has no consumer left. + if push_tx.send(update).await.is_err() { + return; + } + } else { + // No change within the hold. Pace the next request to the + // intended hold window so a non-holding or older server + // (which returns immediately and never sets a moving + // version) is not hammered; a compliant server already + // consumed ~`hold`, so this adds nothing. + tokio::time::sleep_until(started + hold).await; + } + } + Err(err) => { + tracing::warn!(error = %err, "receiver allow-list push subscription failed; retrying"); + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(ALLOWLIST_PUSH_MAX_BACKOFF); + } + } + } +} + +fn parse_update(update: ReceiverAllowListUpdate) -> Vec { + update + .receiver_endpoint_ids + .into_iter() + .filter_map(|endpoint_id| match EndpointId::from_str(&endpoint_id) { + Ok(node_id) => Some(node_id), + Err(error) => { + tracing::warn!( + endpoint_id = %endpoint_id, + error = %error, + "skipping malformed receiver endpoint id in allow-list update", + ); + None + } + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + use axum::{ + Json, Router, + extract::State, + http::{HeaderMap, StatusCode, header::AUTHORIZATION}, + response::{IntoResponse, Response}, + routing::{get, post}, + }; + use rt_iroh::EndpointBuilder; + use tokio::sync::{Mutex as TokioMutex, watch}; + + type BoxError = Box; + type TestResult = Result<(), BoxError>; + + #[derive(Clone)] + struct TestAllowListServerState { + bearer_token: &'static str, + receiver_endpoint_ids: Arc>>, + // Retained count of completed authorized fetches. Unlike a `Notify`, + // a `watch` holds its latest value, so a fetch that completes before a + // waiter starts observing is not lost — letting tests deterministically + // sequence "initial fetch happened" against later poll fetches. + fetches: watch::Sender, + } + + async fn test_allowlist_handler( + State(state): State, + headers: HeaderMap, + ) -> Response { + let authorized = headers + .get(AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value == format!("Bearer {}", state.bearer_token)); + if !authorized { + return StatusCode::UNAUTHORIZED.into_response(); + } + + let receiver_endpoint_ids = state.receiver_endpoint_ids.lock().await.clone(); + // Count the fetch only after the current id set has been read, so an + // observed count of N guarantees N fetches each saw the id set in force + // at their read. + state.fetches.send_modify(|count| *count += 1); + Json(serde_json::json!({ "receiver_endpoint_ids": receiver_endpoint_ids })).into_response() + } + + async fn spawn_test_allowlist_server( + receiver_endpoint_ids: Arc>>, + ) -> Result<(String, watch::Receiver, tokio::task::JoinHandle<()>), BoxError> { + let (fetches, fetches_rx) = watch::channel(0u64); + let app = Router::new() + .route("/allowlist/receivers", get(test_allowlist_handler)) + .with_state(TestAllowListServerState { + bearer_token: "thin-secret", + receiver_endpoint_ids, + fetches, + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let url = format!("http://{}", listener.local_addr()?); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + Ok((url, fetches_rx, server)) + } + + #[derive(Clone)] + struct TestCatalogServerState { + bearer_token: &'static str, + received: Arc>>, + } + + async fn test_catalog_handler( + State(state): State, + headers: HeaderMap, + Json(body): Json, + ) -> Response { + let authorized = headers + .get(AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value == format!("Bearer {}", state.bearer_token)); + if !authorized { + return StatusCode::UNAUTHORIZED.into_response(); + } + *state.received.lock().await = Some((headers.clone(), body)); + StatusCode::OK.into_response() + } + + /// `/register` mock: records the request and returns a minted device token. + async fn test_register_handler( + State(state): State, + headers: HeaderMap, + Json(body): Json, + ) -> Response { + let authorized = headers + .get(AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value == format!("Bearer {}", state.bearer_token)); + if !authorized { + return StatusCode::UNAUTHORIZED.into_response(); + } + *state.received.lock().await = Some((headers.clone(), body)); + Json(serde_json::json!({ + "endpoint_id": "fwd-node-1", + "device_kind": "forwarder", + "approval_state": "pending", + "device_token": "rtk_minted_secret" + })) + .into_response() + } + + #[tokio::test] + async fn catalog_client_posts_registration_and_catalog_with_bearer() -> TestResult { + let received = Arc::new(TokioMutex::new(None)); + let app = Router::new() + .route("/register", post(test_register_handler)) + .route("/forwarder/catalog", post(test_catalog_handler)) + .with_state(TestCatalogServerState { + bearer_token: "thin-secret", + received: received.clone(), + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let base_url = format!("http://{}", listener.local_addr()?); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + + let client = ServerCatalogClient::new(base_url, "thin-secret"); + let minted = client + .bootstrap("fwd-node-1") + .await + .expect("bootstrap request succeeds"); + assert_eq!(minted, "rtk_minted_secret"); + let (headers, register_body) = received + .lock() + .await + .take() + .expect("server captured registration request"); + assert_eq!( + headers.get(AUTHORIZATION).unwrap().to_str().unwrap(), + "Bearer thin-secret" + ); + assert_eq!(register_body["endpoint_id"], "fwd-node-1"); + assert_eq!(register_body["device_kind"], "forwarder"); + + client + .push_catalog(&ForwarderCatalog { + endpoint_id: "fwd-node-1".to_owned(), + display_name: Some("Start Line".to_owned()), + direct_addrs: vec!["127.0.0.1:12345".to_owned()], + streams: vec![ForwarderCatalogStream { + stream_id: "reader-a".to_owned(), + epoch: 3, + next_seq: 42, + }], + }) + .await + .expect("catalog request succeeds"); + let (headers, catalog_body) = received + .lock() + .await + .take() + .expect("server captured catalog request"); + assert_eq!( + headers.get(AUTHORIZATION).unwrap().to_str().unwrap(), + "Bearer thin-secret" + ); + assert_eq!(catalog_body["endpoint_id"], "fwd-node-1"); + assert_eq!(catalog_body["display_name"], "Start Line"); + assert_eq!( + catalog_body["direct_addrs"], + serde_json::json!(["127.0.0.1:12345"]) + ); + assert_eq!(catalog_body["streams"][0]["stream_id"], "reader-a"); + assert_eq!(catalog_body["streams"][0]["epoch"], 3); + assert_eq!(catalog_body["streams"][0]["next_seq"], 42); + + server.abort(); + Ok(()) + } + + #[tokio::test] + async fn approved_receiver_appears_in_forwarder_list() -> TestResult { + let receiver = EndpointBuilder::test([50; 32]).bind().await?; + let receiver_endpoint_ids = Arc::new(TokioMutex::new(vec![receiver.endpoint_id().to_string()])); + let (base_url, _fetches, server) = + spawn_test_allowlist_server(receiver_endpoint_ids).await?; + let client = ServerAllowListClient::new(base_url, "thin-secret"); + let allow = AllowList::default(); + + fetch_and_apply_once(&client, &allow).await?; + + assert!(allow.contains(&receiver.endpoint_id())); + server.abort(); + receiver.close().await; + Ok(()) + } + + #[tokio::test] + async fn revoke_propagates() -> TestResult { + let receiver = EndpointBuilder::test([51; 32]).bind().await?; + let allow = AllowList::new([receiver.endpoint_id()]); + let receiver_endpoint_ids = Arc::new(TokioMutex::new(vec![receiver.endpoint_id().to_string()])); + let (base_url, _fetches, server) = + spawn_test_allowlist_server(receiver_endpoint_ids).await?; + let client = ServerAllowListClient::new(base_url, "thin-secret"); + let (tx, rx) = mpsc::channel(1); + let sync = tokio::spawn(run_allowlist_distribution( + allow.clone(), + client, + rx, + Duration::from_secs(60), + )); + + tx.send(ReceiverAllowListUpdate::replace(Vec::new())) + .await?; + tokio::time::timeout(Duration::from_secs(5), async { + while allow.contains(&receiver.endpoint_id()) { + tokio::task::yield_now().await; + } + }) + .await?; + + assert!(!allow.contains(&receiver.endpoint_id())); + sync.abort(); + server.abort(); + receiver.close().await; + Ok(()) + } + + #[tokio::test] + async fn poll_backstop_refreshes() -> TestResult { + let receiver = EndpointBuilder::test([52; 32]).bind().await?; + let receiver_endpoint_ids = Arc::new(TokioMutex::new(Vec::new())); + let (base_url, mut fetches, server) = + spawn_test_allowlist_server(receiver_endpoint_ids.clone()).await?; + let client = ServerAllowListClient::new(base_url, "thin-secret"); + let allow = AllowList::default(); + let (_tx, rx) = mpsc::channel(1); + // Short poll interval with real time: the backstop must re-fetch on its + // own without any pushed update. + let sync = tokio::spawn(run_allowlist_distribution( + allow.clone(), + client, + rx, + Duration::from_millis(50), + )); + + // Wait (on retained state) until the initial fetch has read the empty + // set, so the receiver can only be admitted by a later poll fetch. + tokio::time::timeout( + Duration::from_secs(5), + fetches.wait_for(|&count| count >= 1), + ) + .await??; + assert!(!allow.contains(&receiver.endpoint_id())); + + // Publish the receiver; only the polling backstop can pick this up. + *receiver_endpoint_ids.lock().await = vec![receiver.endpoint_id().to_string()]; + + tokio::time::timeout(Duration::from_secs(5), async { + while !allow.contains(&receiver.endpoint_id()) { + tokio::task::yield_now().await; + } + }) + .await?; + + assert!(allow.contains(&receiver.endpoint_id())); + sync.abort(); + server.abort(); + receiver.close().await; + Ok(()) + } + + #[tokio::test] + async fn stale_poll_does_not_override_newer_revocation() -> TestResult { + use std::sync::atomic::{AtomicU64, Ordering}; + + let receiver = EndpointBuilder::test([55; 32]).bind().await?; + // Start empty; the initial fetch admits the receiver from the old list. + let allow = AllowList::default(); + + #[derive(Clone)] + struct StallState { + // Old, pre-revocation list returned by every fetch: it still + // authorizes the receiver. + old_list: Vec, + // 1-based count of requests received by the server. + requests: Arc, + // Broadcasts the latest request count so the test can sequence on it. + observed: watch::Sender, + // Releases exactly one gated (poll) request when notified once. + release: Arc, + } + + async fn stall_handler(State(state): State) -> Response { + let n = state.requests.fetch_add(1, Ordering::SeqCst) + 1; + state.observed.send_modify(|count| *count = (*count).max(n)); + // The initial fetch (request 1) returns immediately so the receiver + // is admitted. Every later (poll) request blocks until explicitly + // released, so the test controls exactly when a poll completes. + if n >= 2 { + state.release.notified().await; + } + Json(serde_json::json!({ "receiver_endpoint_ids": state.old_list })).into_response() + } + + let (observed_tx, mut observed_rx) = watch::channel(0u64); + let release = Arc::new(tokio::sync::Notify::new()); + let app = Router::new() + .route("/allowlist/receivers", get(stall_handler)) + .with_state(StallState { + old_list: vec![receiver.endpoint_id().to_string()], + requests: Arc::new(AtomicU64::new(0)), + observed: observed_tx, + release: release.clone(), + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let base_url = format!("http://{}", listener.local_addr()?); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + + let client = ServerAllowListClient::new(base_url, "thin-secret"); + let (tx, rx) = mpsc::channel(1); + // Short poll interval so a poll fetch starts promptly after the initial + // fetch admits the receiver. + let sync = tokio::spawn(run_allowlist_distribution( + allow.clone(), + client, + rx, + Duration::from_millis(50), + )); + + // Initial fetch (request 1) admits the receiver from the old list. + tokio::time::timeout(Duration::from_secs(5), async { + while !allow.contains(&receiver.endpoint_id()) { + tokio::task::yield_now().await; + } + }) + .await?; + + // A poll (request 2) is now in flight and blocked on the release gate. + tokio::time::timeout( + Duration::from_secs(5), + observed_rx.wait_for(|&count| count >= 2), + ) + .await??; + + // Apply a pushed revocation while the poll fetch is still in flight. + tx.send(ReceiverAllowListUpdate::replace(Vec::new())) + .await?; + tokio::time::timeout(Duration::from_secs(5), async { + while allow.contains(&receiver.endpoint_id()) { + tokio::task::yield_now().await; + } + }) + .await?; + + // Let the stale poll (request 2) complete with the old, pre-revocation + // list. notify_one releases exactly one waiter, so request 3 stays + // blocked and cannot itself re-admit the receiver. + release.notify_one(); + + // Request 3 starting proves the stale poll result has been fully + // processed by the distribution loop (the poll slot freed, a new tick + // fired). Request 3 then blocks on the gate without responding. + tokio::time::timeout( + Duration::from_secs(5), + observed_rx.wait_for(|&count| count >= 3), + ) + .await??; + + assert!( + !allow.contains(&receiver.endpoint_id()), + "a stale in-flight poll must not re-authorize a receiver revoked while it was outstanding" + ); + + sync.abort(); + server.abort(); + receiver.close().await; + Ok(()) + } + + #[derive(Clone)] + struct PushServerState { + bearer_token: &'static str, + ids: Arc>>, + version: Arc, + changed: Arc, + } + + #[derive(serde::Deserialize)] + struct PushQuery { + since: Option, + wait: Option, + } + + /// Mock long-poll allow-list endpoint mirroring the real server: holds the + /// request when the caller's `since` matches the current version, releasing + /// on a version bump, and always echoes the current `version`. + async fn push_allowlist_handler( + State(state): State, + headers: HeaderMap, + axum::extract::Query(query): axum::extract::Query, + ) -> Response { + let authorized = headers + .get(AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value == format!("Bearer {}", state.bearer_token)); + if !authorized { + return StatusCode::UNAUTHORIZED.into_response(); + } + let current = state.version.load(std::sync::atomic::Ordering::SeqCst); + let wait = query.wait.unwrap_or(0); + if wait > 0 && query.since == Some(current) { + let _ = tokio::time::timeout(Duration::from_secs(wait), state.changed.notified()).await; + } + let version = state.version.load(std::sync::atomic::Ordering::SeqCst); + let ids = state.ids.lock().await.clone(); + Json(serde_json::json!({ "receiver_endpoint_ids": ids, "version": version })) + .into_response() + } + + #[tokio::test] + async fn push_subscription_delivers_initial_snapshot_then_releases_on_bump() -> TestResult { + let ids = Arc::new(TokioMutex::new(Vec::::new())); + let version = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let changed = Arc::new(tokio::sync::Notify::new()); + let app = Router::new() + .route("/allowlist/receivers", get(push_allowlist_handler)) + .with_state(PushServerState { + bearer_token: "thin-secret", + ids: ids.clone(), + version: version.clone(), + changed: changed.clone(), + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let base_url = format!("http://{}", listener.local_addr()?); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + + let client = ServerAllowListClient::new(base_url, "thin-secret"); + let (tx, mut rx) = mpsc::channel(8); + let sub = tokio::spawn(run_allowlist_push_subscription( + client, + tx, + Duration::from_secs(5), + )); + + // The initial request omits `since`, so the server returns immediately + // with the empty snapshot at version 0. + let first = tokio::time::timeout(Duration::from_secs(2), rx.recv()) + .await? + .expect("initial snapshot delivered"); + assert_eq!(first.version, 0); + assert!(first.receiver_endpoint_ids.is_empty()); + + // The subscription is now held at version 0. An approval bumps the + // version and must release it within milliseconds with the new set. + *ids.lock().await = vec!["receiver-1".to_owned()]; + version.store(1, std::sync::atomic::Ordering::SeqCst); + changed.notify_waiters(); + + let second = tokio::time::timeout(Duration::from_secs(2), rx.recv()) + .await? + .expect("approval releases the held long-poll"); + assert_eq!(second.version, 1); + assert_eq!(second.receiver_endpoint_ids, vec!["receiver-1".to_owned()]); + + sub.abort(); + server.abort(); + Ok(()) + } + + #[tokio::test] + async fn debug_redacts_bearer_token() { + let client = ServerAllowListClient::new("http://thin.example", "super-secret-token"); + let rendered = format!("{client:?}"); + assert!( + !rendered.contains("super-secret-token"), + "bearer token must never appear in Debug output: {rendered}" + ); + assert!(rendered.contains("")); + } + + #[tokio::test] + async fn hung_server_request_times_out() -> TestResult { + // A handler that never responds: only the client request timeout can + // unblock the fetch. + async fn hang() -> Response { + std::future::pending::<()>().await; + StatusCode::OK.into_response() + } + let app = Router::new().route("/allowlist/receivers", get(hang)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let base_url = format!("http://{}", listener.local_addr()?); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + + let client = ServerAllowListClient::with_timeout( + base_url, + "thin-secret", + Duration::from_millis(200), + ); + let allow = AllowList::default(); + + // Outer guard far exceeds the request timeout: if the request timeout + // works, the fetch returns an error well before this elapses. + let result = tokio::time::timeout( + Duration::from_secs(5), + fetch_and_apply_once(&client, &allow), + ) + .await?; + assert!( + matches!(result, Err(AllowListRefreshError::Http(_))), + "hung request must surface as a bounded HTTP error, got {result:?}" + ); + + server.abort(); + Ok(()) + } + + #[tokio::test] + async fn malformed_active_id_does_not_block_valid_revocation() -> TestResult { + let retained = EndpointBuilder::test([53; 32]).bind().await?; + let revoked = EndpointBuilder::test([54; 32]).bind().await?; + let allow = AllowList::new([retained.endpoint_id(), revoked.endpoint_id()]); + + // The server can contain arbitrary endpoint_id text from receiver + // registration. An invalid active id must be ignored, while the valid + // omission of `revoked` still removes its authorization. + let receiver_endpoint_ids = Arc::new(TokioMutex::new(vec![ + retained.endpoint_id().to_string(), + "not-a-valid-endpoint-id".to_owned(), + ])); + let (base_url, _fetches, server) = + spawn_test_allowlist_server(receiver_endpoint_ids).await?; + let client = ServerAllowListClient::new(base_url, "thin-secret"); + + let result = fetch_and_apply_once(&client, &allow).await?; + assert_eq!(result, vec![revoked.endpoint_id()]); + assert!( + allow.contains(&retained.endpoint_id()), + "valid active receiver must remain authorized" + ); + assert!( + !allow.contains(&revoked.endpoint_id()), + "valid revocation must apply even when the response also contains a malformed id" + ); + + server.abort(); + retained.close().await; + revoked.close().await; + Ok(()) + } +} From 7ccffa34dc294f4b545df171c61bcaad7713dbb5 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 17:01:03 -0400 Subject: [PATCH 27/38] fix(forwarder): retry device-token bootstrap in-process and re-poll after failed allow-list applies --- services/forwarder/src/p2p/allowlist.rs | 5 +- services/forwarder/src/p2p/mod.rs | 150 +++++++++++++++++++- services/forwarder/src/p2p/server_client.rs | 116 +++++++++++++-- 3 files changed, 257 insertions(+), 14 deletions(-) diff --git a/services/forwarder/src/p2p/allowlist.rs b/services/forwarder/src/p2p/allowlist.rs index adb33626..7577edf5 100644 --- a/services/forwarder/src/p2p/allowlist.rs +++ b/services/forwarder/src/p2p/allowlist.rs @@ -147,8 +147,7 @@ impl AllowList { let effective: HashSet = new_allowed.union(&state.pinned).copied().collect(); - let revoked: Vec = - state.allowed.difference(&effective).copied().collect(); + let revoked: Vec = state.allowed.difference(&effective).copied().collect(); state.allowed = effective; let mut connections_to_close = Vec::new(); for endpoint_id in &revoked { @@ -272,7 +271,7 @@ mod tests { use std::time::Duration; - use rt_iroh::{Endpoint, EndpointBuilder, EndpointAddr}; + use rt_iroh::{Endpoint, EndpointAddr, EndpointBuilder}; type BoxError = Box; type TestResult = Result<(), BoxError>; diff --git a/services/forwarder/src/p2p/mod.rs b/services/forwarder/src/p2p/mod.rs index 91db44ef..da319d70 100644 --- a/services/forwarder/src/p2p/mod.rs +++ b/services/forwarder/src/p2p/mod.rs @@ -59,11 +59,17 @@ pub use server_client::{ const DEFAULT_P2P_SECRET_KEY_PATH: &str = "/var/lib/rusty-timer/p2p-secret.key"; const DEFAULT_FORWARDER_CATALOG_PUSH_INTERVAL: Duration = Duration::from_secs(30); - /// Backoff bounds for retrying the first-boot device-token bootstrap when the -/// server is unreachable (e.g. the forwarder started before the network came up). +/// server is unreachable (e.g. the forwarder started before the network came +/// up). Test builds use tiny delays so the retry loop is exercised quickly. +#[cfg(not(test))] const BOOTSTRAP_RETRY_INITIAL: Duration = Duration::from_secs(5); +#[cfg(test)] +const BOOTSTRAP_RETRY_INITIAL: Duration = Duration::from_millis(10); +#[cfg(not(test))] const BOOTSTRAP_RETRY_MAX: Duration = Duration::from_secs(300); +#[cfg(test)] +const BOOTSTRAP_RETRY_MAX: Duration = Duration::from_millis(40); /// Running forwarder P2P server tasks. #[derive(Debug)] @@ -595,6 +601,13 @@ mod tests { use crate::status_http::{StatusConfig, StatusServer}; use crate::status_store::SubsystemStatus; use crate::storage::journal::Journal; + use axum::{ + Json, Router, + extract::State, + http::{HeaderMap, StatusCode, header::AUTHORIZATION}, + response::{IntoResponse, Response}, + routing::{get, post}, + }; use rt_iroh::EndpointBuilder; use rt_p2p_protocol::{ ControlC2F, ControlF2C, DataC2F, DataF2C, DataSubscribe, SubscribeMode, control_c2f, @@ -678,6 +691,139 @@ mod tests { Ok(()) } + #[derive(Clone)] + struct BootstrapRetryServerState { + register_attempts: Arc, + catalog_pushes: tokio::sync::watch::Sender, + } + + async fn bootstrap_retry_register_handler( + State(state): State, + headers: HeaderMap, + Json(_body): Json, + ) -> Response { + let authorized = headers + .get(AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value == "Bearer thin-voucher"); + if !authorized { + return StatusCode::UNAUTHORIZED.into_response(); + } + + let attempt = state + .register_attempts + .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + + 1; + if attempt <= 2 { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + + Json(serde_json::json!({ + "endpoint_id": "fwd-node-1", + "device_kind": "forwarder", + "approval_state": "pending", + "device_token": "rtk_minted_secret" + })) + .into_response() + } + + async fn bootstrap_retry_catalog_handler( + State(state): State, + headers: HeaderMap, + Json(_body): Json, + ) -> Response { + let authorized = headers + .get(AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value == "Bearer rtk_minted_secret"); + if !authorized { + return StatusCode::UNAUTHORIZED.into_response(); + } + + state.catalog_pushes.send_modify(|count| *count += 1); + StatusCode::OK.into_response() + } + + async fn bootstrap_retry_allowlist_handler(headers: HeaderMap) -> Response { + let authorized = headers + .get(AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value == "Bearer rtk_minted_secret"); + if !authorized { + return StatusCode::UNAUTHORIZED.into_response(); + } + Json(serde_json::json!({ "receiver_endpoint_ids": [] })).into_response() + } + + #[tokio::test] + async fn server_integration_retries_bootstrap_until_catalog_starts() -> TestResult { + let receiver = EndpointBuilder::test([44; 32]).bind().await?; + let dir = tempfile::tempdir()?; + let journal_path = dir.path().join("journal.sqlite3"); + let journal = Arc::new(Mutex::new(Journal::open(&journal_path)?)); + let token_file = dir.path().join("server-token"); + std::fs::write(&token_file, "thin-voucher\n")?; + let device_token_file = dir.path().join("p2p-device-token"); + let (catalog_pushes, mut catalog_pushes_rx) = tokio::sync::watch::channel(0u64); + let state = BootstrapRetryServerState { + register_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)), + catalog_pushes, + }; + let app = Router::new() + .route("/register", post(bootstrap_retry_register_handler)) + .route("/forwarder/catalog", post(bootstrap_retry_catalog_handler)) + .route( + "/allowlist/receivers", + get(bootstrap_retry_allowlist_handler), + ) + .with_state(state.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let base_url = format!("http://{}", listener.local_addr()?); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + + let mut config = p2p_config(receiver.endpoint_id().to_string()); + config.server_url = Some(base_url); + config.server_token_file = Some(token_file.to_string_lossy().into_owned()); + config.device_token_file = Some(device_token_file.to_string_lossy().into_owned()); + config.allowlist_request_timeout_secs = 1; + let runtime = start_forwarder_p2p( + &config, + &journal_path, + Arc::clone(&journal), + &["10.0.0.5:10000".to_owned()], + None, + status_feed().await?, + Arc::new(NoopRemoteConfigHandler), + Arc::new(NoopReaderControlHandler), + ) + .await? + .expect("p2p enabled"); + + tokio::time::timeout( + Duration::from_secs(2), + catalog_pushes_rx.wait_for(|&count| count >= 1), + ) + .await??; + assert_eq!( + state + .register_attempts + .load(std::sync::atomic::Ordering::SeqCst), + 3, + "bootstrap should retry in-process after initial failures" + ); + assert_eq!( + read_device_token(&device_token_file)?.as_deref(), + Some("rtk_minted_secret") + ); + + runtime.shutdown().await; + server.abort(); + receiver.close().await; + Ok(()) + } + #[tokio::test] async fn startup_helper_binds_seeded_loopback_and_serves_control_plus_data() -> TestResult { let receiver = EndpointBuilder::test([43; 32]).bind().await?; diff --git a/services/forwarder/src/p2p/server_client.rs b/services/forwarder/src/p2p/server_client.rs index 5881d2fe..b36cf2e5 100644 --- a/services/forwarder/src/p2p/server_client.rs +++ b/services/forwarder/src/p2p/server_client.rs @@ -301,6 +301,14 @@ type PollFetch = std::pin::Pin< >, >; +fn start_poll_fetch(client: &ServerAllowListClient, generation: u64) -> (u64, PollFetch) { + let client = client.clone(); + ( + generation, + Box::pin(async move { client.fetch().await }) as PollFetch, + ) +} + /// Keeps `allow_list` fresh from startup fetches, pushed snapshots, and polling. pub async fn run_allowlist_distribution( allow_list: AllowList, @@ -326,6 +334,7 @@ pub async fn run_allowlist_distribution( // outstanding. At most one poll fetch runs at a time; a tick that lands // while one is still in flight is dropped (the next tick re-polls). let mut poll_fetch: Option = None; + let mut poll_retry_after_current = false; // Monotonic count of applied updates. Bumped on every successful apply // (pushed or polled). A poll captures this when it begins fetching; if the // count has moved by the time the fetch completes, a newer update applied @@ -343,10 +352,20 @@ pub async fn run_allowlist_distribution( Some(update) => { match apply_receiver_update(&allow_list, update) { Ok(_) => generation = generation.wrapping_add(1), - Err(err) => tracing::warn!( - error = %err, - "pushed receiver allow-list update failed", - ), + Err(err) => { + tracing::warn!( + error = %err, + "pushed receiver allow-list update failed", + ); + if poll_fetch.is_none() { + let (generation_at_start, fetch) = + start_poll_fetch(&client, generation); + poll_generation = generation_at_start; + poll_fetch = Some(fetch); + } else { + poll_retry_after_current = true; + } + } } } None => push_closed = true, @@ -354,9 +373,9 @@ pub async fn run_allowlist_distribution( } _ = ticker.tick() => { if poll_fetch.is_none() { - let client = client.clone(); - poll_generation = generation; - poll_fetch = Some(Box::pin(async move { client.fetch().await })); + let (generation_at_start, fetch) = start_poll_fetch(&client, generation); + poll_generation = generation_at_start; + poll_fetch = Some(fetch); } } result = async { poll_fetch.as_mut().expect("poll fetch present").await }, if poll_fetch.is_some() => { @@ -383,6 +402,12 @@ pub async fn run_allowlist_distribution( "polled receiver allow-list refresh failed", ), } + if poll_retry_after_current { + poll_retry_after_current = false; + let (generation_at_start, fetch) = start_poll_fetch(&client, generation); + poll_generation = generation_at_start; + poll_fetch = Some(fetch); + } } } } @@ -643,7 +668,8 @@ mod tests { #[tokio::test] async fn approved_receiver_appears_in_forwarder_list() -> TestResult { let receiver = EndpointBuilder::test([50; 32]).bind().await?; - let receiver_endpoint_ids = Arc::new(TokioMutex::new(vec![receiver.endpoint_id().to_string()])); + let receiver_endpoint_ids = + Arc::new(TokioMutex::new(vec![receiver.endpoint_id().to_string()])); let (base_url, _fetches, server) = spawn_test_allowlist_server(receiver_endpoint_ids).await?; let client = ServerAllowListClient::new(base_url, "thin-secret"); @@ -661,7 +687,8 @@ mod tests { async fn revoke_propagates() -> TestResult { let receiver = EndpointBuilder::test([51; 32]).bind().await?; let allow = AllowList::new([receiver.endpoint_id()]); - let receiver_endpoint_ids = Arc::new(TokioMutex::new(vec![receiver.endpoint_id().to_string()])); + let receiver_endpoint_ids = + Arc::new(TokioMutex::new(vec![receiver.endpoint_id().to_string()])); let (base_url, _fetches, server) = spawn_test_allowlist_server(receiver_endpoint_ids).await?; let client = ServerAllowListClient::new(base_url, "thin-secret"); @@ -733,6 +760,77 @@ mod tests { Ok(()) } + #[tokio::test] + async fn pushed_apply_failure_triggers_prompt_poll_retry() -> TestResult { + #[cfg(unix)] + use std::os::unix::fs::PermissionsExt as _; + + #[cfg(unix)] + struct PermissionGuard { + path: std::path::PathBuf, + mode: u32, + } + + #[cfg(unix)] + impl Drop for PermissionGuard { + fn drop(&mut self) { + let _ = std::fs::set_permissions( + &self.path, + std::fs::Permissions::from_mode(self.mode), + ); + } + } + + let dir = tempfile::tempdir()?; + let cache_dir = dir.path().join("cache"); + std::fs::create_dir(&cache_dir)?; + let cache_path = cache_dir.join("allowlist"); + let allow = AllowList::load(&cache_path)?; + + #[cfg(unix)] + let _guard = { + let original_mode = std::fs::metadata(&cache_dir)?.permissions().mode(); + std::fs::set_permissions(&cache_dir, std::fs::Permissions::from_mode(0o555))?; + PermissionGuard { + path: cache_dir.clone(), + mode: original_mode, + } + }; + + let receiver_endpoint_ids = Arc::new(TokioMutex::new(Vec::new())); + let (base_url, mut fetches, server) = + spawn_test_allowlist_server(receiver_endpoint_ids).await?; + let client = ServerAllowListClient::new(base_url, "thin-secret"); + let (tx, rx) = mpsc::channel(1); + let sync = tokio::spawn(run_allowlist_distribution( + allow, + client, + rx, + Duration::from_secs(5), + )); + + tokio::time::timeout( + Duration::from_secs(5), + fetches.wait_for(|&count| count >= 1), + ) + .await??; + + tx.send(ReceiverAllowListUpdate::replace(vec![ + "receiver-1".to_owned(), + ])) + .await?; + + tokio::time::timeout( + Duration::from_secs(1), + fetches.wait_for(|&count| count >= 2), + ) + .await??; + + sync.abort(); + server.abort(); + Ok(()) + } + #[tokio::test] async fn stale_poll_does_not_override_newer_revocation() -> TestResult { use std::sync::atomic::{AtomicU64, Ordering}; From 65caf07577a42d6c9e124bfdefa690fa642f2975 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 17:07:52 -0400 Subject: [PATCH 28/38] test(forwarder): gate permission-based retry test to unix; fix stale token-resolution doc --- services/forwarder/src/p2p/server_client.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/forwarder/src/p2p/server_client.rs b/services/forwarder/src/p2p/server_client.rs index b36cf2e5..95be7a09 100644 --- a/services/forwarder/src/p2p/server_client.rs +++ b/services/forwarder/src/p2p/server_client.rs @@ -760,6 +760,9 @@ mod tests { Ok(()) } + // Unix-only: the failure injection relies on a read-only cache directory, + // which does not deny writes on Windows. + #[cfg(unix)] #[tokio::test] async fn pushed_apply_failure_triggers_prompt_poll_retry() -> TestResult { #[cfg(unix)] From a22a21ded356b591ffa45e8b05c7d566d2375b13 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 17:18:42 -0400 Subject: [PATCH 29/38] fix(forwarder): serve current reader connectivity and generation in the P2P catalog at connect time --- services/forwarder/src/p2p/mod.rs | 268 +++++++++++++++++++++++++++--- 1 file changed, 243 insertions(+), 25 deletions(-) diff --git a/services/forwarder/src/p2p/mod.rs b/services/forwarder/src/p2p/mod.rs index da319d70..0dc5ad8c 100644 --- a/services/forwarder/src/p2p/mod.rs +++ b/services/forwarder/src/p2p/mod.rs @@ -31,11 +31,13 @@ use std::sync::Arc; use std::time::Duration; use crate::config::P2pConfig; -use crate::status_store::ForwarderStatusFeed; +use crate::status_store::{ + ForwarderStatusEvent, ForwarderStatusFeed, ForwarderStatusSnapshot, ReaderConnectionState, +}; use crate::storage::journal::Journal; use rt_iroh::{EndpointAddr, EndpointBuilder, EndpointId, RelayMode, SecretKey}; use rt_p2p_protocol::{StreamCatalog, StreamEntry}; -use tokio::sync::{Mutex, mpsc}; +use tokio::sync::{Mutex, broadcast, mpsc, watch}; use tokio::task::JoinHandle; pub use allowlist::AllowList; @@ -142,7 +144,19 @@ pub async fn start_forwarder_p2p( } } - let catalog = Arc::new(ReaderCatalog::new(reader_streams)); + // Seed the catalog from the status store's current reader states: main.rs + // initializes reader status (init_readers) before starting P2P, so the + // snapshot already knows every configured reader's connectivity. The + // returned delta receiver is handed to the catalog task so no state change + // between snapshot and task startup can be missed. + let (status_rx, status_snapshot) = status_feed.subscribe_and_snapshot().await; + let (catalog, catalog_task) = LiveReaderCatalog::start( + reader_streams, + status_feed.clone(), + &status_snapshot, + status_rx, + ); + let catalog = Arc::new(catalog); let endpoint = P2pEndpoint::bind_with_builder( endpoint_builder(config)?, allow_list.clone(), @@ -156,7 +170,10 @@ pub async fn start_forwarder_p2p( .with_reader_control(reader_control); let run_endpoint = endpoint.clone(); - let mut tasks = vec![tokio::spawn(async move { run_endpoint.run().await })]; + let mut tasks = vec![ + tokio::spawn(async move { run_endpoint.run().await }), + catalog_task, + ]; if let Some((base_url, voucher)) = server_credentials(config)? { let request_timeout = Duration::from_secs(config.allowlist_request_timeout_secs); @@ -272,37 +289,124 @@ pub enum P2pStartError { CatalogValueOutOfRange(#[from] TryFromIntError), } +/// Live [`CatalogProvider`]: serves the forwarder's advertised streams with +/// current per-reader connectivity. +/// +/// A background task (tracked in [`P2pRuntime`] tasks) observes the status +/// feed and updates `reader_connected` per entry, bumping `generation` +/// monotonically on every real connectivity change, so a receiver that +/// (re)connects sees fresh state and a changed `HelloOk.catalog_generation`. +/// Mid-connection catalog pushes are out of scope: peers observe changes at +/// connect time only. #[derive(Debug)] -struct ReaderCatalog { - catalog: StreamCatalog, +struct LiveReaderCatalog { + rx: watch::Receiver, } -impl ReaderCatalog { - fn new(reader_streams: &[String]) -> Self { - Self { - catalog: StreamCatalog { - generation: 1, - entries: reader_streams - .iter() - .map(|stream| StreamEntry { - stream_id: stream.as_bytes().to_vec(), - display_name: stream.clone(), - network_addr: stream.clone(), - reader_connected: true, - hardware_reader_id: stream.clone(), - }) - .collect(), - }, - } +impl LiveReaderCatalog { + /// Builds the initial catalog (generation 1) seeded from `snapshot` and + /// spawns the task that applies status-feed deltas. + fn start( + reader_streams: &[String], + feed: ForwarderStatusFeed, + snapshot: &ForwarderStatusSnapshot, + status_rx: broadcast::Receiver, + ) -> (Self, JoinHandle<()>) { + let initial = StreamCatalog { + generation: 1, + entries: reader_streams + .iter() + .map(|stream| StreamEntry { + stream_id: stream.as_bytes().to_vec(), + display_name: stream.clone(), + network_addr: stream.clone(), + reader_connected: snapshot_reader_connected(snapshot, stream), + hardware_reader_id: stream.clone(), + }) + .collect(), + }; + let (tx, rx) = watch::channel(initial); + let task = tokio::spawn(run_catalog_updates(feed, status_rx, tx)); + (Self { rx }, task) } } -impl CatalogProvider for ReaderCatalog { +impl CatalogProvider for LiveReaderCatalog { fn catalog(&self) -> StreamCatalog { - self.catalog.clone() + self.rx.borrow().clone() + } +} + +fn snapshot_reader_connected(snapshot: &ForwarderStatusSnapshot, stream: &str) -> bool { + snapshot + .readers + .iter() + .any(|(id, status)| id == stream && status.state == ReaderConnectionState::Connected) +} + +/// Applies reader connectivity deltas from the status feed to the catalog. +/// +/// Exits when the status feed closes; otherwise runs until aborted at P2P +/// shutdown. +async fn run_catalog_updates( + feed: ForwarderStatusFeed, + mut status_rx: broadcast::Receiver, + tx: watch::Sender, +) { + loop { + match status_rx.recv().await { + Ok(ForwarderStatusEvent::ReaderStatus { stream_id, status }) => { + apply_reader_connectivity( + &tx, + &stream_id, + status.state == ReaderConnectionState::Connected, + ); + } + Ok(_) => {} + Err(broadcast::error::RecvError::Lagged(skipped)) => { + tracing::warn!( + skipped, + "p2p: catalog status receiver lagged; reseeding from snapshot" + ); + // A dropped delta could be a connect/disconnect; re-subscribe + // atomically with a fresh snapshot so the catalog cannot stay + // stale. + let (new_rx, snapshot) = feed.subscribe_and_snapshot().await; + status_rx = new_rx; + for (stream_id, status) in &snapshot.readers { + apply_reader_connectivity( + &tx, + stream_id, + status.state == ReaderConnectionState::Connected, + ); + } + } + Err(broadcast::error::RecvError::Closed) => break, + } } } +/// Sets `reader_connected` for the entry matching `stream_id`, bumping the +/// catalog generation only on an actual change. Events for unknown streams or +/// with unchanged connectivity are no-ops (no generation bump). +fn apply_reader_connectivity(tx: &watch::Sender, stream_id: &str, connected: bool) { + tx.send_if_modified(|catalog| { + let Some(entry) = catalog + .entries + .iter_mut() + .find(|entry| entry.stream_id == stream_id.as_bytes()) + else { + return false; + }; + if entry.reader_connected == connected { + return false; + } + entry.reader_connected = connected; + catalog.generation += 1; + true + }); +} + fn endpoint_builder(config: &P2pConfig) -> Result { let secret_key = match config.secret_key_seed_hex.as_deref() { Some(seed) => SecretKey::from_bytes(&decode_seed(seed)?), @@ -936,6 +1040,120 @@ mod tests { Ok(()) } + /// Connects, performs the control-plane Hello handshake, and returns the + /// negotiated `HelloOk.catalog_generation` together with the served + /// `StreamCatalog` frame, then drops the connection. + async fn hello_and_catalog( + receiver: &rt_iroh::Endpoint, + forwarder_addr: &NodeAddr, + ) -> Result<(u64, StreamCatalog), BoxError> { + let connection = receiver.connect(forwarder_addr.clone()).await?; + let (mut control_send, mut control_recv) = connection.open_bi().await?; + control::write_frame( + &mut control_send, + &ControlC2F { + msg: Some(control_c2f::Msg::Hello(control::forwarder_hello())), + }, + ) + .await?; + let hello_generation = match control::read_frame::(&mut control_recv) + .await? + .msg + { + Some(control_f2c::Msg::HelloOk(hello_ok)) => hello_ok.catalog_generation, + other => return Err(format!("expected HelloOk, got {other:?}").into()), + }; + let catalog = match control::read_frame::(&mut control_recv) + .await? + .msg + { + Some(control_f2c::Msg::StreamCatalog(catalog)) => catalog, + other => return Err(format!("expected StreamCatalog, got {other:?}").into()), + }; + connection.close(0u32.into(), b"test done"); + Ok((hello_generation, catalog)) + } + + #[tokio::test] + async fn reconnect_serves_current_reader_connectivity_and_bumped_generation() -> TestResult { + let receiver = EndpointBuilder::test([63; 32]).bind().await?; + let dir = tempfile::tempdir()?; + let journal_path = dir.path().join("journal.sqlite3"); + let journal = Arc::new(Mutex::new(Journal::open(&journal_path)?)); + let stream_key = "10.0.0.5:10000"; + + // Reader is already Connected before P2P starts (main.rs initializes + // reader status before start_forwarder_p2p), so the initial catalog + // must be seeded from the status snapshot, not hardcoded. + let store = crate::status_store::StatusStore::new(SubsystemStatus::ready()); + store.init_readers(&[(stream_key.to_owned(), 10001)]).await; + store + .update_reader_state( + stream_key, + crate::status_store::ReaderConnectionState::Connected, + ) + .await; + + let runtime = start_forwarder_p2p( + &p2p_config(receiver.node_id().to_string()), + &journal_path, + Arc::clone(&journal), + &[stream_key.to_owned()], + None, + store.status_feed(), + Arc::new(NoopRemoteConfigHandler), + Arc::new(NoopReaderControlHandler), + ) + .await? + .expect("p2p enabled"); + let forwarder_addr = runtime.node_addr().await; + receiver.add_node_addr(forwarder_addr.clone())?; + + let (g1, catalog) = hello_and_catalog(&receiver, &forwarder_addr).await?; + assert_eq!( + g1, catalog.generation, + "HelloOk.catalog_generation must match the served StreamCatalog generation" + ); + assert!( + catalog.entries[0].reader_connected, + "initial catalog must reflect the reader's Connected state at p2p start" + ); + + store + .update_reader_state( + stream_key, + crate::status_store::ReaderConnectionState::Disconnected, + ) + .await; + + // The catalog task observes the status feed asynchronously: poll with + // fresh connections until the disconnect is visible at connect time. + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + loop { + let (g2, catalog) = hello_and_catalog(&receiver, &forwarder_addr).await?; + assert_eq!( + g2, catalog.generation, + "HelloOk.catalog_generation must match the served StreamCatalog generation" + ); + if g2 > g1 && !catalog.entries[0].reader_connected { + break; + } + if tokio::time::Instant::now() >= deadline { + return Err(format!( + "catalog never reflected reader disconnect: generation {g2} (initial {g1}), \ + reader_connected {}", + catalog.entries[0].reader_connected + ) + .into()); + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + + runtime.shutdown().await; + receiver.close().await; + Ok(()) + } + #[tokio::test] async fn freshly_configured_reader_subscribable_before_any_data() -> TestResult { let receiver = EndpointBuilder::test([62; 32]).bind().await?; From 8cab87422b58a6afc1a0eb7d389b60877498cb11 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 17:44:23 -0400 Subject: [PATCH 30/38] fix(forwarder): serve cached server reachability from the status endpoint --- services/forwarder/src/lib.rs | 1 + services/forwarder/src/main.rs | 11 + services/forwarder/src/server_status_task.rs | 336 +++++++++++++++++++ services/forwarder/src/status_http.rs | 243 +++++--------- services/forwarder/src/status_store.rs | 62 ++++ 5 files changed, 501 insertions(+), 152 deletions(-) create mode 100644 services/forwarder/src/server_status_task.rs diff --git a/services/forwarder/src/lib.rs b/services/forwarder/src/lib.rs index 9535ac77..08464c71 100644 --- a/services/forwarder/src/lib.rs +++ b/services/forwarder/src/lib.rs @@ -13,6 +13,7 @@ pub mod reader_control; pub mod reader_control_service; pub mod reader_task; pub mod replay; +pub mod server_status_task; pub mod status_http; pub mod status_store; pub mod storage; diff --git a/services/forwarder/src/main.rs b/services/forwarder/src/main.rs index 5fdf0a1a..0375a9b6 100644 --- a/services/forwarder/src/main.rs +++ b/services/forwarder/src/main.rs @@ -712,6 +712,17 @@ async fn main() { }); } + // Spawn background server reachability poll (only when a server URL is + // configured — the status endpoint serves this cache instead of making + // outbound requests inline). + if cfg.p2p.server_url.is_some() { + forwarder::server_status_task::spawn_server_status_task( + config_state.clone(), + status_store.clone(), + shutdown_rx.clone(), + ); + } + // Spawn UPS monitoring task (if enabled) let ups_handle = if cfg.ups.enabled { let ss = status_store.clone(); diff --git a/services/forwarder/src/server_status_task.rs b/services/forwarder/src/server_status_task.rs new file mode 100644 index 00000000..f8e60898 --- /dev/null +++ b/services/forwarder/src/server_status_task.rs @@ -0,0 +1,336 @@ +//! Background poll of the central server's status board. +//! +//! Refreshes the cached [`ServerDeviceStatus`] snapshot in the status store +//! every 30 seconds so the `/api/v1/status` HTTP handler can serve server +//! reachability without performing outbound I/O inline (local UI latency must +//! not depend on WAN state). + +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use tokio::sync::watch; +use tokio::task::JoinHandle; +use tracing::info; + +use crate::config_service::ConfigState; +use crate::status_store::{ServerDeviceStatus, StatusStore}; + +/// How often the server status board is polled. +const POLL_INTERVAL: Duration = Duration::from_secs(30); + +/// Timeout for each outbound status request. +const REQUEST_TIMEOUT: Duration = Duration::from_secs(1); + +#[derive(Debug, serde::Deserialize)] +struct ServerStatusBoardJson { + #[serde(default)] + devices: Vec, +} + +#[derive(Debug, serde::Deserialize)] +struct ServerStatusDeviceJson { + endpoint_id: String, + approval_state: String, +} + +/// Spawn the background task that polls the server status board and stores +/// the resulting [`ServerDeviceStatus`] snapshot in `store`. +/// +/// The first poll fires immediately; subsequent polls run every 30s. The task +/// exits when the shutdown watch channel changes. +pub fn spawn_server_status_task( + config_state: Arc, + store: StatusStore, + mut shutdown_rx: watch::Receiver, +) -> JoinHandle<()> { + tokio::spawn(async move { + let mut interval = tokio::time::interval(POLL_INTERVAL); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + _ = interval.tick() => {} + _ = shutdown_rx.changed() => { + info!("server status poll task shutting down"); + return; + } + } + let status = poll_server_status(&config_state, &store).await; + store.set_server_status(status).await; + } + }) +} + +fn now_unix_ms() -> Option { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .ok() + .and_then(|d| i64::try_from(d.as_millis()).ok()) +} + +fn checked(status: ServerDeviceStatus) -> ServerDeviceStatus { + ServerDeviceStatus { + checked_unix_ms: now_unix_ms(), + ..status + } +} + +fn unreachable_status(endpoint_id: String, message: String) -> ServerDeviceStatus { + checked(ServerDeviceStatus { + configured: true, + endpoint_id: Some(endpoint_id), + reachable: Some(false), + approval_state: None, + waiting_for_approval: false, + message: Some(message), + cached: true, + checked_unix_ms: None, + }) +} + +/// Perform one poll of the server status board and produce the snapshot the +/// status endpoint will serve. +async fn poll_server_status( + config_state: &Arc, + store: &StatusStore, +) -> ServerDeviceStatus { + let endpoint_id = store.subsystem_arc().lock().await.p2p_endpoint_id.clone(); + let Some(endpoint_id) = endpoint_id else { + return checked(ServerDeviceStatus::not_configured()); + }; + let server_url = { + let _guard = config_state.write_lock.lock().await; + match crate::config::load_config_from_path(&config_state.path) { + Ok(config) => config.p2p.server_url, + Err(error) => { + return checked(ServerDeviceStatus { + configured: true, + endpoint_id: Some(endpoint_id), + reachable: None, + approval_state: None, + waiting_for_approval: false, + message: Some(format!("Forwarder config unavailable: {error}")), + cached: true, + checked_unix_ms: None, + }); + } + } + }; + let Some(server_url) = server_url else { + return checked(ServerDeviceStatus::not_configured()); + }; + + let client = match reqwest::Client::builder().timeout(REQUEST_TIMEOUT).build() { + Ok(client) => client, + Err(error) => { + return unreachable_status( + endpoint_id, + format!("Server status client unavailable: {error}"), + ); + } + }; + let status_url = format!("{}/status", server_url.trim_end_matches('/')); + let response = match client.get(status_url).send().await { + Ok(response) => response, + Err(error) => { + return unreachable_status(endpoint_id, format!("Server status unavailable: {error}")); + } + }; + let board = match response.error_for_status() { + Ok(response) => match response.json::().await { + Ok(board) => board, + Err(error) => { + return unreachable_status( + endpoint_id, + format!("Server status response was invalid: {error}"), + ); + } + }, + Err(error) => { + return unreachable_status( + endpoint_id, + format!("Server status returned an error: {error}"), + ); + } + }; + + match board + .devices + .into_iter() + .find(|device| device.endpoint_id == endpoint_id) + { + Some(device) => { + let waiting_for_approval = device.approval_state == "pending"; + checked(ServerDeviceStatus { + configured: true, + endpoint_id: Some(endpoint_id), + reachable: Some(true), + approval_state: Some(device.approval_state), + waiting_for_approval, + message: waiting_for_approval + .then(|| "Waiting for server admin approval".to_owned()), + cached: true, + checked_unix_ms: None, + }) + } + None => checked(ServerDeviceStatus { + configured: true, + endpoint_id: Some(endpoint_id), + reachable: Some(true), + approval_state: None, + waiting_for_approval: true, + message: Some("Waiting for this forwarder to register with the server".to_owned()), + cached: true, + checked_unix_ms: None, + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::status_store::SubsystemStatus; + use std::io::Write; + use tempfile::NamedTempFile; + + fn config_file_with_server_url(server_url: &str) -> (tempfile::TempDir, NamedTempFile) { + let token_dir = tempfile::tempdir().expect("create temp token dir"); + let token_path = token_dir.path().join("fake-token"); + std::fs::write(&token_path, "test-token\n").expect("write token file"); + let token_path = token_path.display().to_string().replace('\\', "/"); + + let mut config_file = NamedTempFile::new().expect("create temp config"); + write!( + config_file, + r#"schema_version = 1 +[p2p] +server_url = "{server_url}" +[auth] +token_file = "{token_path}" +[[readers]] +target = "192.168.1.100:10000" +"# + ) + .expect("write config"); + (token_dir, config_file) + } + + fn store_with_endpoint_id(endpoint_id: &str) -> StatusStore { + let mut subsystem = SubsystemStatus::ready(); + subsystem.set_p2p_endpoint_id(endpoint_id.to_owned()); + StatusStore::new(subsystem) + } + + async fn serve_status_board_once(body: String) -> std::net::SocketAddr { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind mock server"); + let addr = listener.local_addr().expect("mock server addr"); + tokio::spawn(async move { + if let Ok((mut socket, _)) = listener.accept().await { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let mut buf = [0u8; 1024]; + let _ = socket.read(&mut buf).await; + let response = format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", + body.len(), + body + ); + let _ = socket.write_all(response.as_bytes()).await; + } + }); + addr + } + + #[tokio::test] + async fn poll_reports_reachable_active_device() { + let body = r#"{"devices":[{"endpoint_id":"ep-1","approval_state":"active"}]}"#; + let addr = serve_status_board_once(body.to_owned()).await; + let (_token_dir, config_file) = config_file_with_server_url(&format!("http://{addr}")); + let config_state = Arc::new(ConfigState::new(config_file.path().to_path_buf())); + let store = store_with_endpoint_id("ep-1"); + + let status = poll_server_status(&config_state, &store).await; + + assert!(status.configured); + assert_eq!(status.endpoint_id.as_deref(), Some("ep-1")); + assert_eq!(status.reachable, Some(true)); + assert_eq!(status.approval_state.as_deref(), Some("active")); + assert!(!status.waiting_for_approval); + assert!(status.cached); + assert!(status.checked_unix_ms.is_some()); + } + + #[tokio::test] + async fn poll_reports_unreachable_when_server_never_responds() { + // Accepts connections but never responds. + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind hang listener"); + let addr = listener.local_addr().expect("hang listener addr"); + std::thread::spawn(move || { + let mut held = Vec::new(); + for stream in listener.incoming() { + match stream { + Ok(s) => held.push(s), + Err(_) => break, + } + } + }); + let (_token_dir, config_file) = config_file_with_server_url(&format!("http://{addr}")); + let config_state = Arc::new(ConfigState::new(config_file.path().to_path_buf())); + let store = store_with_endpoint_id("ep-1"); + + let status = poll_server_status(&config_state, &store).await; + + assert!(status.configured); + assert_eq!(status.reachable, Some(false)); + assert!(status.cached); + assert!(status.checked_unix_ms.is_some()); + assert!(status.message.is_some()); + } + + #[tokio::test] + async fn poll_reports_not_configured_without_endpoint_id() { + let (_token_dir, config_file) = config_file_with_server_url("http://127.0.0.1:9"); + let config_state = Arc::new(ConfigState::new(config_file.path().to_path_buf())); + let store = StatusStore::new(SubsystemStatus::ready()); + + let status = poll_server_status(&config_state, &store).await; + + assert!(!status.configured); + assert_eq!(status.reachable, None); + assert!(status.cached); + assert!(status.checked_unix_ms.is_some()); + } + + #[tokio::test] + async fn task_populates_store_and_stops_on_shutdown() { + let body = r#"{"devices":[{"endpoint_id":"ep-1","approval_state":"pending"}]}"#; + let addr = serve_status_board_once(body.to_owned()).await; + let (_token_dir, config_file) = config_file_with_server_url(&format!("http://{addr}")); + let config_state = Arc::new(ConfigState::new(config_file.path().to_path_buf())); + let store = store_with_endpoint_id("ep-1"); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + + let handle = spawn_server_status_task(config_state, store.clone(), shutdown_rx); + + let mut snapshot = None; + for _ in 0..100 { + snapshot = store.server_status().await; + if snapshot.is_some() { + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + let snapshot = snapshot.expect("poll task populated server status"); + assert_eq!(snapshot.reachable, Some(true)); + assert_eq!(snapshot.approval_state.as_deref(), Some("pending")); + assert!(snapshot.waiting_for_approval); + assert!(snapshot.cached); + assert!(snapshot.checked_unix_ms.is_some()); + + shutdown_tx.send(true).expect("send shutdown"); + tokio::time::timeout(Duration::from_secs(2), handle) + .await + .expect("task exits on shutdown") + .expect("task join"); + } +} diff --git a/services/forwarder/src/status_http.rs b/services/forwarder/src/status_http.rs index 767db92c..e0ff405b 100644 --- a/services/forwarder/src/status_http.rs +++ b/services/forwarder/src/status_http.rs @@ -673,29 +673,6 @@ fn get_config_state( state.config_state.clone() } -#[derive(serde::Serialize)] -struct ServerDeviceStatusJson { - configured: bool, - endpoint_id: Option, - reachable: Option, - approval_state: Option, - waiting_for_approval: bool, - message: Option, -} - -impl ServerDeviceStatusJson { - fn not_configured() -> Self { - Self { - configured: false, - endpoint_id: None, - reachable: None, - approval_state: None, - waiting_for_approval: false, - message: None, - } - } -} - #[derive(serde::Serialize)] struct StatusJsonResponse { forwarder_id: String, @@ -705,7 +682,7 @@ struct StatusJsonResponse { p2p_connected: bool, restart_needed: bool, ups_status: Option, - server: ServerDeviceStatusJson, + server: crate::status_store::ServerDeviceStatus, readers: Vec, } @@ -721,137 +698,18 @@ struct ReaderStatusJson { reader_info: Option, } -#[derive(Debug, serde::Deserialize)] -struct ServerStatusBoardJson { - #[serde(default)] - devices: Vec, -} - -#[derive(Debug, serde::Deserialize)] -struct ServerStatusDeviceJson { - endpoint_id: String, - approval_state: String, -} - -async fn forwarder_server_status( - state: &AppState, -) -> ServerDeviceStatusJson { - let endpoint_id = state.subsystem.lock().await.p2p_endpoint_id.clone(); - let Some(endpoint_id) = endpoint_id else { - return ServerDeviceStatusJson::not_configured(); - }; - let Some(config_state) = get_config_state(state) else { - return ServerDeviceStatusJson::not_configured(); - }; - let server_url = { - let _guard = config_state.write_lock.lock().await; - match crate::config::load_config_from_path(&config_state.path) { - Ok(config) => config.p2p.server_url, - Err(error) => { - return ServerDeviceStatusJson { - configured: true, - endpoint_id: Some(endpoint_id), - reachable: None, - approval_state: None, - waiting_for_approval: false, - message: Some(format!("Forwarder config unavailable: {error}")), - }; - } - } - }; - let Some(server_url) = server_url else { - return ServerDeviceStatusJson::not_configured(); - }; - - let client = match reqwest::Client::builder() - .timeout(Duration::from_secs(1)) - .build() - { - Ok(client) => client, - Err(error) => { - return ServerDeviceStatusJson { - configured: true, - endpoint_id: Some(endpoint_id), - reachable: Some(false), - approval_state: None, - waiting_for_approval: false, - message: Some(format!("Server status client unavailable: {error}")), - }; - } - }; - let status_url = format!("{}/status", server_url.trim_end_matches('/')); - let response = match client.get(status_url).send().await { - Ok(response) => response, - Err(error) => { - return ServerDeviceStatusJson { - configured: true, - endpoint_id: Some(endpoint_id), - reachable: Some(false), - approval_state: None, - waiting_for_approval: false, - message: Some(format!("Server status unavailable: {error}")), - }; - } - }; - let board = match response.error_for_status() { - Ok(response) => match response.json::().await { - Ok(board) => board, - Err(error) => { - return ServerDeviceStatusJson { - configured: true, - endpoint_id: Some(endpoint_id), - reachable: Some(false), - approval_state: None, - waiting_for_approval: false, - message: Some(format!("Server status response was invalid: {error}")), - }; - } - }, - Err(error) => { - return ServerDeviceStatusJson { - configured: true, - endpoint_id: Some(endpoint_id), - reachable: Some(false), - approval_state: None, - waiting_for_approval: false, - message: Some(format!("Server status returned an error: {error}")), - }; - } - }; - - match board - .devices - .into_iter() - .find(|device| device.endpoint_id == endpoint_id) - { - Some(device) => { - let waiting_for_approval = device.approval_state == "pending"; - ServerDeviceStatusJson { - configured: true, - endpoint_id: Some(endpoint_id), - reachable: Some(true), - approval_state: Some(device.approval_state), - waiting_for_approval, - message: waiting_for_approval - .then(|| "Waiting for server admin approval".to_owned()), - } - } - None => ServerDeviceStatusJson { - configured: true, - endpoint_id: Some(endpoint_id), - reachable: Some(true), - approval_state: None, - waiting_for_approval: true, - message: Some("Waiting for this forwarder to register with the server".to_owned()), - }, - } -} - async fn status_json_handler( State(state): State>, ) -> Response { - let server = forwarder_server_status(&state).await; let ss = state.subsystem.lock().await; + // Served from the cache maintained by `server_status_task`; the handler + // itself performs no outbound I/O so local status latency never depends + // on WAN state. Before the first poll completes (or when no poll task + // runs), serve the not-configured shape with `checked_unix_ms: None`. + let server = ss + .server_status() + .cloned() + .unwrap_or_else(crate::status_store::ServerDeviceStatus::not_configured); let mut readers: Vec<_> = ss .readers .iter() @@ -970,7 +828,6 @@ async fn events_handler( >, > { use axum::response::sse::{Event, KeepAlive, Sse}; - use std::time::Duration; use tokio_stream::{StreamExt, wrappers::BroadcastStream}; let rx = state.ui_tx.subscribe(); @@ -2106,6 +1963,88 @@ mod tests { assert_eq!(body["readers"][0]["state"], "connected"); } + #[tokio::test] + async fn status_json_serves_cached_server_status_without_outbound_io() { + use std::io::Write; + use tempfile::NamedTempFile; + + // A "server" that accepts TCP connections but never responds: any + // inline outbound HTTP call in the status handler would block on it + // until the request timeout (~1s). + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind hang listener"); + let hang_addr = listener.local_addr().expect("hang listener addr"); + std::thread::spawn(move || { + let mut held = Vec::new(); + for stream in listener.incoming() { + match stream { + Ok(s) => held.push(s), + Err(_) => break, + } + } + }); + + let mut config_file = NamedTempFile::new().expect("create temp config"); + let (_token_dir, token_path) = temp_token_file(); + write!( + config_file, + r#"schema_version = 1 +[p2p] +server_url = "http://{hang_addr}" +[auth] +token_file = "{token_path}" +[[readers]] +target = "192.168.1.100:10000" +"# + ) + .expect("write config"); + + let server = StatusServer::start_with_config( + StatusConfig { + bind: "127.0.0.1:0".to_owned(), + forwarder_version: "0.2.0".to_owned(), + }, + SubsystemStatus::ready(), + Arc::new(Mutex::new(NoJournal)), + Arc::new(ConfigState::new(config_file.path().to_path_buf())), + Arc::new(Notify::new()), + ) + .await + .expect("start status server"); + server + .store() + .set_p2p_endpoint_id("endpoint-under-test".to_owned()) + .await; + + let client = reqwest::Client::new(); + let start = std::time::Instant::now(); + let resp = client + .get(format!("http://{}/api/v1/status", server.local_addr())) + .send() + .await + .expect("GET /api/v1/status"); + let elapsed = start.elapsed(); + assert_eq!(resp.status(), 200); + assert!( + elapsed < Duration::from_millis(500), + "status endpoint must not perform outbound I/O inline (took {elapsed:?})" + ); + + let body: serde_json::Value = resp.json().await.expect("json body"); + let server_json = body.get("server").expect("server object"); + assert!( + server_json.get("reachable").is_some(), + "server.reachable missing: {server_json}" + ); + assert_eq!( + server_json["cached"], true, + "server.cached must be true: {server_json}" + ); + assert!( + server_json.get("checked_unix_ms").is_some(), + "server.checked_unix_ms missing: {server_json}" + ); + } + #[tokio::test] async fn refresh_reader_preserves_static_reader_info_fields() { let reader_ip = "192.168.1.10"; diff --git a/services/forwarder/src/status_store.rs b/services/forwarder/src/status_store.rs index f8f602d2..9dac4322 100644 --- a/services/forwarder/src/status_store.rs +++ b/services/forwarder/src/status_store.rs @@ -60,6 +60,44 @@ pub struct UpsStatusState { pub status: Option, } +/// Cached snapshot of this forwarder's registration status on the central +/// server, refreshed by the background poll task (`server_status_task`). +/// +/// The `/api/v1/status` handler serves this snapshot directly and never +/// performs outbound I/O; `checked_unix_ms` is the staleness signal (None +/// until the first poll completes). +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ServerDeviceStatus { + pub configured: bool, + pub endpoint_id: Option, + pub reachable: Option, + pub approval_state: Option, + pub waiting_for_approval: bool, + pub message: Option, + /// Always `true`: signals to clients that this snapshot is served from + /// the poll cache rather than a live server round-trip. + pub cached: bool, + /// Unix ms timestamp of the poll that produced this snapshot. + pub checked_unix_ms: Option, +} + +impl ServerDeviceStatus { + /// Shape served when no server is configured — and before the first poll + /// completes (with `checked_unix_ms: None`). + pub fn not_configured() -> Self { + Self { + configured: false, + endpoint_id: None, + reachable: None, + approval_state: None, + waiting_for_approval: false, + message: None, + cached: true, + checked_unix_ms: None, + } + } +} + /// Forwarder-local status updates consumed by P2P control sessions. #[derive(Debug, Clone)] pub enum ForwarderStatusEvent { @@ -143,6 +181,8 @@ pub struct SubsystemStatus { pub(crate) restart_needed: bool, /// UPS status snapshot (None if UPS monitoring is not configured). pub(crate) ups_status: Option, + /// Cached server reachability snapshot (None until the first poll). + pub(crate) server_status: Option, /// Readers whose read counters changed since the last coalesced P2P /// status broadcast (see [`spawn_read_count_broadcaster`]). pub(crate) read_counts_dirty: HashSet, @@ -164,6 +204,7 @@ impl SubsystemStatus { update_mode: rt_updater::UpdateMode::default(), restart_needed: false, ups_status: None, + server_status: None, read_counts_dirty: HashSet::new(), } } @@ -183,6 +224,7 @@ impl SubsystemStatus { update_mode: rt_updater::UpdateMode::default(), restart_needed: false, ups_status: None, + server_status: None, read_counts_dirty: HashSet::new(), } } @@ -264,6 +306,16 @@ impl SubsystemStatus { pub fn ups_status(&self) -> Option<&UpsStatusState> { self.ups_status.as_ref() } + + /// Set the cached server reachability snapshot. + pub fn set_server_status(&mut self, status: ServerDeviceStatus) { + self.server_status = Some(status); + } + + /// Return the cached server reachability snapshot, if any poll completed. + pub fn server_status(&self) -> Option<&ServerDeviceStatus> { + self.server_status.as_ref() + } } // --------------------------------------------------------------------------- @@ -733,6 +785,16 @@ impl StatusStore { self.publish_display_state().await; } + /// Store the latest server reachability snapshot from the poll task. + pub async fn set_server_status(&self, status: ServerDeviceStatus) { + self.subsystem.lock().await.set_server_status(status); + } + + /// Return the cached server reachability snapshot, if any poll completed. + pub async fn server_status(&self) -> Option { + self.subsystem.lock().await.server_status().cloned() + } + pub fn control_clients( &self, ) -> &Arc>>> { From 5acf1da072d4478f3b0fae13bf49cae0a81f1013 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 18:11:56 -0400 Subject: [PATCH 31/38] fix(forwarder): notify UI on server-status changes; sync ServerDeviceStatus TS contract --- apps/forwarder-ui/src/lib/api.test.ts | 2 + apps/forwarder-ui/src/lib/api.ts | 2 + apps/forwarder-ui/src/lib/config-load.test.ts | 2 + .../src/lib/reader-status-cache.test.ts | 2 + apps/forwarder-ui/src/lib/sse.ts | 11 +- apps/forwarder-ui/src/routes/+page.svelte | 5 + services/forwarder/src/status_http.rs | 3 + services/forwarder/src/status_store.rs | 108 +++++++++++++++++- services/forwarder/src/ui_events.rs | 24 ++++ 9 files changed, 155 insertions(+), 4 deletions(-) diff --git a/apps/forwarder-ui/src/lib/api.test.ts b/apps/forwarder-ui/src/lib/api.test.ts index d9d93d52..5f9eae53 100644 --- a/apps/forwarder-ui/src/lib/api.test.ts +++ b/apps/forwarder-ui/src/lib/api.test.ts @@ -34,6 +34,8 @@ describe("forwarder api client", () => { approval_state: "pending", waiting_for_approval: true, message: "Waiting for server admin approval", + cached: true, + checked_unix_ms: 1700000000000, }, readers: [], }), diff --git a/apps/forwarder-ui/src/lib/api.ts b/apps/forwarder-ui/src/lib/api.ts index f12c51ec..a0e5fb16 100644 --- a/apps/forwarder-ui/src/lib/api.ts +++ b/apps/forwarder-ui/src/lib/api.ts @@ -50,6 +50,8 @@ export interface ServerDeviceStatus { approval_state: string | null; waiting_for_approval: boolean; message: string | null; + cached: boolean; + checked_unix_ms: number | null; } export interface ForwarderStatus { diff --git a/apps/forwarder-ui/src/lib/config-load.test.ts b/apps/forwarder-ui/src/lib/config-load.test.ts index e775df84..d42ccb19 100644 --- a/apps/forwarder-ui/src/lib/config-load.test.ts +++ b/apps/forwarder-ui/src/lib/config-load.test.ts @@ -25,6 +25,8 @@ function makeStatus(restart_needed: boolean): ForwarderStatus { approval_state: null, waiting_for_approval: false, message: null, + cached: true, + checked_unix_ms: null, }, readers: [], }; diff --git a/apps/forwarder-ui/src/lib/reader-status-cache.test.ts b/apps/forwarder-ui/src/lib/reader-status-cache.test.ts index 1dda2e68..1081490b 100644 --- a/apps/forwarder-ui/src/lib/reader-status-cache.test.ts +++ b/apps/forwarder-ui/src/lib/reader-status-cache.test.ts @@ -30,6 +30,8 @@ function makeStatus(readers: ForwarderStatus["readers"]): ForwarderStatus { approval_state: null, waiting_for_approval: false, message: null, + cached: true, + checked_unix_ms: null, }, readers, }; diff --git a/apps/forwarder-ui/src/lib/sse.ts b/apps/forwarder-ui/src/lib/sse.ts index d641f51f..2b292bf9 100644 --- a/apps/forwarder-ui/src/lib/sse.ts +++ b/apps/forwarder-ui/src/lib/sse.ts @@ -1,5 +1,10 @@ import { createSSE, type SseHandle } from "@rusty-timer/shared-ui/lib/sse"; -import type { ReaderInfo, ReaderStatus, UpdateStatusResponse } from "./api"; +import type { + ReaderInfo, + ReaderStatus, + ServerDeviceStatus, + UpdateStatusResponse, +} from "./api"; export type ForwarderSseCallbacks = { onStatusChanged: (data: { @@ -19,6 +24,7 @@ export type ForwarderSseCallbacks = { available: boolean; status: any | null; }) => void; + onServerStatusChanged?: (payload: { server: ServerDeviceStatus }) => void; }; let handle: SseHandle | null = null; @@ -57,6 +63,9 @@ export function initSSE(callbacks: ForwarderSseCallbacks): void { }) => { callbacks.onUpsStatusChanged?.(data); }, + server_status_changed: (data: { server: ServerDeviceStatus }) => { + callbacks.onServerStatusChanged?.(data); + }, }, (connected) => { callbacks.onConnectionChange(connected); diff --git a/apps/forwarder-ui/src/routes/+page.svelte b/apps/forwarder-ui/src/routes/+page.svelte index 560c9f64..44cd9e0c 100644 --- a/apps/forwarder-ui/src/routes/+page.svelte +++ b/apps/forwarder-ui/src/routes/+page.svelte @@ -508,6 +508,11 @@ onUpsStatusChanged: (payload) => { upsState = { available: payload.available, status: payload.status }; }, + onServerStatusChanged: (payload) => { + if (status) { + status = { ...status, server: payload.server }; + } + }, onUpdateStatusChanged: (us) => { if ( (us.status === "available" || us.status === "downloaded") && diff --git a/services/forwarder/src/status_http.rs b/services/forwarder/src/status_http.rs index e0ff405b..439bd5a9 100644 --- a/services/forwarder/src/status_http.rs +++ b/services/forwarder/src/status_http.rs @@ -844,6 +844,9 @@ async fn events_handler( "reader_info_updated" } crate::ui_events::ForwarderUiEvent::UpsStatusChanged { .. } => "ups_status_changed", + crate::ui_events::ForwarderUiEvent::ServerStatusChanged { .. } => { + "server_status_changed" + } }; match serde_json::to_string(&event) { Ok(json) => Some(Ok(Event::default().event(event_type).data(json))), diff --git a/services/forwarder/src/status_store.rs b/services/forwarder/src/status_store.rs index 9dac4322..7329640a 100644 --- a/services/forwarder/src/status_store.rs +++ b/services/forwarder/src/status_store.rs @@ -96,6 +96,18 @@ impl ServerDeviceStatus { checked_unix_ms: None, } } + + /// Compare the UI-meaningful fields of two snapshots, ignoring the + /// bookkeeping fields (`cached` and `checked_unix_ms`) that change on + /// every poll. + pub(crate) fn ui_meaningful_eq(&self, other: &Self) -> bool { + self.configured == other.configured + && self.endpoint_id == other.endpoint_id + && self.reachable == other.reachable + && self.approval_state == other.approval_state + && self.waiting_for_approval == other.waiting_for_approval + && self.message == other.message + } } /// Forwarder-local status updates consumed by P2P control sessions. @@ -308,8 +320,18 @@ impl SubsystemStatus { } /// Set the cached server reachability snapshot. - pub fn set_server_status(&mut self, status: ServerDeviceStatus) { + /// + /// Returns `true` when the new snapshot differs from the previous one in + /// UI-meaningful fields (see [`ServerDeviceStatus::ui_meaningful_eq`]). + /// Before the first poll the status endpoint serves the not-configured + /// baseline, so the first snapshot is compared against that. + pub fn set_server_status(&mut self, status: ServerDeviceStatus) -> bool { + let changed = match &self.server_status { + Some(prev) => !prev.ui_meaningful_eq(&status), + None => !ServerDeviceStatus::not_configured().ui_meaningful_eq(&status), + }; self.server_status = Some(status); + changed } /// Return the cached server reachability snapshot, if any poll completed. @@ -785,9 +807,19 @@ impl StatusStore { self.publish_display_state().await; } - /// Store the latest server reachability snapshot from the poll task. + /// Store the latest server reachability snapshot from the poll task, + /// notifying the UI only when UI-meaningful fields changed. pub async fn set_server_status(&self, status: ServerDeviceStatus) { - self.subsystem.lock().await.set_server_status(status); + let changed = self + .subsystem + .lock() + .await + .set_server_status(status.clone()); + if changed { + let _ = self + .ui_tx + .send(crate::ui_events::ForwarderUiEvent::ServerStatusChanged { server: status }); + } } /// Return the cached server reachability snapshot, if any poll completed. @@ -1085,3 +1117,73 @@ pub(crate) async fn mark_restart_needed_and_emit( restart_needed: true, }); } + +#[cfg(test)] +mod tests { + use super::*; + use tokio::sync::broadcast::error::TryRecvError; + + fn sample_server_status(checked_unix_ms: Option) -> ServerDeviceStatus { + ServerDeviceStatus { + configured: true, + endpoint_id: Some("node-1".to_owned()), + reachable: Some(true), + approval_state: Some("active".to_owned()), + waiting_for_approval: false, + message: None, + cached: true, + checked_unix_ms, + } + } + + #[tokio::test] + async fn set_server_status_emits_ui_event_only_on_meaningful_change() { + let store = StatusStore::new(SubsystemStatus::ready()); + let mut rx = store.ui_sender().subscribe(); + + // First poll differs from the not-configured baseline: one event. + store + .set_server_status(sample_server_status(Some(1_000))) + .await; + match rx.try_recv() { + Ok(crate::ui_events::ForwarderUiEvent::ServerStatusChanged { server }) => { + assert_eq!(server.endpoint_id.as_deref(), Some("node-1")); + } + other => panic!("expected ServerStatusChanged, got {other:?}"), + } + + // Identical snapshot except checked_unix_ms (the common 30s poll case): + // no event. + store + .set_server_status(sample_server_status(Some(31_000))) + .await; + assert!(matches!(rx.try_recv(), Err(TryRecvError::Empty))); + + // UI-meaningful change (reachable flips): exactly one event. + let mut changed = sample_server_status(Some(61_000)); + changed.reachable = Some(false); + changed.message = Some("server unreachable".to_owned()); + store.set_server_status(changed).await; + assert!(matches!( + rx.try_recv(), + Ok(crate::ui_events::ForwarderUiEvent::ServerStatusChanged { .. }) + )); + assert!(matches!(rx.try_recv(), Err(TryRecvError::Empty))); + } + + #[tokio::test] + async fn set_server_status_no_event_when_first_poll_matches_baseline() { + let store = StatusStore::new(SubsystemStatus::ready()); + let mut rx = store.ui_sender().subscribe(); + + // Before the first poll the status endpoint serves the not-configured + // baseline, so a first poll with the same UI-meaningful fields is not + // a change. + let status = ServerDeviceStatus { + checked_unix_ms: Some(1_000), + ..ServerDeviceStatus::not_configured() + }; + store.set_server_status(status).await; + assert!(matches!(rx.try_recv(), Err(TryRecvError::Empty))); + } +} diff --git a/services/forwarder/src/ui_events.rs b/services/forwarder/src/ui_events.rs index 51aa24d9..a0ce8d79 100644 --- a/services/forwarder/src/ui_events.rs +++ b/services/forwarder/src/ui_events.rs @@ -40,6 +40,9 @@ pub enum ForwarderUiEvent { available: bool, status: Option, }, + ServerStatusChanged { + server: crate::status_store::ServerDeviceStatus, + }, } #[cfg(test)] @@ -129,6 +132,27 @@ mod tests { assert!(json["status"].is_null()); } + #[test] + fn server_status_changed_serializes_with_type_tag() { + let event = ForwarderUiEvent::ServerStatusChanged { + server: crate::status_store::ServerDeviceStatus { + configured: true, + endpoint_id: Some("node-1".to_owned()), + reachable: Some(true), + approval_state: Some("active".to_owned()), + waiting_for_approval: false, + message: None, + cached: true, + checked_unix_ms: Some(1_700_000_000_000), + }, + }; + let json: serde_json::Value = serde_json::to_value(&event).unwrap(); + assert_eq!(json["type"], "server_status_changed"); + assert_eq!(json["server"]["configured"], true); + assert_eq!(json["server"]["reachable"], true); + assert_eq!(json["server"]["cached"], true); + } + #[test] fn log_entry_serializes_with_type_tag() { let event = ForwarderUiEvent::LogEntry { From bb7ec82563f57298231d530534065770608da514 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 18:17:53 -0400 Subject: [PATCH 32/38] refactor(forwarder): emit server-status event under lock; destructure ui_meaningful_eq --- services/forwarder/src/status_store.rs | 34 +++++++++++++++++--------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/services/forwarder/src/status_store.rs b/services/forwarder/src/status_store.rs index 7329640a..823d0b97 100644 --- a/services/forwarder/src/status_store.rs +++ b/services/forwarder/src/status_store.rs @@ -101,12 +101,24 @@ impl ServerDeviceStatus { /// bookkeeping fields (`cached` and `checked_unix_ms`) that change on /// every poll. pub(crate) fn ui_meaningful_eq(&self, other: &Self) -> bool { - self.configured == other.configured - && self.endpoint_id == other.endpoint_id - && self.reachable == other.reachable - && self.approval_state == other.approval_state - && self.waiting_for_approval == other.waiting_for_approval - && self.message == other.message + // Destructure so adding a field to ServerDeviceStatus is a compile + // error here instead of a silent gating bug. + let Self { + configured, + endpoint_id, + reachable, + approval_state, + waiting_for_approval, + message, + cached: _, + checked_unix_ms: _, + } = self; + *configured == other.configured + && *endpoint_id == other.endpoint_id + && *reachable == other.reachable + && *approval_state == other.approval_state + && *waiting_for_approval == other.waiting_for_approval + && *message == other.message } } @@ -810,16 +822,16 @@ impl StatusStore { /// Store the latest server reachability snapshot from the poll task, /// notifying the UI only when UI-meaningful fields changed. pub async fn set_server_status(&self, status: ServerDeviceStatus) { - let changed = self - .subsystem - .lock() - .await - .set_server_status(status.clone()); + // Emit while holding the subsystem lock so state change and UI event + // stay ordered, matching set_ready/set_p2p_connected/set_ups_status. + let mut ss = self.subsystem.lock().await; + let changed = ss.set_server_status(status.clone()); if changed { let _ = self .ui_tx .send(crate::ui_events::ForwarderUiEvent::ServerStatusChanged { server: status }); } + drop(ss); } /// Return the cached server reachability snapshot, if any poll completed. From 495ffe8fc7527a96d1202f1fb06baf3e9a7087e1 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 18:34:38 -0400 Subject: [PATCH 33/38] fix(forwarder): validate heartbeat nonces, surface fanout drops, remove unbounded legacy replay path --- services/forwarder/src/local_fanout.rs | 92 +++++++++++++++- services/forwarder/src/main.rs | 5 +- services/forwarder/src/p2p/control.rs | 126 +++++++++++++++++++++- services/forwarder/src/p2p/mod.rs | 4 +- services/forwarder/src/replay.rs | 112 +++++-------------- services/forwarder/src/status_http.rs | 33 ++++++ services/forwarder/src/status_store.rs | 19 ++++ services/forwarder/src/storage/journal.rs | 17 --- services/forwarder/tests/replay_resume.rs | 67 +++++++----- 9 files changed, 337 insertions(+), 138 deletions(-) diff --git a/services/forwarder/src/local_fanout.rs b/services/forwarder/src/local_fanout.rs index fe6c67a6..cd662939 100644 --- a/services/forwarder/src/local_fanout.rs +++ b/services/forwarder/src/local_fanout.rs @@ -21,6 +21,7 @@ use std::collections::HashMap; use std::net::SocketAddr; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use tokio::io::AsyncWriteExt; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::{Mutex, broadcast}; @@ -78,6 +79,10 @@ pub struct FanoutServer { tx: BroadcastSender, /// The bound local address, stored so that Drop can clean up the registry. local_addr: SocketAddr, + /// Running total of messages dropped because consumers lagged behind the + /// broadcast channel. Shared with the status store when wired via + /// [`FanoutServer::set_drop_counter`]. + dropped: Arc, } impl FanoutServer { @@ -102,9 +107,16 @@ impl FanoutServer { listener, tx, local_addr, + dropped: Arc::new(AtomicU64::new(0)), }) } + /// Replace the drop counter with a shared handle (e.g. the status store's + /// `fanout_dropped_total`) so lag-induced drops are visible in status JSON. + pub fn set_drop_counter(&mut self, counter: Arc) { + self.dropped = counter; + } + /// Return the bound local address (useful when port 0 was used). pub fn local_addr(&self) -> SocketAddr { self.local_addr @@ -132,7 +144,7 @@ impl FanoutServer { pub async fn run(self) { while let Ok((stream, _peer_addr)) = self.listener.accept().await { let rx = self.tx.subscribe(); - tokio::spawn(serve_consumer(stream, rx)); + tokio::spawn(serve_consumer(stream, rx, Arc::clone(&self.dropped))); } } } @@ -156,7 +168,16 @@ impl Drop for FanoutServer { /// Drive one consumer connection: forward every broadcast message to the TCP /// writer until the broadcast sender is dropped or the TCP write fails. -async fn serve_consumer(mut stream: TcpStream, mut rx: broadcast::Receiver>) { +/// +/// Messages missed because the consumer lagged behind the broadcast channel +/// are counted per consumer and accumulated into the shared `dropped` total. +async fn serve_consumer( + mut stream: TcpStream, + mut rx: broadcast::Receiver>, + dropped: Arc, +) { + let peer = stream.peer_addr().ok(); + let mut consumer_dropped: u64 = 0; loop { match rx.recv().await { Ok(data) => { @@ -165,9 +186,18 @@ async fn serve_consumer(mut stream: TcpStream, mut rx: broadcast::Receiver { - // Consumer is too slow; skip missed messages and continue. - continue; + Err(broadcast::error::RecvError::Lagged(missed)) => { + // Consumer is too slow; count the skipped messages, then + // continue from the retained window. + if consumer_dropped == 0 { + tracing::warn!( + peer = ?peer, + missed, + "fanout consumer lagged; dropping messages" + ); + } + consumer_dropped += missed; + dropped.fetch_add(missed, Ordering::Relaxed); } Err(broadcast::error::RecvError::Closed) => { // Channel closed (server shutting down). @@ -175,6 +205,13 @@ async fn serve_consumer(mut stream: TcpStream, mut rx: broadcast::Receiver 0 { + tracing::warn!( + peer = ?peer, + dropped = consumer_dropped, + "fanout consumer disconnected after dropping messages" + ); + } } // --------------------------------------------------------------------------- @@ -184,6 +221,51 @@ async fn serve_consumer(mut stream: TcpStream, mut rx: broadcast::Receiver>(CAPACITY); + for i in 0..SENT { + tx.send(vec![u8::try_from(i % 256).unwrap()]).unwrap(); + } + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let mut client = TcpStream::connect(addr).await.unwrap(); + let (server_stream, _) = listener.accept().await.unwrap(); + + let dropped = Arc::new(AtomicU64::new(0)); + let task = tokio::spawn(serve_consumer(server_stream, rx, Arc::clone(&dropped))); + + // Drain everything the consumer forwards, then close the channel so + // the consumer task exits. + drop(tx); + let mut received = Vec::new(); + client.read_to_end(&mut received).await.unwrap(); + task.await.unwrap(); + + let expected_dropped = SENT - CAPACITY as u64; + assert_eq!( + dropped.load(Ordering::Relaxed), + expected_dropped, + "drop counter must record the messages the lagged consumer missed" + ); + assert_eq!( + received.len() as u64, + SENT - expected_dropped, + "the retained window must still be delivered" + ); + } /// Verify that dropping a FanoutServer removes its entry from the registry. #[tokio::test] diff --git a/services/forwarder/src/main.rs b/services/forwarder/src/main.rs index 0375a9b6..629df305 100644 --- a/services/forwarder/src/main.rs +++ b/services/forwarder/src/main.rs @@ -181,7 +181,7 @@ async fn main() { .unwrap_or_else(|| ep.default_local_fallback_port()); let bind_addr = format!("0.0.0.0:{}", local_port); - let fanout = match FanoutServer::bind(&bind_addr).await { + let mut fanout = match FanoutServer::bind(&bind_addr).await { Ok(f) => f, Err(e) => { eprintln!( @@ -192,6 +192,9 @@ async fn main() { } }; + // Surface lag-induced consumer drops in the status JSON. + fanout.set_drop_counter(status_server.store().fanout_drop_counter()); + let fanout_addr = fanout.local_addr(); info!( reader_ip = %ep.ip, diff --git a/services/forwarder/src/p2p/control.rs b/services/forwarder/src/p2p/control.rs index 435d4577..ba2615c3 100644 --- a/services/forwarder/src/p2p/control.rs +++ b/services/forwarder/src/p2p/control.rs @@ -674,7 +674,21 @@ async fn run_control_loop( frame = rx.recv() => { match frame { Some(control) => match control.msg { - Some(control_c2f::Msg::Pong(_)) => outstanding = 0, + Some(control_c2f::Msg::Pong(pong)) => { + // Only a pong echoing the last sent ping nonce + // counts as liveness; anything else (stale, + // fabricated) is logged and ignored. + if pong.nonce == nonce { + outstanding = 0; + } else { + tracing::warn!( + %peer, + expected = nonce, + got = pong.nonce, + "p2p: pong nonce mismatch; not counting as liveness" + ); + } + } Some(control_c2f::Msg::Ping(ping)) => { let pong = ControlF2C { msg: Some(control_f2c::Msg::Pong(Pong { nonce: ping.nonce })), @@ -2307,6 +2321,116 @@ mod tests { forwarder.close().await; Ok(()) } + #[tokio::test] + async fn bogus_pong_nonce_does_not_reset_heartbeat() -> TestResult { + let heartbeat = HeartbeatConfig { + interval: Duration::from_millis(50), + max_missed: 3, + }; + let (forwarder, forwarder_addr, handle) = spawn_forwarder( + [80; 32], + StaticCatalog::new(sample_catalog()), + LONG_HANDSHAKE, + heartbeat, + ) + .await?; + + let receiver = EndpointBuilder::test([81; 32]).bind().await?; + let (connection, mut send, mut recv) = tokio::time::timeout( + LONG_HANDSHAKE, + open_control(&receiver, forwarder_addr, forwarder_hello()), + ) + .await??; + + let _hello_ok = read_frame::(&mut recv).await?; + let _catalog = read_frame::(&mut recv).await?; + + // Answer every ping promptly, but with a WRONG nonce. These must not + // count as liveness: the loop must still declare the peer dead after + // max_missed unanswered pings. + let responder = tokio::spawn(async move { + while let Ok(frame) = read_frame::(&mut recv).await { + if let Some(control_f2c::Msg::Ping(ping)) = frame.msg { + let bogus = ControlC2F { + msg: Some(control_c2f::Msg::Pong(Pong { + nonce: ping.nonce + 999, + })), + }; + if write_frame(&mut send, &bogus).await.is_err() { + break; + } + } + } + }); + + let result = tokio::time::timeout(Duration::from_secs(5), handle).await??; + assert!( + result.is_err(), + "bogus pong nonces must not reset heartbeat liveness, got {result:?}" + ); + + responder.abort(); + connection.close(0u32.into(), b"done"); + receiver.close().await; + forwarder.close().await; + Ok(()) + } + + #[tokio::test] + async fn matching_pong_nonce_keeps_heartbeat_alive() -> TestResult { + let heartbeat = HeartbeatConfig { + interval: Duration::from_millis(50), + max_missed: 3, + }; + let (forwarder, forwarder_addr, handle) = spawn_forwarder( + [82; 32], + StaticCatalog::new(sample_catalog()), + LONG_HANDSHAKE, + heartbeat, + ) + .await?; + + let receiver = EndpointBuilder::test([83; 32]).bind().await?; + let (connection, mut send, mut recv) = tokio::time::timeout( + LONG_HANDSHAKE, + open_control(&receiver, forwarder_addr, forwarder_hello()), + ) + .await??; + + let _hello_ok = read_frame::(&mut recv).await?; + let _catalog = read_frame::(&mut recv).await?; + + // Echo every ping's nonce back: the loop must stay alive well past + // max_missed * interval. + let responder = tokio::spawn(async move { + while let Ok(frame) = read_frame::(&mut recv).await { + if let Some(control_f2c::Msg::Ping(ping)) = frame.msg { + let pong = ControlC2F { + msg: Some(control_c2f::Msg::Pong(Pong { nonce: ping.nonce })), + }; + if write_frame(&mut send, &pong).await.is_err() { + break; + } + } + } + }); + + // 500ms >> max_missed (3) * interval (50ms); correct pongs must keep + // the loop alive the whole time. + tokio::time::sleep(Duration::from_millis(500)).await; + assert!( + !handle.is_finished(), + "matching pong nonces must keep the heartbeat alive" + ); + + responder.abort(); + handle.abort(); + connection.close(0u32.into(), b"done"); + receiver.close().await; + forwarder.close().await; + Ok(()) + } + #[tokio::test] async fn heartbeat_timeout_closes() -> TestResult { let heartbeat = HeartbeatConfig { diff --git a/services/forwarder/src/p2p/mod.rs b/services/forwarder/src/p2p/mod.rs index 0dc5ad8c..10de347e 100644 --- a/services/forwarder/src/p2p/mod.rs +++ b/services/forwarder/src/p2p/mod.rs @@ -346,8 +346,8 @@ fn snapshot_reader_connected(snapshot: &ForwarderStatusSnapshot, stream: &str) - /// Applies reader connectivity deltas from the status feed to the catalog. /// -/// Exits when the status feed closes; otherwise runs until aborted at P2P -/// shutdown. +/// Runs until aborted at P2P shutdown: this task holds a feed clone (and thus +/// a sender clone) itself, so the status channel never closes underneath it. async fn run_catalog_updates( feed: ForwarderStatusFeed, mut status_rx: broadcast::Receiver, diff --git a/services/forwarder/src/replay.rs b/services/forwarder/src/replay.rs index 90eb559f..de140efd 100644 --- a/services/forwarder/src/replay.rs +++ b/services/forwarder/src/replay.rs @@ -3,20 +3,12 @@ //! Used by the P2P session to determine which events need to be //! (re-)transmitted after a reconnect or on initial connect. -use crate::storage::journal::{Journal, JournalError, JournalEvent, ReplayJournal}; +use crate::storage::journal::{JournalError, JournalEvent, ReplayJournal}; // --------------------------------------------------------------------------- // Public types // --------------------------------------------------------------------------- -/// A group of pending events for a single (stream_key, stream_epoch) pair. -#[derive(Debug)] -pub struct ReplayResult { - pub stream_key: String, - pub stream_epoch: i64, - pub events: Vec, -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct GapNotice { pub requested_cursor: i64, @@ -83,57 +75,6 @@ impl ReplayEngine { gap: None, }) } - - /// Return all pending events for a stream, grouped by epoch. - /// - /// Each `ReplayResult` covers one epoch. If there are unacked events in - /// multiple epochs (e.g., old epoch + new epoch after bump), all are returned. - /// - /// Epochs are returned in ascending order (oldest first = acked drains first). - pub fn pending_events( - &self, - journal: &Journal, - stream_key: &str, - ) -> Result, JournalError> { - let (acked_epoch, acked_seq) = journal.ack_cursor(stream_key)?; - let mut results: Vec = Vec::new(); - - // 1. Replay old-epoch backlog (events in acked_epoch with seq > acked_seq) - if acked_epoch > 0 { - let events = journal.unacked_events(stream_key, acked_epoch, acked_seq)?; - if !events.is_empty() { - results.push(ReplayResult { - stream_key: stream_key.to_owned(), - stream_epoch: acked_epoch, - events, - }); - } - } - - // 2. Replay events in epochs after acked_epoch (new epoch events, all from seq > 0) - // We query a range of possible epochs. Since we don't have a direct "list epochs" - // method, use a helper: get all journal entries for stream_key with epoch > acked_epoch. - let newer_events = journal.unacked_events_across_epochs(stream_key, acked_epoch)?; - - // Group by epoch - let mut epoch_groups: std::collections::BTreeMap> = - std::collections::BTreeMap::new(); - for ev in newer_events { - epoch_groups.entry(ev.stream_epoch).or_default().push(ev); - } - - for (epoch, events) in epoch_groups { - if !events.is_empty() { - results.push(ReplayResult { - stream_key: stream_key.to_owned(), - stream_epoch: epoch, - events, - }); - } - } - - Ok(results) - } } impl Default for ReplayEngine { @@ -292,7 +233,7 @@ mod tests { } #[test] - fn pending_events_groups_and_orders_epochs_when_ack_cursor_is_zero() { + fn read_after_spans_epoch_bump_in_seq_order() { let (mut journal, _file) = make_journal(); journal.ensure_stream_state("10.0.0.10:10000", 1).unwrap(); @@ -311,22 +252,20 @@ mod tests { .insert_event("10.0.0.10:10000", 2, seq3, None, b"epoch2-seq1", "RAW") .unwrap(); - let replay = ReplayEngine::new() - .pending_events(&journal, "10.0.0.10:10000") + let batch = ReplayEngine::new() + .read_after(&journal, "10.0.0.10:10000", 0, 10) .unwrap(); - assert_eq!(replay.len(), 2); - assert_eq!(replay[0].stream_epoch, 1); - assert_eq!(replay[1].stream_epoch, 2); - assert_eq!(replay[0].events.len(), 2); - assert_eq!(replay[1].events.len(), 1); - assert_eq!(replay[0].events[0].seq, 1); - assert_eq!(replay[0].events[1].seq, 2); - assert_eq!(replay[1].events[0].seq, 3); + assert!(batch.gap.is_none()); + assert_eq!(batch.records.len(), 3); + let seqs: Vec = batch.records.iter().map(|e| e.seq).collect(); + assert_eq!(seqs, vec![1, 2, 3]); + let epochs: Vec = batch.records.iter().map(|e| e.stream_epoch).collect(); + assert_eq!(epochs, vec![1, 1, 2]); } #[test] - fn pending_events_includes_old_epoch_backlog_and_newer_epochs() { + fn read_after_returns_old_epoch_backlog_and_newer_epochs() { let (mut journal, _file) = make_journal(); journal.ensure_stream_state("10.0.0.20:10000", 1).unwrap(); @@ -346,30 +285,29 @@ mod tests { .unwrap(); } - let replay = ReplayEngine::new() - .pending_events(&journal, "10.0.0.20:10000") + let (_epoch, acked_seq) = journal.ack_cursor("10.0.0.20:10000").unwrap(); + let batch = ReplayEngine::new() + .read_after(&journal, "10.0.0.20:10000", acked_seq, 10) .unwrap(); - assert_eq!(replay.len(), 2); - assert_eq!(replay[0].stream_epoch, 1); - assert_eq!(replay[0].events.len(), 2); - assert_eq!(replay[0].events[0].seq, 2); - assert_eq!(replay[0].events[1].seq, 3); - assert_eq!(replay[1].stream_epoch, 2); - assert_eq!(replay[1].events.len(), 2); - assert_eq!(replay[1].events[0].seq, 4); - assert_eq!(replay[1].events[1].seq, 5); + assert!(batch.gap.is_none()); + assert_eq!(batch.records.len(), 4); + let seqs: Vec = batch.records.iter().map(|e| e.seq).collect(); + assert_eq!(seqs, vec![2, 3, 4, 5]); + let epochs: Vec = batch.records.iter().map(|e| e.stream_epoch).collect(); + assert_eq!(epochs, vec![1, 1, 2, 2]); } #[test] - fn pending_events_is_empty_for_initialized_stream_without_events() { + fn read_after_is_empty_for_initialized_stream_without_events() { let (mut journal, _file) = make_journal(); journal.ensure_stream_state("10.0.0.30:10000", 1).unwrap(); - let replay = ReplayEngine::new() - .pending_events(&journal, "10.0.0.30:10000") + let batch = ReplayEngine::new() + .read_after(&journal, "10.0.0.30:10000", 0, 10) .unwrap(); - assert!(replay.is_empty()); + assert!(batch.gap.is_none()); + assert!(batch.records.is_empty()); } } diff --git a/services/forwarder/src/status_http.rs b/services/forwarder/src/status_http.rs index 439bd5a9..78063581 100644 --- a/services/forwarder/src/status_http.rs +++ b/services/forwarder/src/status_http.rs @@ -683,6 +683,8 @@ struct StatusJsonResponse { restart_needed: bool, ups_status: Option, server: crate::status_store::ServerDeviceStatus, + /// Total local-fanout messages dropped because consumers lagged. + fanout_dropped_total: u64, readers: Vec, } @@ -742,6 +744,7 @@ async fn status_json_handler( restart_needed: ss.restart_needed(), ups_status: ss.ups_status().cloned(), server, + fanout_dropped_total: state.store.fanout_dropped_total(), readers, }; @@ -1966,6 +1969,36 @@ mod tests { assert_eq!(body["readers"][0]["state"], "connected"); } + #[tokio::test] + async fn status_json_includes_fanout_dropped_total() { + let server = StatusServer::start( + StatusConfig { + bind: "127.0.0.1:0".to_owned(), + forwarder_version: "test".to_owned(), + }, + SubsystemStatus::ready(), + ) + .await + .expect("start status server"); + + server + .store() + .fanout_drop_counter() + .fetch_add(3, Ordering::SeqCst); + + let addr = server.local_addr(); + let resp = reqwest::get(format!("http://{}/api/v1/status", addr)) + .await + .expect("GET /api/v1/status"); + assert_eq!(resp.status(), 200); + + let body: serde_json::Value = resp.json().await.expect("json body"); + assert_eq!( + body["fanout_dropped_total"], 3, + "status JSON must surface the fanout drop total" + ); + } + #[tokio::test] async fn status_json_serves_cached_server_status_without_outbound_io() { use std::io::Write; diff --git a/services/forwarder/src/status_store.rs b/services/forwarder/src/status_store.rs index 823d0b97..e4693e9b 100644 --- a/services/forwarder/src/status_store.rs +++ b/services/forwarder/src/status_store.rs @@ -9,6 +9,7 @@ use rt_updater::workflow::WorkflowState; use serde::Serialize; use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, Instant}; use tokio::sync::{Mutex, Notify, broadcast}; @@ -378,6 +379,9 @@ pub struct StatusStore { >, >, reconnect_notifies: Arc>>>, + /// Running total of local-fanout messages dropped because consumers + /// lagged (see `local_fanout::serve_consumer`). + fanout_dropped_total: Arc, } fn bridge_download_progress_events( @@ -591,6 +595,7 @@ impl StatusStore { control_clients: Arc::new(std::sync::RwLock::new(HashMap::new())), download_trackers: Arc::new(std::sync::RwLock::new(HashMap::new())), reconnect_notifies: Arc::new(std::sync::RwLock::new(HashMap::new())), + fanout_dropped_total: Arc::new(AtomicU64::new(0)), }; spawn_read_count_broadcaster(subsystem, status_event_tx); store @@ -601,6 +606,20 @@ impl StatusStore { self.status_event_tx.clone() } + /// Return the shared fanout drop counter handle for wiring into + /// [`crate::local_fanout::FanoutServer::set_drop_counter`]. + #[must_use] + pub fn fanout_drop_counter(&self) -> Arc { + Arc::clone(&self.fanout_dropped_total) + } + + /// Return the running total of fanout messages dropped due to lagged + /// consumers. + #[must_use] + pub fn fanout_dropped_total(&self) -> u64 { + self.fanout_dropped_total.load(Ordering::Relaxed) + } + /// Return a clone of the internal subsystem status Arc. pub fn subsystem_arc(&self) -> Arc> { self.subsystem.clone() diff --git a/services/forwarder/src/storage/journal.rs b/services/forwarder/src/storage/journal.rs index a3516532..a704b8e1 100644 --- a/services/forwarder/src/storage/journal.rs +++ b/services/forwarder/src/storage/journal.rs @@ -620,23 +620,6 @@ impl Journal { .map_err(Into::into) } - /// Return all events for stream key with epoch strictly greater than `after_epoch`. - pub fn unacked_events_across_epochs( - &self, - stream_key: &str, - after_epoch: i64, - ) -> Result, JournalError> { - let mut stmt = self.conn.prepare( - "SELECT rowid, stream_id, epoch, seq, reader_timestamp, raw_frame, read_kind, - CAST(received_unix_ms AS TEXT) - FROM events - WHERE stream_id = ?1 AND epoch > ?2 - ORDER BY epoch ASC, seq ASC", - )?; - let rows = stmt.query_map(params![stream_key, after_epoch], map_event)?; - collect_events(rows) - } - /// Delete up to `limit` acked events for `stream_key`. pub fn prune_acked(&mut self, stream_key: &str, limit: i64) -> Result { let (_, acked_seq) = self.ack_cursor(stream_key)?; diff --git a/services/forwarder/tests/replay_resume.rs b/services/forwarder/tests/replay_resume.rs index f2614f7d..1abeb95d 100644 --- a/services/forwarder/tests/replay_resume.rs +++ b/services/forwarder/tests/replay_resume.rs @@ -36,16 +36,17 @@ fn replay_starts_after_ack_cursor() { // Ack through seq 3 j.update_ack_cursor("192.168.2.10", 1, 3).unwrap(); - // Replay should return events 4 and 5 + // Replay from the durable ack cursor should return events 4 and 5 let engine = ReplayEngine::new(); - let result = engine.pending_events(&j, "192.168.2.10").unwrap(); - - assert_eq!(result.len(), 1, "one epoch replay batch expected"); - let batch = &result[0]; - assert_eq!(batch.stream_epoch, 1); - assert_eq!(batch.events.len(), 2, "events 4 and 5 should be pending"); - assert_eq!(batch.events[0].seq, 4); - assert_eq!(batch.events[1].seq, 5); + let (_epoch, acked_seq) = j.ack_cursor("192.168.2.10").unwrap(); + let batch = engine + .read_after(&j, "192.168.2.10", acked_seq, 100) + .unwrap(); + + assert!(batch.gap.is_none()); + assert_eq!(batch.records.len(), 2, "events 4 and 5 should be pending"); + assert_eq!(batch.records[0].seq, 4); + assert_eq!(batch.records[1].seq, 5); } /// Test: no replay events when all events are acked. @@ -64,10 +65,14 @@ fn replay_returns_empty_when_fully_acked() { j.update_ack_cursor("192.168.2.20", 1, 3).unwrap(); let engine = ReplayEngine::new(); - let result = engine.pending_events(&j, "192.168.2.20").unwrap(); - let total_events: usize = result.iter().map(|b| b.events.len()).sum(); + let (_epoch, acked_seq) = j.ack_cursor("192.168.2.20").unwrap(); + let batch = engine + .read_after(&j, "192.168.2.20", acked_seq, 100) + .unwrap(); + assert!(batch.gap.is_none()); assert_eq!( - total_events, 0, + batch.records.len(), + 0, "no events should be pending when fully acked" ); } @@ -96,15 +101,23 @@ fn replay_includes_old_epoch_unacked_events() { } let engine = ReplayEngine::new(); - let result = engine.pending_events(&j, "192.168.2.30").unwrap(); - - // Should have both epoch 1 and epoch 2 batches - assert!(!result.is_empty(), "should have pending events"); - let total_events: usize = result.iter().map(|b| b.events.len()).sum(); + let (_epoch, acked_seq) = j.ack_cursor("192.168.2.30").unwrap(); + let batch = engine + .read_after(&j, "192.168.2.30", acked_seq, 100) + .unwrap(); + + // The stream-wide cursor spans the epoch bump: all 4 events come back in + // seq order, carrying their original epochs. + assert!(batch.gap.is_none()); assert_eq!( - total_events, 4, + batch.records.len(), + 4, "all 4 unacked events (2 epoch1 + 2 epoch2) should be pending" ); + let epochs: Vec = batch.records.iter().map(|e| e.stream_epoch).collect(); + assert_eq!(epochs, vec![1, 1, 2, 2]); + let seqs: Vec = batch.records.iter().map(|e| e.seq).collect(); + assert_eq!(seqs, vec![1, 2, 3, 4]); } /// Test: after replay and ack, cursor advances correctly. @@ -124,17 +137,21 @@ fn replay_cursor_advances_after_ack() { // Only seq 3 should be pending let engine = ReplayEngine::new(); - let result = engine.pending_events(&j, "192.168.2.40").unwrap(); - let total: usize = result.iter().map(|b| b.events.len()).sum(); - assert_eq!(total, 1); - assert_eq!(result[0].events[0].seq, 3); + let (_epoch, acked_seq) = j.ack_cursor("192.168.2.40").unwrap(); + let batch = engine + .read_after(&j, "192.168.2.40", acked_seq, 100) + .unwrap(); + assert_eq!(batch.records.len(), 1); + assert_eq!(batch.records[0].seq, 3); // Now ack seq 3 too j.update_ack_cursor("192.168.2.40", 1, 3).unwrap(); - let result2 = engine.pending_events(&j, "192.168.2.40").unwrap(); - let total2: usize = result2.iter().map(|b| b.events.len()).sum(); - assert_eq!(total2, 0, "nothing pending after full ack"); + let (_epoch, acked_seq) = j.ack_cursor("192.168.2.40").unwrap(); + let batch = engine + .read_after(&j, "192.168.2.40", acked_seq, 100) + .unwrap(); + assert_eq!(batch.records.len(), 0, "nothing pending after full ack"); } /// Test: ack cursor ignores stale (lower) seq updates. From 5819a74a54be9e9ffd154c9d425959a869edf539 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 18:45:43 -0400 Subject: [PATCH 34/38] fix(forwarder): accept pongs for any outstanding heartbeat nonce --- services/forwarder/src/p2p/control.rs | 158 ++++++++++++++++++++++---- 1 file changed, 138 insertions(+), 20 deletions(-) diff --git a/services/forwarder/src/p2p/control.rs b/services/forwarder/src/p2p/control.rs index ba2615c3..cf3ef8c3 100644 --- a/services/forwarder/src/p2p/control.rs +++ b/services/forwarder/src/p2p/control.rs @@ -640,7 +640,8 @@ async fn run_control_loop( ticker.tick().await; let mut nonce: u64 = 0; - let mut outstanding: u32 = 0; + let mut highest_acked: u64 = 0; + let mut outstanding: u64 = 0; let (control_response_tx, mut control_response_rx) = mpsc::channel::(16); // Remote-config verbs (get/set/restart) reply with arbitrary ControlF2C // frames rather than a ReaderControlResponse, so they use a dedicated @@ -664,7 +665,7 @@ async fn run_control_loop( break Ok(()); } outstanding += 1; - if outstanding >= config.max_missed { + if outstanding >= u64::from(config.max_missed) { break Err(format!( "heartbeat timed out after {outstanding} unanswered pings" ) @@ -675,17 +676,27 @@ async fn run_control_loop( match frame { Some(control) => match control.msg { Some(control_c2f::Msg::Pong(pong)) => { - // Only a pong echoing the last sent ping nonce - // counts as liveness; anything else (stale, - // fabricated) is logged and ignored. - if pong.nonce == nonce { - outstanding = 0; + // A pong counts as liveness when it acknowledges a + // ping we actually sent (<= nonce) and haven't seen + // acked yet (> highest_acked). Accepting pongs for + // earlier outstanding pings lets slow-but-alive + // peers survive RTTs up to max_missed * interval + // instead of a single interval. Stale duplicates + // and fabricated nonces are logged and ignored. + if pong.nonce > highest_acked && pong.nonce <= nonce { + highest_acked = pong.nonce; + // Pings after the acked one are still + // unanswered; the guard above guarantees + // nonce >= highest_acked. + outstanding = nonce - highest_acked; } else { tracing::warn!( %peer, - expected = nonce, + expected_min = highest_acked + 1, + expected_max = nonce, got = pong.nonce, - "p2p: pong nonce mismatch; not counting as liveness" + "p2p: pong nonce outside outstanding window; \ + not counting as liveness" ); } } @@ -2401,30 +2412,137 @@ mod tests { let _catalog = read_frame::(&mut recv).await?; // Echo every ping's nonce back: the loop must stay alive well past - // max_missed * interval. + // max_missed pings. Reply-per-ping (no fixed sleeps) keeps this + // immune to scheduler delay on loaded CI. + let mut pings_seen = 0u32; + while pings_seen < 8 { + let frame = + tokio::time::timeout(LONG_HANDSHAKE, read_frame::(&mut recv)).await??; + if let Some(control_f2c::Msg::Ping(ping)) = frame.msg { + pings_seen += 1; + let pong = ControlC2F { + msg: Some(control_c2f::Msg::Pong(Pong { nonce: ping.nonce })), + }; + write_frame(&mut send, &pong).await?; + } + } + assert!( + !handle.is_finished(), + "matching pong nonces must keep the heartbeat alive" + ); + + handle.abort(); + connection.close(0u32.into(), b"done"); + receiver.close().await; + forwarder.close().await; + Ok(()) + } + + #[tokio::test] + async fn pong_for_previous_outstanding_nonce_keeps_heartbeat_alive() -> TestResult { + let heartbeat = HeartbeatConfig { + interval: Duration::from_millis(50), + max_missed: 3, + }; + let (forwarder, forwarder_addr, handle) = spawn_forwarder( + [84; 32], + StaticCatalog::new(sample_catalog()), + LONG_HANDSHAKE, + heartbeat, + ) + .await?; + + let receiver = EndpointBuilder::test([85; 32]).bind().await?; + let (connection, mut send, mut recv) = tokio::time::timeout( + LONG_HANDSHAKE, + open_control(&receiver, forwarder_addr, forwarder_hello()), + ) + .await??; + + let _hello_ok = read_frame::(&mut recv).await?; + let _catalog = read_frame::(&mut recv).await?; + + // Reply one ping late: upon receiving ping N, pong nonce N-1. Every + // pong references a previous-but-still-outstanding ping, mimicking a + // slow-but-alive peer whose RTT exceeds one heartbeat interval. + // Liveness must tolerate this (up to max_missed * interval), so the + // loop must survive well past max_missed pings. Reply-per-ping (no + // fixed sleeps) keeps this immune to scheduler delay on loaded CI. + let mut pings_seen = 0u32; + while pings_seen < 8 { + let frame = + tokio::time::timeout(LONG_HANDSHAKE, read_frame::(&mut recv)).await??; + if let Some(control_f2c::Msg::Ping(ping)) = frame.msg { + pings_seen += 1; + if ping.nonce > 1 { + let pong = ControlC2F { + msg: Some(control_c2f::Msg::Pong(Pong { + nonce: ping.nonce - 1, + })), + }; + write_frame(&mut send, &pong).await?; + } + } + } + assert!( + !handle.is_finished(), + "pongs for previous outstanding nonces must keep the heartbeat alive" + ); + + handle.abort(); + connection.close(0u32.into(), b"done"); + receiver.close().await; + forwarder.close().await; + Ok(()) + } + + #[tokio::test] + async fn stale_duplicate_pong_does_not_reset_heartbeat() -> TestResult { + let heartbeat = HeartbeatConfig { + interval: Duration::from_millis(50), + max_missed: 3, + }; + let (forwarder, forwarder_addr, handle) = spawn_forwarder( + [86; 32], + StaticCatalog::new(sample_catalog()), + LONG_HANDSHAKE, + heartbeat, + ) + .await?; + + let receiver = EndpointBuilder::test([87; 32]).bind().await?; + let (connection, mut send, mut recv) = tokio::time::timeout( + LONG_HANDSHAKE, + open_control(&receiver, forwarder_addr, forwarder_hello()), + ) + .await??; + + let _hello_ok = read_frame::(&mut recv).await?; + let _catalog = read_frame::(&mut recv).await?; + + // Ack the first ping legitimately, then replay that same pong for + // every later ping. Duplicates of an already-acked nonce must not + // count as liveness, so the heartbeat must still time out. let responder = tokio::spawn(async move { while let Ok(frame) = read_frame::(&mut recv).await { - if let Some(control_f2c::Msg::Ping(ping)) = frame.msg { - let pong = ControlC2F { - msg: Some(control_c2f::Msg::Pong(Pong { nonce: ping.nonce })), + if let Some(control_f2c::Msg::Ping(_)) = frame.msg { + let stale = ControlC2F { + msg: Some(control_c2f::Msg::Pong(Pong { nonce: 1 })), }; - if write_frame(&mut send, &pong).await.is_err() { + if write_frame(&mut send, &stale).await.is_err() { break; } } } }); - // 500ms >> max_missed (3) * interval (50ms); correct pongs must keep - // the loop alive the whole time. - tokio::time::sleep(Duration::from_millis(500)).await; + let result = tokio::time::timeout(Duration::from_secs(5), handle).await??; assert!( - !handle.is_finished(), - "matching pong nonces must keep the heartbeat alive" + result.is_err(), + "stale duplicate pongs must not reset heartbeat liveness, got {result:?}" ); responder.abort(); - handle.abort(); connection.close(0u32.into(), b"done"); receiver.close().await; forwarder.close().await; From b1e548020dd3a13444aa9b3acc8623557637144a Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 18:58:58 -0400 Subject: [PATCH 35/38] docs(forwarder): update p2p module doc for production reader-control handler --- services/forwarder/src/p2p/mod.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/forwarder/src/p2p/mod.rs b/services/forwarder/src/p2p/mod.rs index 10de347e..ceb36515 100644 --- a/services/forwarder/src/p2p/mod.rs +++ b/services/forwarder/src/p2p/mod.rs @@ -11,9 +11,10 @@ //! Scope: production startup wires the endpoint, accept loop, allow-listed //! control-plane handshake, data-stream subscriber handler, server allow-list //! distribution components ([`ServerAllowListClient`] and -//! [`run_allowlist_distribution`]), and forwarder status events. Reader control -//! actions are still handled by the no-op adapter until a production adapter is -//! installed. +//! [`run_allowlist_distribution`]), and forwarder status events. Reader +//! control actions are served by [`ForwarderReaderControlHandler`], gated by +//! the negotiated `CAP_READER_CONTROL` capability and +//! `control.allow_reader_control`. mod allowlist; mod control; From 3c42646e5567db2abb8995996cf9534bf8e8ba15 Mon Sep 17 00:00:00 2001 From: Isaac Wismer Date: Fri, 3 Jul 2026 19:10:35 -0400 Subject: [PATCH 36/38] fix(receiver-ui): make protected forwarder config sections read-only in the remote config modal --- .../components/ForwarderConfigModal.svelte | 15 ++++++- .../src/lib/forwarder-config-modal.test.ts | 43 +++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/apps/receiver-ui/src/lib/components/ForwarderConfigModal.svelte b/apps/receiver-ui/src/lib/components/ForwarderConfigModal.svelte index c99ccf78..e84b8bcf 100644 --- a/apps/receiver-ui/src/lib/components/ForwarderConfigModal.svelte +++ b/apps/receiver-ui/src/lib/components/ForwarderConfigModal.svelte @@ -257,11 +257,15 @@
+

+ Managed locally on the forwarder — not editable remotely. +