diff --git a/CHANGELOG.md b/CHANGELOG.md index 97e6157..76de5c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - `SampleQueue::push_owned`, which moves a block into a queue with no copy. - `ClockState::reset_offset`, `ClockState::retarget`, `ClockState::is_running`, and `clock::FIRST_OFFSET_TIMEOUT`. +- `clock::FIRST_PROBE_INTERVAL` and `clock::FIRST_RETRY`. ### Fixed @@ -45,6 +46,22 @@ and [Semantic Versioning](https://semver.org/spec/v2.0.0.html). too. - `docs/versioning.md` gives only what a version number means. Its release steps are now in `docs/releasing.md`. +- Without a clock offset, an inlet sends its clock probes 4 ms apart, and + the burst stops when each probe has its answer. The first offset on + loopback now arrives in about 40 ms. Before, it took about 370 ms. A burst + that finds no offset tries again after 250 ms. The later bursts keep the + configured cadence. +- Each read in a clock probe slot waits only for the time that is left in the + slot. Before, a fixed 40 ms read timeout ended a 64 ms slot early. +- Each data connection uses TCP keepalive: a first probe after 2 s without + traffic, then one each second, and an end after 3 probes without an + answer. On Linux, data without an acknowledgement for 6 s also ends the + connection. A connection that died with no close now ends in about 5 s, + and a recovering inlet connects again. Before, an inlet waited for the + 15 s watchdog, and kept the dead socket while the outlet was visible. +- On Unix, a read that gives `ETIMEDOUT` ends the connection. A read timeout + gives `EAGAIN` there. +- `socket2` now uses its `all` feature. - `release.yml` publishes only the crates that crates.io does not hold at the version. A release that stops partway now completes when a person reruns the failed jobs. Before, the workflow refused a partial state, and a person had diff --git a/crates/labstream-net/Cargo.toml b/crates/labstream-net/Cargo.toml index 26a9f1a..5cf31e3 100644 --- a/crates/labstream-net/Cargo.toml +++ b/crates/labstream-net/Cargo.toml @@ -15,9 +15,11 @@ categories.workspace = true labstream-wire.workspace = true labstream-proto.workspace = true labstream-time.workspace = true -# Only for one socket option that the standard library does not expose. -# `src/udp_server.cpp:60` sets it, and a second outlet needs it. -socket2 = "0.6" +# Only for the socket options that the standard library does not expose. +# `src/udp_server.cpp:60` sets one, and a second outlet needs it. The `all` +# feature gives the keepalive probe count and the TCP user timeout +# (`src/keepalive.rs`). +socket2 = { version = "0.6", features = ["all"] } # Only for `clock_gettime`, which gives the clock that liblsl reads. Windows # has no such function. `clock()` reads the performance counter there, through diff --git a/crates/labstream-net/src/clock.rs b/crates/labstream-net/src/clock.rs index 2e08158..06ef466 100644 --- a/crates/labstream-net/src/clock.rs +++ b/crates/labstream-net/src/clock.rs @@ -42,6 +42,22 @@ pub fn update_min_probes() -> usize { /// (`src/stream_inlet_impl.h:44`). pub const FIRST_OFFSET_TIMEOUT: Duration = Duration::from_secs(5); +/// The gap between two probes while no offset exists. +/// +/// The first burst sends its probes close together, so the first offset +/// arrives in tens of milliseconds and not in a whole burst of +/// [`probe_interval()`] slots. The kept estimate is still the one with the +/// lowest round-trip time. The later bursts use the configured cadence. +/// This is a local choice: the outlet answers each probe as it comes. +pub const FIRST_PROBE_INTERVAL: Duration = Duration::from_millis(4); + +/// The wait before the next burst when a burst found no first offset. +/// +/// A burst without enough answers tries again after this time and not after +/// a full [`update_interval()`]. The wait keeps an outlet that does not answer +/// from a flood of probes. +pub const FIRST_RETRY: Duration = Duration::from_millis(250); + /// The published result of the most recent burst. #[derive(Debug)] pub struct ClockState { @@ -214,6 +230,39 @@ fn probe_socket(target: SocketAddr) -> Option { Some(sock) } +/// Read answers into `burst` until `end`, or until it holds `enough` answers. +/// +/// Each read waits only for the time that is left. A fixed read timeout +/// would end the slot early, or hold a short slot open for too long. +fn collect(sock: &UdpSocket, burst: &mut Burst, end: Instant, enough: Option) { + let mut buf = [0u8; 4096]; + loop { + if enough.is_some_and(|n| burst.accepted() >= n) { + return; + } + let left = end.saturating_duration_since(Instant::now()); + // A read timeout of zero is an error, so the slot ends here. + if left < Duration::from_micros(100) { + return; + } + let _ = sock.set_read_timeout(Some(left)); + match sock.recv_from(&mut buf) { + Ok((n, _)) => { + let t3 = crate::clock(); + if let Some(a) = timesync::parse_answer(&buf[..n]) { + burst.accept(&a, t3); + } + } + Err(ref e) + if e.kind() == std::io::ErrorKind::WouldBlock + || e.kind() == std::io::ErrorKind::TimedOut => {} + // An ICMP error from an earlier probe can show as a read error. + // The slot then waits out its time with a short pause. + Err(_) => std::thread::sleep(left.min(Duration::from_millis(5))), + } + } +} + /// Start a thread that keeps the offset up to date. /// /// The thread sends one burst every [`update_interval()`]. A burst that collects @@ -260,8 +309,16 @@ pub fn start(target: SocketAddr, state: Arc, rng: Arc>) { (*r as u32 as i32).abs() }; let mut burst = Burst::new(wave_id); - let burst_end = - Instant::now() + probe_interval() * probe_count() as u32 + probe_max_rtt(); + // Without an offset, the burst is a fast one. A reset or a new + // target clears the offset, so a recovery also gets a fast burst. + let fast = state.offset().is_none(); + let gap = if fast { + FIRST_PROBE_INTERVAL.min(probe_interval()) + } else { + probe_interval() + }; + let burst_start = Instant::now(); + let burst_end = burst_start + gap * probe_count() as u32 + probe_max_rtt(); for _ in 0..probe_count() { if state.stop.load(Ordering::SeqCst) { @@ -272,34 +329,13 @@ pub fn start(target: SocketAddr, state: Arc, rng: Arc>) { let _ = sock.send_to(&msg, target); // Collect whatever arrives before the next probe goes out. - let slot_end = Instant::now() + probe_interval(); - let mut buf = [0u8; 4096]; - while Instant::now() < slot_end { - match sock.recv_from(&mut buf) { - Ok((n, _)) => { - let t3 = crate::clock(); - if let Some(a) = timesync::parse_answer(&buf[..n]) { - burst.accept(&a, t3); - } - } - Err(_) => break, - } - } + collect(&sock, &mut burst, Instant::now() + gap, None); } - // Drain the tail of the burst. - let mut buf = [0u8; 4096]; - while Instant::now() < burst_end { - match sock.recv_from(&mut buf) { - Ok((n, _)) => { - let t3 = crate::clock(); - if let Some(a) = timesync::parse_answer(&buf[..n]) { - burst.accept(&a, t3); - } - } - Err(_) => break, - } - } + // Collect the late answers. A fast burst stops when each probe + // has its answer. + let all = if fast { Some(probe_count()) } else { None }; + collect(&sock, &mut burst, burst_end, all); if std::env::var_os("LSL_TRACE_CLOCK").is_some() { eprintln!( @@ -319,10 +355,12 @@ pub fn start(target: SocketAddr, state: Arc, rng: Arc>) { } } - // Wait out the rest of the interval. - let rest = update_interval() - .saturating_sub(probe_interval() * probe_count() as u32 + probe_max_rtt()); - let wake = Instant::now() + rest; + // Wait out the rest of the interval. Without an offset, try again + // soon. + let mut wake = burst_start + update_interval(); + if state.offset().is_none() { + wake = wake.min(Instant::now() + FIRST_RETRY); + } while Instant::now() < wake { if state.stop.load(Ordering::SeqCst) { return; diff --git a/crates/labstream-net/src/inlet.rs b/crates/labstream-net/src/inlet.rs index b3c5136..0e65ceb 100644 --- a/crates/labstream-net/src/inlet.rs +++ b/crates/labstream-net/src/inlet.rs @@ -660,10 +660,7 @@ fn read_and_recover( last_data = Instant::now(); None } - Err(ref e) - if e.kind() == std::io::ErrorKind::WouldBlock - || e.kind() == std::io::ErrorKind::TimedOut => - { + Err(ref e) if read_timed_out(e) => { // A source that stopped without closing looks exactly like a // quiet one. The watchdog is what tells them apart. if last_data.elapsed() > watchdog_threshold() { @@ -862,6 +859,21 @@ fn find_again(known: &StreamInfo) -> Found { Found::Moved(Box::new(found.into_iter().next().unwrap())) } +/// True when a read ended because its read timeout ran out. +/// +/// On Unix, a read timeout gives `EAGAIN` (`WouldBlock`). `ETIMEDOUT` +/// (`TimedOut`) there means that the kernel ended the connection, for example +/// after keepalive probes got no answer. That is a lost connection, not a +/// quiet one. Windows gives `WSAETIMEDOUT` for a read timeout, so on Windows +/// both kinds mean a quiet connection. +fn read_timed_out(e: &std::io::Error) -> bool { + match e.kind() { + std::io::ErrorKind::WouldBlock => true, + std::io::ErrorKind::TimedOut => cfg!(not(unix)), + _ => false, + } +} + /// Ask a data port for the whole description. /// /// The exchange is one line out and a document back. The sender closes the @@ -896,6 +908,7 @@ fn handshake( let mut stream = TcpStream::connect_timeout(&addr, timeout)?; stream.set_nodelay(true)?; stream.set_read_timeout(Some(timeout))?; + crate::keepalive::enable(&stream); let mut params = FeedParams::new(&info.uid, info.format); params.hostname = crate::info::hostname(); @@ -1056,6 +1069,17 @@ mod tests { assert_eq!(out.len(), 2); } + #[test] + fn a_kernel_timeout_is_a_lost_connection_on_unix() { + use std::io::{Error, ErrorKind}; + assert!(read_timed_out(&Error::from(ErrorKind::WouldBlock))); + assert_eq!( + read_timed_out(&Error::from(ErrorKind::TimedOut)), + cfg!(not(unix)) + ); + assert!(!read_timed_out(&Error::from(ErrorKind::ConnectionReset))); + } + #[test] fn a_query_socket_takes_the_multicast_ttl_of_the_configuration() { let leg = match Leg::open(false) { diff --git a/crates/labstream-net/src/keepalive.rs b/crates/labstream-net/src/keepalive.rs new file mode 100644 index 0000000..4eee4ff --- /dev/null +++ b/crates/labstream-net/src/keepalive.rs @@ -0,0 +1,83 @@ +//! Find a dead TCP connection in seconds. +//! +//! A connection can die with no close: a cable is pulled, a Wi-Fi link drops, +//! or a NAT forgets the connection. Each side then waits for bytes that never +//! come. liblsl finds this only through its watchdog (15 s), and only when the +//! outlet is also gone from discovery. +//! +//! TCP keepalive sends a probe on a quiet connection, and the kernel ends the +//! connection when the peer does not answer. A peer that is alive answers each +//! probe, so a stream that is quiet but alive is not ended. The probes are an +//! option of the local socket and carry no data, so a liblsl peer sees no +//! difference. + +use std::net::TcpStream; +use std::time::Duration; + +/// How long a connection is quiet before the first keepalive probe. +pub(crate) const IDLE: Duration = Duration::from_secs(2); +/// The time between two keepalive probes. +pub(crate) const INTERVAL: Duration = Duration::from_secs(1); +/// How many probes go unanswered before the kernel ends the connection. +pub(crate) const RETRIES: u32 = 3; +/// How long sent data can stay without an acknowledgement on Linux. +/// +/// Keepalive does not run while data waits for an acknowledgement. An outlet +/// that writes to a dead consumer is in that state. Without this value, Linux +/// retransmits for about 15 minutes before it gives up. +#[cfg(any(target_os = "linux", target_os = "android"))] +pub(crate) const USER_TIMEOUT: Duration = Duration::from_secs(6); + +/// Turn on keepalive with short timers on a connected stream. +/// +/// An error is ignored. The connection then works as before, and the +/// watchdog still applies. +pub(crate) fn enable(stream: &TcpStream) { + let sock = socket2::SockRef::from(stream); + let params = socket2::TcpKeepalive::new() + .with_time(IDLE) + .with_interval(INTERVAL); + #[cfg(any( + target_os = "linux", + target_os = "android", + target_os = "macos", + target_os = "ios", + target_os = "freebsd", + target_os = "windows", + ))] + let params = params.with_retries(RETRIES); + let _ = sock.set_tcp_keepalive(¶ms); + #[cfg(any(target_os = "linux", target_os = "android"))] + let _ = sock.set_tcp_user_timeout(Some(USER_TIMEOUT)); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::TcpListener; + + #[test] + fn a_connected_stream_takes_the_short_timers() { + let listener = match TcpListener::bind("127.0.0.1:0") { + Ok(l) => l, + Err(_) => { + eprintln!("no local network; skipping"); + return; + } + }; + let stream = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + enable(&stream); + let sock = socket2::SockRef::from(&stream); + assert!(sock.keepalive().unwrap()); + // The timers are read back only where this machine checked the + // getters. Elsewhere the test checks that keepalive is on. + #[cfg(any(target_os = "linux", target_os = "android"))] + { + assert_eq!(sock.tcp_keepalive_time().unwrap(), IDLE); + assert_eq!(sock.tcp_keepalive_interval().unwrap(), INTERVAL); + assert_eq!(sock.tcp_keepalive_retries().unwrap(), RETRIES); + } + #[cfg(any(target_os = "linux", target_os = "android"))] + assert_eq!(sock.tcp_user_timeout().unwrap(), Some(USER_TIMEOUT)); + } +} diff --git a/crates/labstream-net/src/lib.rs b/crates/labstream-net/src/lib.rs index 2a02d03..956c975 100644 --- a/crates/labstream-net/src/lib.rs +++ b/crates/labstream-net/src/lib.rs @@ -19,6 +19,7 @@ pub mod config; pub mod desc; pub mod info; pub mod inlet; +mod keepalive; pub mod outlet; pub mod queue; pub mod resolver; diff --git a/crates/labstream-net/src/outlet.rs b/crates/labstream-net/src/outlet.rs index 9b1ebf1..9f5ac98 100644 --- a/crates/labstream-net/src/outlet.rs +++ b/crates/labstream-net/src/outlet.rs @@ -482,6 +482,7 @@ fn tcp_loop(listener: TcpListener, shared: Arc) { fn serve_client(mut stream: TcpStream, shared: Arc) -> std::io::Result<()> { stream.set_nodelay(true)?; stream.set_read_timeout(Some(Duration::from_secs(30)))?; + crate::keepalive::enable(&stream); // Read the request line first. Only a feed request carries a header block, // so a read that waits for the empty line would hang on the other two. diff --git a/crates/labstream-net/tests/clock_live.rs b/crates/labstream-net/tests/clock_live.rs index 2fa2a19..b24c675 100644 --- a/crates/labstream-net/tests/clock_live.rs +++ b/crates/labstream-net/tests/clock_live.rs @@ -249,3 +249,28 @@ fn a_retarget_moves_the_probes_to_another_outlet() { state.stop(); drop(b); } + +/// The first offset decides when the first sample can leave the +/// post-processor, so its delay is a start-up delay of every inlet. +#[test] +fn the_first_offset_arrives_quickly() { + let (_outlet, inlet) = match pair("ClkLiveFast") { + Some(p) => p, + None => { + eprintln!("no local network; skipping"); + return; + } + }; + let t = std::time::Instant::now(); + let got = inlet.wait_for_time_correction(Duration::from_secs(5)); + let took = t.elapsed(); + println!("first offset after {took:?}"); + assert!(got.is_some(), "an offset arrives"); + // The first burst sends its probes 4 ms apart, so on this machine the + // offset arrives in about 40 ms. A burst at the regular cadence takes + // more than 300 ms. The bound is wide, for a slow test machine. + assert!( + took < Duration::from_millis(250), + "the first offset took {took:?}" + ); +}