diff --git a/CHANGELOG.md b/CHANGELOG.md index e2c1ac9..97e6157 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,29 @@ and [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - `docs/releasing.md`. It gives each step of a release to crates.io and to GitHub, a test of the pipeline with no publish, the recovery for each failure, a publish by hand, and a checklist. +- `Outlet::push_samples`, which sends a block with one timestamp for each + sample. +- `SampleQueue::push_owned`, which moves a block into a queue with no copy. +- `ClockState::reset_offset`, `ClockState::retarget`, `ClockState::is_running`, + and `clock::FIRST_OFFSET_TIMEOUT`. + +### Fixed + +- An inlet with `CLOCKSYNC` waits up to 5 s for the first clock offset, as + liblsl does. Before, the first timestamps used an offset of 0, and the real + offset arrived about 0.6 s later as a step. If no offset arrives in 5 s, the + samples continue with no correction. liblsl raises a timeout error there. +- A recovering inlet connects again to the same outlet after its TCP + connection closes. Before, the stream stopped with no error while the + outlet still ran. After a restart of the outlet, the clock probes now go to + the new service port, and the offset resets. +- A consumer connection writes the samples of one push call with one socket + write. Before, each sample had its own write, and a partial chunk could + wait 200 ms. +- The inlet reader moves its buffer once for each read, and takes the queue + lock once for each read. Before, it did both once for each sample. +- A discovery query uses the multicast TTL of the configuration. Before, it + always used 1. ### Changed diff --git a/crates/labstream-capi/src/lib.rs b/crates/labstream-capi/src/lib.rs index 07080f8..2730cd6 100644 --- a/crates/labstream-capi/src/lib.rs +++ b/crates/labstream-capi/src/lib.rs @@ -1295,13 +1295,14 @@ unsafe fn push_chunk_typed( first -= (samples - 1) as f64 / b.info.nominal_srate; } - for k in 0..samples { - let values: Vec = (0..chans).map(|c| build(k * chans + c)).collect(); - b.outlet.push(&Sample { + // One call for the whole chunk, so a consumer gets it with one write. + let block: Vec = (0..samples) + .map(|k| Sample { timestamp: if k == 0 { first } else { LSL_DEDUCED_TIMESTAMP }, - values, - }); - } + values: (0..chans).map(|c| build(k * chans + c)).collect(), + }) + .collect(); + b.outlet.push_samples(block); LSL_NO_ERROR } @@ -1325,14 +1326,15 @@ unsafe fn push_chunk_stamped( return LSL_ARGUMENT_ERROR; } let samples = count_values as usize / chans; - for k in 0..samples { - let values: Vec = (0..chans).map(|c| build(k * chans + c)).collect(); - let t = *stamps.add(k); - b.outlet.push(&Sample { - timestamp: if t == 0.0 { labstream_net::clock() } else { t }, - values, - }); - } + // One call for the whole chunk, so a consumer gets it with one write. + // `push_samples` replaces a timestamp of zero with the current clock. + let block: Vec = (0..samples) + .map(|k| Sample { + timestamp: *stamps.add(k), + values: (0..chans).map(|c| build(k * chans + c)).collect(), + }) + .collect(); + b.outlet.push_samples(block); LSL_NO_ERROR } diff --git a/crates/labstream-net/src/clock.rs b/crates/labstream-net/src/clock.rs index c381ca0..2e08158 100644 --- a/crates/labstream-net/src/clock.rs +++ b/crates/labstream-net/src/clock.rs @@ -7,9 +7,9 @@ use labstream_proto::timesync::{self, Burst}; use labstream_time::OffsetSource; -use std::net::UdpSocket; +use std::net::{SocketAddr, UdpSocket}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Condvar, Mutex}; use std::time::{Duration, Instant}; // The cadence below is the default. A configuration file changes it, and both @@ -36,6 +36,12 @@ pub fn update_min_probes() -> usize { crate::config::get().time_update_min_probes as usize } +/// How long the post-processor waits for the first offset. +/// +/// liblsl passes the same value to `time_correction` +/// (`src/stream_inlet_impl.h:44`). +pub const FIRST_OFFSET_TIMEOUT: Duration = Duration::from_secs(5); + /// The published result of the most recent burst. #[derive(Debug)] pub struct ClockState { @@ -49,8 +55,18 @@ pub struct ClockState { reset: AtomicBool, /// Tells the thread to stop. stop: AtomicBool, + /// True while a probe thread runs. A wait for an offset ends when no + /// thread runs, because no value can then arrive. + alive: AtomicBool, /// How many bursts have published. updates: AtomicU64, + /// The service port that the probes go to. A recovery can change it. + target: Mutex>, + /// Grows by one for each change of the target. A burst that started + /// before a change publishes nothing. + epoch: AtomicU64, + /// Wakes a caller of [`ClockState::wait`]. + signal: (Mutex<()>, Condvar), } const NO_VALUE: u64 = u64::MAX; @@ -63,7 +79,11 @@ impl Default for ClockState { have: AtomicBool::new(false), reset: AtomicBool::new(false), stop: AtomicBool::new(false), + alive: AtomicBool::new(false), updates: AtomicU64::new(0), + target: Mutex::new(None), + epoch: AtomicU64::new(0), + signal: (Mutex::new(()), Condvar::new()), } } } @@ -91,16 +111,37 @@ impl ClockState { self.updates.load(Ordering::SeqCst) } + /// True while a probe thread runs and can publish a value. + pub fn is_running(&self) -> bool { + self.alive.load(Ordering::SeqCst) && !self.stop.load(Ordering::SeqCst) + } + /// Wait until a first measurement arrives. + /// + /// The wait ends early when no probe thread runs, because no value can + /// then arrive. liblsl ends its wait when the connection is lost + /// (`src/time_receiver.cpp:59-61`). pub fn wait(&self, timeout: Duration) -> Option { let end = Instant::now() + timeout; - while Instant::now() < end { + let (lock, cvar) = &self.signal; + let mut guard = lock.lock().unwrap(); + loop { if let Some(v) = self.offset() { return Some(v); } - std::thread::sleep(Duration::from_millis(10)); + if !self.is_running() { + return None; + } + let left = end.saturating_duration_since(Instant::now()); + if left.is_zero() { + return self.offset(); + } + // A short slice guards against a lost wake-up. + guard = cvar + .wait_timeout(guard, left.min(Duration::from_millis(50))) + .unwrap() + .0; } - self.offset() } /// Record that the connection reset, so the post-processor clears its fit. @@ -108,9 +149,37 @@ impl ClockState { self.reset.store(true, Ordering::SeqCst); } + /// Forget the offset after the inlet attached to a new source. + /// + /// The old offset describes a process that has gone. The reset flag goes + /// up only if an offset existed, as in liblsl + /// (`src/time_receiver.cpp:212-218`). + pub fn reset_offset(&self) { + if self.offset().is_some() { + self.reset.store(true, Ordering::SeqCst); + } + self.offset_bits.store(NO_VALUE, Ordering::SeqCst); + self.uncertainty_bits.store(NO_VALUE, Ordering::SeqCst); + } + + /// Send the next bursts to a different service port. + /// + /// A burst that is in progress publishes nothing. liblsl sets the new + /// address in its recovery callback (`src/time_receiver.cpp:26-33`). + pub fn retarget(&self, target: SocketAddr) { + *self.target.lock().unwrap() = Some(target); + self.epoch.fetch_add(1, Ordering::SeqCst); + } + /// Stop the probe thread. pub fn stop(&self) { self.stop.store(true, Ordering::SeqCst); + self.notify(); + } + + fn notify(&self) { + let _guard = self.signal.0.lock().unwrap(); + self.signal.1.notify_all(); } fn publish(&self, offset: f64, uncertainty: f64) { @@ -119,29 +188,68 @@ impl ClockState { .store(uncertainty.to_bits(), Ordering::SeqCst); self.have.store(true, Ordering::SeqCst); self.updates.fetch_add(1, Ordering::SeqCst); + self.notify(); + } +} + +/// Clears the running flag when the probe thread ends, also on a panic. +struct AliveGuard(Arc); + +impl Drop for AliveGuard { + fn drop(&mut self) { + self.0.alive.store(false, Ordering::SeqCst); + self.0.notify(); } } +/// Open a probe socket of the same protocol as the target. +fn probe_socket(target: SocketAddr) -> Option { + let any: std::net::IpAddr = if target.is_ipv6() { + std::net::Ipv6Addr::UNSPECIFIED.into() + } else { + std::net::Ipv4Addr::UNSPECIFIED.into() + }; + let sock = UdpSocket::bind((any, 0)).ok()?; + let _ = sock.set_read_timeout(Some(Duration::from_millis(40))); + Some(sock) +} + /// Start a thread that keeps the offset up to date. /// /// The thread sends one burst every [`update_interval()`]. A burst that collects /// fewer than [`update_min_probes()`] answers publishes nothing, so a lost packet /// leaves the previous value in place instead of replacing it with a guess. -pub fn start(target: std::net::SocketAddr, state: Arc, rng: Arc>) { +/// +/// [`ClockState::retarget`] moves the probes to a different service port. +pub fn start(target: SocketAddr, state: Arc, rng: Arc>) { + *state.target.lock().unwrap() = Some(target); + // The flag goes up before the thread starts. A caller that waits at once + // therefore sees a running thread. + state.alive.store(true, Ordering::SeqCst); std::thread::spawn(move || { - // The probe socket has to speak the protocol of the target. - let any: std::net::IpAddr = if target.is_ipv6() { - std::net::Ipv6Addr::UNSPECIFIED.into() - } else { - std::net::Ipv4Addr::UNSPECIFIED.into() - }; - let sock = match UdpSocket::bind((any, 0)) { - Ok(s) => s, - Err(_) => return, + let _alive = AliveGuard(Arc::clone(&state)); + let mut sock = match probe_socket(target) { + Some(s) => s, + None => return, }; - let _ = sock.set_read_timeout(Some(Duration::from_millis(40))); + let mut bound_v6 = target.is_ipv6(); while !state.stop.load(Ordering::SeqCst) { + let epoch = state.epoch.load(Ordering::SeqCst); + let target = match *state.target.lock().unwrap() { + Some(t) => t, + None => return, + }; + // A recovery can move the stream from IPv4 to IPv6. The socket + // must then change protocol too (`src/time_receiver.cpp:30-32`). + if target.is_ipv6() != bound_v6 { + sock = match probe_socket(target) { + Some(s) => s, + None => return, + }; + bound_v6 = target.is_ipv6(); + } + // A new identifier for each burst, so a late answer from an earlier burst // is dropped. SPEC.md 3.4. let wave_id = { @@ -155,7 +263,7 @@ pub fn start(target: std::net::SocketAddr, state: Arc, rng: Arc, rng: Arc break, } } - let _ = i; } // Drain the tail of the burst. @@ -202,7 +309,11 @@ pub fn start(target: std::net::SocketAddr, state: Arc, rng: Arc= update_min_probes() { + // A burst that started before a change of target measured the old + // source. It publishes nothing. + if burst.accepted() >= update_min_probes() + && state.epoch.load(Ordering::SeqCst) == epoch + { if let (Some(o), Some(u)) = (burst.time_offset(), burst.uncertainty()) { state.publish(o, u); } @@ -226,22 +337,50 @@ pub fn start(target: std::net::SocketAddr, state: Arc, rng: Arc, srate: f64, + /// True after one wait for the first offset ran out. Later calls then do + /// not wait again until a reset clears the offset. + waited_out: bool, + /// The most recent offset that this source gave. + last: f64, } impl LiveSource { /// Build a source for a stream with the given nominal rate. pub fn new(state: Arc, srate: f64) -> Self { - LiveSource { state, srate } + LiveSource { + state, + srate, + waited_out: false, + last: 0.0, + } } } impl OffsetSource for LiveSource { fn correction(&mut self) -> f64 { - // Before the first burst finishes there is no measurement, and a - // correction of zero leaves the timestamp alone. liblsl waits instead - // (`src/time_receiver.cpp:57-76`), and a caller that needs the wait can - // use `ClockState::wait`. - self.state.offset().unwrap_or(0.0) + // Before the first burst finishes there is no measurement. liblsl + // waits up to 5 s for it (`src/stream_inlet_impl.h:44`, + // `src/time_receiver.cpp:57-76`). Without the wait, the first + // timestamps and the start of the jitter fit use no offset, and the + // real offset arrives later as a step. + if let Some(v) = self.state.offset() { + self.waited_out = false; + self.last = v; + return v; + } + if !self.waited_out { + if let Some(v) = self.state.wait(FIRST_OFFSET_TIMEOUT) { + self.last = v; + return v; + } + // liblsl raises `timeout_error` here. This source gives the last + // value and does not wait again, so the samples continue to flow. + self.waited_out = true; + if std::env::var_os("LSL_TRACE_CLOCK").is_some() { + eprintln!("TRACE clock: no offset arrived, the timestamps stay uncorrected"); + } + } + self.last } fn srate(&mut self) -> f64 { @@ -249,6 +388,8 @@ impl OffsetSource for LiveSource { } fn was_reset(&mut self) -> bool { + // A reset clears the offset. The next call to `correction` then waits + // again, because a received value set `waited_out` to false. self.state.reset.swap(false, Ordering::SeqCst) } @@ -289,9 +430,93 @@ mod tests { } #[test] - fn a_missing_measurement_leaves_a_timestamp_alone() { + fn with_no_probe_thread_the_correction_does_not_wait() { + // No thread runs, so no value can arrive. The call returns at once. let s = Arc::new(ClockState::default()); let mut src = LiveSource::new(s, 100.0); + let t = Instant::now(); assert_eq!(src.correction(), 0.0); + assert!(t.elapsed() < Duration::from_secs(1)); + } + + #[test] + fn the_correction_waits_for_the_first_offset() { + let s = Arc::new(ClockState::default()); + // Act as a running probe thread that publishes after 200 ms. + s.alive.store(true, Ordering::SeqCst); + let writer = Arc::clone(&s); + let h = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(200)); + writer.publish(-3.5, 0.001); + }); + let mut src = LiveSource::new(Arc::clone(&s), 100.0); + assert_eq!( + src.correction(), + -3.5, + "the first value is the measured offset" + ); + h.join().unwrap(); + } + + #[test] + fn the_post_processor_never_sees_a_zero_offset_first() { + use labstream_time::{flags, PostProcessor}; + let s = Arc::new(ClockState::default()); + s.alive.store(true, Ordering::SeqCst); + let writer = Arc::clone(&s); + let h = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(100)); + writer.publish(10.0, 0.001); + }); + let mut src = LiveSource::new(Arc::clone(&s), 100.0); + let mut post = PostProcessor::new(); + post.set_options(flags::CLOCKSYNC); + assert_eq!(post.process(1.0, &mut src), 11.0); + h.join().unwrap(); + } + + #[test] + fn a_stopped_thread_ends_the_wait() { + let s = Arc::new(ClockState::default()); + s.alive.store(true, Ordering::SeqCst); + let stopper = Arc::clone(&s); + let h = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(100)); + stopper.stop(); + }); + let t = Instant::now(); + assert_eq!(s.wait(FIRST_OFFSET_TIMEOUT), None); + assert!(t.elapsed() < Duration::from_secs(2)); + h.join().unwrap(); + } + + #[test] + fn a_reset_offset_sets_the_flag_only_after_a_value() { + let s = ClockState::default(); + s.reset_offset(); + assert!(!s.reset.load(Ordering::SeqCst), "no value, so no reset"); + s.publish(1.0, 0.001); + s.reset_offset(); + assert!(s.reset.load(Ordering::SeqCst)); + assert_eq!(s.offset(), None); + assert_eq!(s.uncertainty(), None); + } + + #[test] + fn a_new_source_needs_a_new_wait() { + let s = Arc::new(ClockState::default()); + s.alive.store(true, Ordering::SeqCst); + s.publish(2.0, 0.001); + let mut src = LiveSource::new(Arc::clone(&s), 100.0); + assert_eq!(src.correction(), 2.0); + s.reset_offset(); + let writer = Arc::clone(&s); + let h = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(100)); + writer.publish(7.0, 0.001); + }); + assert!(src.was_reset()); + assert_eq!(src.correction(), 7.0, "the value of the new source"); + h.join().unwrap(); } } diff --git a/crates/labstream-net/src/inlet.rs b/crates/labstream-net/src/inlet.rs index ea1df6e..b3c5136 100644 --- a/crates/labstream-net/src/inlet.rs +++ b/crates/labstream-net/src/inlet.rs @@ -218,18 +218,21 @@ impl Leg { for a in crate::local_ipv4_addresses() { if let Ok(s) = UdpSocket::bind((a, 0)) { let _ = s.set_broadcast(true); - let _ = s.set_multicast_ttl_v4(1); + let _ = s.set_multicast_ttl_v4(cfg.multicast_ttl); let _ = s.set_multicast_loop_v4(true); senders.push(s); } } } + // The query reaches as far as the scope of the configuration. liblsl + // sets the same value (`src/resolve_attempt_udp.cpp:45`). let one = UdpSocket::bind((any, 0))?; if v6 { let _ = one.set_multicast_loop_v6(true); + let _ = socket2::SockRef::from(&one).set_multicast_hops_v6(cfg.multicast_ttl); } else { one.set_broadcast(true)?; - let _ = one.set_multicast_ttl_v4(1); + let _ = one.set_multicast_ttl_v4(cfg.multicast_ttl); let _ = one.set_multicast_loop_v4(true); } senders.push(one); @@ -520,6 +523,10 @@ impl Inlet { /// The timestamp passes through the post-processing stages that /// [`Inlet::set_postprocessing`] selected. With none selected the value is /// the one that the sender wrote. + /// + /// With `CLOCKSYNC` selected, the first call waits for the first offset + /// measurement, for up to 5 s. The timeout of the call does not limit that + /// wait. liblsl waits the same time (`src/stream_inlet_impl.h:44`). pub fn pull(&mut self, timeout: Duration) -> std::io::Result> { match self.queue.pop(timeout) { Some(mut s) => { @@ -542,6 +549,9 @@ impl Inlet { /// `max` samples, because a program that shows live signals must show what /// arrived. /// + /// With `CLOCKSYNC` selected, the first call can wait up to 5 s for the + /// first offset measurement, as [`Inlet::pull`] does. + /// /// This is the call for a fast stream. [`Inlet::pull`] takes the queue lock /// and reads the clock for each sample. A stream of 8 channels at 1000 Hz /// therefore costs 1000 locks each second with `pull`, and one lock for each @@ -611,7 +621,8 @@ fn read_and_recover( conn: Arc, buffer_samples: i32, ) { - let mut tmp = [0u8; 65536]; + let mut tmp = vec![0u8; 65536]; + let mut batch: Vec = Vec::new(); let _ = sock.set_read_timeout(Some(Duration::from_millis(200))); let mut last_data = Instant::now(); @@ -626,36 +637,28 @@ fn read_and_recover( break; } - // Decode everything already buffered before reading more. - let mut decoded_any = false; - loop { - match codec.decode(&carry) { - Ok((mut sample, used)) => { - carry.drain(..used); - sample.timestamp = deducer.apply(sample.timestamp); - queue.push(sample); - decoded_any = true; - } - Err(labstream_wire::Error::Truncated) => break, - Err(_) => { - // A stream that cannot be decoded is not worth recovering. - // The bytes are wrong, not missing. - done.store(true, Ordering::SeqCst); - queue.close(); - return; - } - } - } - if decoded_any { + // Decode everything already buffered before reading more. The whole + // read goes into the queue with one lock. + let decoded = decode_buffered(&codec, &mut carry, &mut deducer, &mut batch); + if !batch.is_empty() { last_data = Instant::now(); + queue.push_owned(&mut batch); + } + if decoded.is_err() { + // A stream that cannot be decoded is not worth recovering. + // The bytes are wrong, not missing. + done.store(true, Ordering::SeqCst); + clock.stop(); + queue.close(); + return; } let ended = match sock.read(&mut tmp) { - Ok(0) => true, + Ok(0) => Some(Ending::Closed), Ok(n) => { carry.extend_from_slice(&tmp[..n]); last_data = Instant::now(); - false + None } Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock @@ -663,26 +666,26 @@ fn read_and_recover( { // A source that stopped without closing looks exactly like a // quiet one. The watchdog is what tells them apart. - last_data.elapsed() > watchdog_threshold() + if last_data.elapsed() > watchdog_threshold() { + Some(Ending::Quiet) + } else { + None + } } - Err(_) => true, + Err(_) => Some(Ending::Closed), }; - if !ended { - continue; - } + let ending = match ended { + Some(e) => e, + None => continue, + }; let trace = std::env::var_os("LSL_TRACE_RECOVER").is_some(); if trace { - eprintln!( - "TRACE recover: the connection ended, recover={}", - conn.recover - ); + eprintln!("TRACE recover: {ending:?}, recover={}", conn.recover); } if !conn.recover { break; } - - conn.connected.store(false, Ordering::SeqCst); let known = conn.info.lock().unwrap().clone(); if trace { eprintln!( @@ -696,68 +699,167 @@ fn read_and_recover( // so liblsl refuses to recover. SPEC.md 10.1. break; } - - let attempt = find_again(&known); - if trace { - eprintln!( - "TRACE recover: found {:?}", - attempt.as_ref().map(|f| &f.uid) - ); + if ending == Ending::Closed { + conn.connected.store(false, Ordering::SeqCst); } - match attempt { - Some(fresh) => { - match handshake(&fresh, Duration::from_secs(5), buffer_samples) { - Ok((s, c, rest)) => { - let _ = &c; - let _ = s.set_read_timeout(Some(Duration::from_millis(200))); - sock = s; - carry = rest; - codec = *c.codec().expect("the handshake set the codec"); - // The old fit and the old offset describe a process - // that has gone. SPEC.md 8.5. - if fresh.uid != known.uid { - // The old fit, the old offset, and the timestamp - // accumulator all describe a process that has gone. - clock.mark_reset(); + + // Try until a connection exists again, or the inlet closes. + loop { + if queue.is_closed() { + conn.connected.store(false, Ordering::SeqCst); + done.store(true, Ordering::SeqCst); + clock.stop(); + return; + } + + // A closed connection to an outlet that still runs is the usual + // case after a network fault. The handshake sends the old + // instance identifier, and the outlet answers with its own, so a + // different outlet on the same port cannot attach. liblsl + // connects again to the same endpoint after an error + // (`src/data_receiver.cpp:331-345`). + if ending == Ending::Closed { + if let Ok((s, c, rest)) = handshake(&known, Duration::from_secs(1), buffer_samples) + { + if trace { + eprintln!("TRACE recover: connected again to {}", known.uid); + } + let _ = s.set_read_timeout(Some(Duration::from_millis(200))); + sock = s; + carry = rest; + codec = *c.codec().expect("the handshake set the codec"); + conn.recoveries.fetch_add(1, Ordering::SeqCst); + conn.connected.store(true, Ordering::SeqCst); + last_data = Instant::now(); + break; + } + } + + let found = find_again(&known); + if trace { + eprintln!("TRACE recover: found {found:?}"); + } + match found { + Found::Moved(fresh) => { + match handshake(&fresh, Duration::from_secs(5), buffer_samples) { + Ok((s, c, rest)) => { + let _ = s.set_read_timeout(Some(Duration::from_millis(200))); + sock = s; + carry = rest; + codec = *c.codec().expect("the handshake set the codec"); + // The old offset, the old fit, and the timestamp + // accumulator all describe a process that has + // gone. The probes go to the new service port + // (`src/time_receiver.cpp:26-33`). + // The new target goes in first, so that a burst to + // the old target cannot publish after the reset. + if let Some(target) = service_endpoint(&fresh) { + clock.retarget(target); + } + clock.reset_offset(); deducer = labstream_wire::TimestampDeducer::new(fresh.nominal_srate); + *conn.info.lock().unwrap() = *fresh; + conn.recoveries.fetch_add(1, Ordering::SeqCst); + conn.connected.store(true, Ordering::SeqCst); + last_data = Instant::now(); + break; } - *conn.info.lock().unwrap() = fresh; - conn.recoveries.fetch_add(1, Ordering::SeqCst); - conn.connected.store(true, Ordering::SeqCst); - last_data = Instant::now(); + Err(_) => std::thread::sleep(Duration::from_millis(500)), } - Err(_) => std::thread::sleep(Duration::from_millis(500)), + } + Found::Same | Found::Nothing if ending == Ending::Quiet => { + // The socket is still open. liblsl does nothing in this + // case and continues to read (`src/inlet_connection.cpp:187-188`). + // The next check comes after one more watchdog period. + last_data = Instant::now(); + break; + } + Found::Same | Found::Nothing => { + // Wait a little so that the outlet does not see a flood of + // connections. liblsl waits the same time + // (`src/data_receiver.cpp:345`). + std::thread::sleep(Duration::from_millis(500)); } } - None => std::thread::sleep(Duration::from_millis(500)), } } conn.connected.store(false, Ordering::SeqCst); done.store(true, Ordering::SeqCst); + // No new offset can matter now. A stop also ends a wait for the first one. + clock.stop(); queue.close(); } +/// Decode every whole sample in `carry` and add it to `out`. +/// +/// The bytes of the decoded samples leave `carry` with one move at the end. +/// A move after each sample costs time in proportion to the square of the +/// buffer size, and a burst of data fills the buffer. +/// +/// The bytes of a sample that is not complete stay in `carry`. +fn decode_buffered( + codec: &labstream_wire::Codec, + carry: &mut Vec, + deducer: &mut labstream_wire::TimestampDeducer, + out: &mut Vec, +) -> Result<(), labstream_wire::Error> { + let mut pos = 0; + let result = loop { + match codec.decode(&carry[pos..]) { + Ok((mut sample, used)) => { + pos += used; + sample.timestamp = deducer.apply(sample.timestamp); + out.push(sample); + } + Err(labstream_wire::Error::Truncated) => break Ok(()), + Err(e) => break Err(e), + } + }; + carry.drain(..pos); + result +} + +/// Why the reader stopped reading the current connection. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Ending { + /// The outlet closed the connection, or the socket reported an error. + Closed, + /// No data arrived for one watchdog period. The socket is still open. + Quiet, +} + +/// The result of a search for a lost stream. +#[derive(Debug)] +enum Found { + /// The stream with the old instance identifier is still visible. + Same, + /// Exactly one stream matches, and it has a new instance identifier. + Moved(Box), + /// No stream matches, or more than one does. + Nothing, +} + /// Find the stream again after its source restarted. /// /// The query comes from [`StreamInfo::recovery_query`]. liblsl accepts the /// result only when exactly one stream matches, because a source identifier /// that is not unique would otherwise attach the inlet to the wrong stream /// (`src/inlet_connection.cpp:188-192`). -fn find_again(known: &StreamInfo) -> Option { - let found = resolve(&known.recovery_query(), 1, Duration::from_secs(2)).ok()?; - if found.is_empty() { - return None; - } - // A result that still carries the old identifier means the source never - // went away, so there is nothing to recover. +fn find_again(known: &StreamInfo) -> Found { + let found = match resolve(&known.recovery_query(), 1, Duration::from_secs(2)) { + Ok(f) => f, + Err(_) => return Found::Nothing, + }; + // A result that still carries the old identifier means that the source + // did not go away (`src/inlet_connection.cpp:186-188`). if found.iter().any(|f| f.uid == known.uid) { - return None; + return Found::Same; } if found.len() != 1 { - return None; + return Found::Nothing; } - Some(found.into_iter().next().unwrap()) + Found::Moved(Box::new(found.into_iter().next().unwrap())) } /// Ask a data port for the whole description. @@ -875,3 +977,97 @@ impl Inlet { self.conn.info.lock().unwrap().clone() } } + +#[cfg(test)] +mod tests { + use super::*; + use labstream_wire::{ByteOrder, Codec, Format, TimestampDeducer, Value, DEDUCED_TIMESTAMP}; + + fn stream(codec: &Codec, n: usize) -> (Vec, Vec) { + let mut bytes = Vec::new(); + let samples: Vec = (0..n) + .map(|k| Sample { + timestamp: if k % 5 == 0 { + 100.0 + k as f64 + } else { + DEDUCED_TIMESTAMP + }, + values: vec![Value::F32(k as f32), Value::F32(-1.0), Value::F32(2.5)], + }) + .collect(); + for s in &samples { + codec.encode(s, &mut bytes).unwrap(); + } + (samples, bytes) + } + + /// Decode the bytes that arrive in reads of the given sizes. + fn decode_in_reads(codec: &Codec, bytes: &[u8], cuts: &[usize]) -> Vec { + let mut carry = Vec::new(); + let mut deducer = TimestampDeducer::new(100.0); + let mut out = Vec::new(); + let mut from = 0; + for &to in cuts.iter().chain(std::iter::once(&bytes.len())) { + carry.extend_from_slice(&bytes[from..to]); + decode_buffered(codec, &mut carry, &mut deducer, &mut out).unwrap(); + from = to; + } + assert!(carry.is_empty(), "no byte stays behind at the end"); + out + } + + #[test] + fn a_cut_at_any_byte_gives_the_same_samples() { + let codec = Codec::new(Format::Float32, 3, ByteOrder::native()); + let (_, bytes) = stream(&codec, 40); + let whole = decode_in_reads(&codec, &bytes, &[]); + assert_eq!(whole.len(), 40); + for cut in 1..bytes.len() { + assert_eq!( + decode_in_reads(&codec, &bytes, &[cut]), + whole, + "a cut at {cut}" + ); + } + // Many small reads, three bytes each. + let cuts: Vec = (3..bytes.len()).step_by(3).collect(); + assert_eq!(decode_in_reads(&codec, &bytes, &cuts), whole); + } + + #[test] + fn the_deduced_timestamps_follow_the_rate() { + let codec = Codec::new(Format::Float32, 3, ByteOrder::native()); + let (_, bytes) = stream(&codec, 6); + let got = decode_in_reads(&codec, &bytes, &[]); + assert_eq!(got[0].timestamp, 100.0); + assert!((got[1].timestamp - 100.01).abs() < 1e-9); + assert_eq!(got[5].timestamp, 105.0); + } + + #[test] + fn a_bad_tag_is_an_error_and_keeps_the_good_samples() { + let codec = Codec::new(Format::Float32, 3, ByteOrder::native()); + let (_, mut bytes) = stream(&codec, 2); + bytes.push(0xEE); + let mut carry = bytes; + let mut deducer = TimestampDeducer::new(100.0); + let mut out = Vec::new(); + assert!(decode_buffered(&codec, &mut carry, &mut deducer, &mut out).is_err()); + assert_eq!(out.len(), 2); + } + + #[test] + fn a_query_socket_takes_the_multicast_ttl_of_the_configuration() { + let leg = match Leg::open(false) { + Ok(l) => l, + Err(_) => { + eprintln!("no local network; skipping"); + return; + } + }; + let want = config::get().multicast_ttl; + for s in &leg.senders { + assert_eq!(s.multicast_ttl_v4().unwrap(), want); + } + } +} diff --git a/crates/labstream-net/src/outlet.rs b/crates/labstream-net/src/outlet.rs index 49bb691..9b1ebf1 100644 --- a/crates/labstream-net/src/outlet.rs +++ b/crates/labstream-net/src/outlet.rs @@ -1,7 +1,7 @@ //! The outlet: a discovery responder and a TCP feed server. use crate::info::StreamInfo; -use crate::queue::Fanout; +use crate::queue::{Fanout, SampleQueue}; use crate::{clock, config}; use labstream_proto::discovery; use labstream_proto::header::parse_block; @@ -266,6 +266,21 @@ fn write_blocking(shared: &Shared, sample: &Sample) { }); } +/// Write several samples to every consumer, with one write for each consumer. +fn write_blocking_many(shared: &Shared, samples: &[Sample]) { + let mut out = Vec::new(); + let mut consumers = shared.sync.lock().unwrap(); + consumers.retain_mut(|c| { + out.clear(); + for s in samples { + if c.codec.encode(s, &mut out).is_err() { + return false; + } + } + c.stream.write_all(&out).is_ok() && c.stream.flush().is_ok() + }); +} + impl Drop for Outlet { fn drop(&mut self) { *self.shared.stop.lock().unwrap() = true; @@ -656,60 +671,89 @@ fn serve_client(mut stream: TcpStream, shared: Arc) -> std::io::Result<( .add(accepted.max_buffer_length.max(1) as usize); // A chunk decides when one write happens. It is not a wire unit. SPEC.md 9. - let chunk = if accepted.max_chunk_length > 0 { + // A value of zero means no limit (`src/tcp_server.cpp:749-753`). This + // outlet has no chunk size of its own. + let max_chunk = if accepted.max_chunk_length > 0 { accepted.max_chunk_length as usize } else { - 1 + usize::MAX }; - let mut pending = Vec::new(); + let stop = || *shared.stop.lock().unwrap(); + let sent = feed(&queue, &codec, &mut stream, max_chunk, &stop, trace); + if trace { + eprintln!("TRACE session over after {sent} samples"); + } + shared.fanout.remove(&queue); + Ok(()) +} + +/// Copy the samples of one consumer queue to its socket. +/// +/// liblsl writes when a sample carries the push-through flag, or when the +/// count reaches the chunk size (`src/tcp_server.cpp:782`). Each push call of +/// liblsl sets the flag on its last sample by default +/// (`src/stream_outlet_impl.h:74`, `src/stream_outlet_impl.h:208-209`). +/// +/// Here, the samples of one push call go into the queue with one lock. The +/// writer takes all samples that wait, and it writes when the count reaches +/// the chunk size or when the queue is empty. An empty queue therefore acts +/// as the push-through flag. When the writer is slower than the source, one +/// write can carry more than one push call. No sample waits for a later push. +/// +/// The result is the number of samples written. +fn feed( + queue: &SampleQueue, + codec: &Codec, + out: &mut W, + max_chunk: usize, + stop: &dyn Fn() -> bool, + trace: bool, +) -> usize { + let mut batch: Vec = Vec::new(); + let mut pending: Vec = Vec::new(); let mut count = 0usize; let mut sent = 0usize; + let mut write = |pending: &mut Vec, count: &mut usize, sent: &mut usize| -> bool { + if trace && *sent == 0 { + eprintln!("TRACE first data write, {} bytes", pending.len()); + } + if let Err(e) = out.write_all(pending) { + if trace { + eprintln!("TRACE write failed after {sent} samples: {e}"); + } + return false; + } + let _ = out.flush(); + *sent += *count; + pending.clear(); + *count = 0; + true + }; loop { - if *shared.stop.lock().unwrap() || queue.is_closed() { + if stop() || queue.is_closed() { break; } - match queue.pop(Duration::from_millis(200)) { - Some(s) => { - if codec.encode(&s, &mut pending).is_err() { - break; - } - count += 1; - if count >= chunk { - if trace && sent == 0 { - eprintln!("TRACE first data write, {} bytes", pending.len()); - } - if let Err(e) = stream.write_all(&pending) { - if trace { - eprintln!("TRACE write failed after {sent} samples: {e}"); - } - break; - } - sent += count; - let _ = stream.flush(); - pending.clear(); - count = 0; - } + batch.clear(); + if queue.pop_many(&mut batch, usize::MAX, Duration::from_millis(200)) == 0 { + continue; + } + for s in &batch { + if codec.encode(s, &mut pending).is_err() { + return sent; } - None => { - // Nothing arrived. Write whatever is waiting so a partial - // chunk does not sit here. - if !pending.is_empty() { - if stream.write_all(&pending).is_err() { - break; - } - let _ = stream.flush(); - pending.clear(); - count = 0; - } + count += 1; + if count >= max_chunk && !write(&mut pending, &mut count, &mut sent) { + return sent; } } + // The queue was empty when the batch came out, so nothing more is on + // the way. Write what is left. + if !pending.is_empty() && !write(&mut pending, &mut count, &mut sent) { + return sent; + } } - if trace { - eprintln!("TRACE session over after {sent} samples"); - } - shared.fanout.remove(&queue); - Ok(()) + sent } fn find_double_crlf(buf: &[u8]) -> Option { @@ -719,34 +763,32 @@ fn find_double_crlf(buf: &[u8]) -> Option { impl Outlet { /// Send several samples that share one timestamp, with one lock. /// - /// The timestamps are the timestamps of [`Outlet::push_chunk`]. The only - /// difference is the number of locks: this takes the set of consumers once - /// for the whole block, and each consumer queue once. `push_chunk` takes both - /// once for each sample. + /// This is the same call as [`Outlet::push_chunk`]. It stays for the + /// programs that use it. + pub fn push_chunk_fast(&self, samples: &[Sample], timestamp: f64) { + self.push_chunk(samples, timestamp); + } + + /// Send several samples, each with its own timestamp, with one lock. /// - /// A block of 500 samples to 2 consumers costs 3 locks here and 1500 locks - /// there. + /// The timestamp rules of [`Outlet::push`] apply to each sample. A + /// consumer receives the samples of one call together, so a consumer + /// connection writes them with one socket write. SPEC.md 9. /// - /// The blocking mode is not batched. That mode waits for each write on - /// purpose, so a block gives no gain. - pub fn push_chunk_fast(&self, samples: &[Sample], timestamp: f64) { + /// A blocking outlet writes the samples to each consumer with one write, + /// before the call returns. + pub fn push_samples(&self, mut samples: Vec) { if samples.is_empty() { return; } + for s in &mut samples { + s.timestamp = self.stamp(s.timestamp); + } if self.shared.sync_mode { - self.push_chunk(samples, timestamp); + write_blocking_many(&self.shared, &samples); return; } - let block: Vec = self - .chunk_timestamps(samples, timestamp) - .map(|(t, s)| Sample { - // The rule of `Outlet::push` runs for each sample, so a - // configuration that orders the current clock still gets it. - timestamp: self.stamp(t), - values: s.values.clone(), - }) - .collect(); - self.shared.fanout.push_many(&block); + self.shared.fanout.push_many(&samples); } /// The timestamp of each sample of a chunk. SPEC.md 9.2. @@ -792,28 +834,106 @@ impl Outlet { /// An application that acquires a block and stamps it on arrival therefore /// gets the right time for every sample in the block. /// - /// [`Outlet::push_chunk_fast`] does the same with one lock for the block. + /// The samples go to each consumer with one lock, and a consumer + /// connection writes them with one socket write. SPEC.md 9. pub fn push_chunk(&self, samples: &[Sample], timestamp: f64) { if samples.is_empty() { return; } - let mut first = if timestamp == 0.0 { - crate::clock() - } else { - timestamp - }; - if self.info.nominal_srate != 0.0 { - first -= (samples.len() - 1) as f64 / self.info.nominal_srate; - } - for (k, s) in samples.iter().enumerate() { - self.push(&Sample { - timestamp: if k == 0 { - first - } else { - labstream_wire::DEDUCED_TIMESTAMP - }, + let block: Vec = self + .chunk_timestamps(samples, timestamp) + .map(|(t, s)| Sample { + timestamp: t, values: s.values.clone(), - }); + }) + .collect(); + self.push_samples(block); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use labstream_wire::{ByteOrder, Format, Value}; + use std::sync::atomic::{AtomicBool, Ordering}; + + /// A writer that counts its calls and keeps the bytes. + #[derive(Default)] + struct Counting { + writes: usize, + bytes: Vec, + } + + impl Write for Counting { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.writes += 1; + self.bytes.extend_from_slice(buf); + Ok(buf.len()) } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + fn samples(n: usize) -> Vec { + (0..n) + .map(|k| Sample { + timestamp: k as f64, + values: vec![Value::F32(k as f32), Value::F32(0.5)], + }) + .collect() + } + + /// Feed the queue that the caller filled, then stop. + fn run(queue: &SampleQueue, max_chunk: usize) -> (usize, Counting) { + let codec = Codec::new(Format::Float32, 2, ByteOrder::native()); + let mut out = Counting::default(); + let once = AtomicBool::new(false); + // The first call to `stop` lets one round run. The second one stops. + let stop = || once.swap(true, Ordering::SeqCst); + let sent = feed(queue, &codec, &mut out, max_chunk, &stop, false); + (sent, out) + } + + #[test] + fn a_chunk_with_no_limit_goes_out_in_one_write() { + let q = SampleQueue::new(1000); + q.push_many(&samples(100)); + let (sent, out) = run(&q, usize::MAX); + assert_eq!(sent, 100); + assert_eq!(out.writes, 1, "one write for the whole chunk"); + } + + #[test] + fn a_chunk_limit_splits_the_writes() { + let q = SampleQueue::new(1000); + q.push_many(&samples(100)); + let (sent, out) = run(&q, 16); + assert_eq!(sent, 100); + // Six full chunks of 16, then the last 4 samples. + assert_eq!(out.writes, 7); + } + + #[test] + fn a_single_sample_is_not_held_back() { + // A limit that one sample does not reach. The sample still goes out + // at once, as the push-through flag of liblsl does. + let q = SampleQueue::new(1000); + q.push_many(&samples(1)); + let (sent, out) = run(&q, 16); + assert_eq!(sent, 1); + assert_eq!(out.writes, 1); + } + + #[test] + fn the_bytes_decode_to_the_same_samples() { + let q = SampleQueue::new(1000); + let sent = samples(50); + q.push_many(&sent); + let (_, out) = run(&q, 7); + let codec = Codec::new(Format::Float32, 2, ByteOrder::native()); + let (got, used) = codec.decode_all(&out.bytes).unwrap(); + assert_eq!(used, out.bytes.len()); + assert_eq!(got, sent); } } diff --git a/crates/labstream-net/src/queue.rs b/crates/labstream-net/src/queue.rs index 6c1a347..e2bb66a 100644 --- a/crates/labstream-net/src/queue.rs +++ b/crates/labstream-net/src/queue.rs @@ -110,6 +110,34 @@ impl SampleQueue { self.room.notify_all(); } + /// Move every sample out of `samples` into the queue, with one lock. + /// + /// The result is the same as [`SampleQueue::push_many`], but no sample is + /// copied. `samples` is empty after the call and keeps its capacity, so + /// the caller can fill it again with no new allocation. + pub fn push_owned(&self, samples: &mut Vec) { + if samples.is_empty() { + return; + } + let mut q = self.inner.lock().unwrap(); + let mut lost = 0u64; + // Samples that the queue drops at once are never moved in. + let skip = samples.len().saturating_sub(self.capacity); + for s in samples.drain(..).skip(skip) { + if q.len() >= self.capacity { + q.pop_front(); + lost += 1; + } + q.push_back(s); + } + drop(q); + lost += skip as u64; + if lost > 0 { + self.dropped.fetch_add(lost, Ordering::SeqCst); + } + self.room.notify_all(); + } + /// Take the oldest sample, waiting up to `timeout`. /// /// Returns `None` when the timeout passes with nothing waiting, or when the @@ -322,6 +350,36 @@ mod tests { assert!(many.pop(Duration::from_millis(10)).is_none()); } + #[test] + fn an_owned_block_write_gives_what_single_writes_give() { + // The same rules as `push_many`, with the existing samples first, then + // a block larger than the queue. + for (before, block_len) in [(0, 50), (30, 50), (30, 600), (0, 100), (99, 1)] { + let one = SampleQueue::new(100); + let owned = SampleQueue::new(100); + for i in 0..before { + one.push(sample(i)); + owned.push(sample(i)); + } + let mut block: Vec = (1000..1000 + block_len).map(sample).collect(); + for s in &block { + one.push(s.clone()); + } + owned.push_owned(&mut block); + assert!(block.is_empty(), "the call takes every sample"); + + assert_eq!(one.len(), owned.len()); + assert_eq!(one.dropped(), owned.dropped(), "{before} then {block_len}"); + while let Some(a) = one.pop(Duration::from_millis(10)) { + let b = owned + .pop(Duration::from_millis(10)) + .expect("the same count"); + assert_eq!(index(&a), index(&b)); + } + assert!(owned.pop(Duration::from_millis(10)).is_none()); + } + } + #[test] fn a_block_read_takes_what_waits_in_order() { let q = SampleQueue::new(100); diff --git a/crates/labstream-net/tests/clock_live.rs b/crates/labstream-net/tests/clock_live.rs index f31361a..2fa2a19 100644 --- a/crates/labstream-net/tests/clock_live.rs +++ b/crates/labstream-net/tests/clock_live.rs @@ -203,3 +203,49 @@ fn a_deduced_timestamp_is_rebuilt_from_the_rate() { } println!("rebuilt {got:?}"); } + +#[test] +fn a_retarget_moves_the_probes_to_another_outlet() { + use labstream_net::clock::{self, ClockState}; + use std::sync::{Arc, Mutex}; + let make = |name: &str| { + let mut i = labstream_net::StreamInfo::new( + name, + "ClockTest", + 1, + labstream_wire::Format::Float32, + 100.0, + ); + i.source_id = format!("{name}_src"); + labstream_net::Outlet::new(i) + }; + let (a, b) = match (make("RetargetA"), make("RetargetB")) { + (Ok(a), Ok(b)) => (a, b), + _ => { + eprintln!("no local network; skipping"); + return; + } + }; + let port = |o: &labstream_net::Outlet| -> std::net::SocketAddr { + format!("127.0.0.1:{}", o.info().v4service_port) + .parse() + .unwrap() + }; + let state = Arc::new(ClockState::default()); + clock::start(port(&a), Arc::clone(&state), Arc::new(Mutex::new(7))); + assert!( + state.wait(Duration::from_secs(5)).is_some(), + "outlet A answers" + ); + + state.retarget(port(&b)); + state.reset_offset(); + // Only outlet B can answer now. A value proves that the probes moved. + drop(a); + assert!( + state.wait(Duration::from_secs(5)).is_some(), + "outlet B answers after the retarget" + ); + state.stop(); + drop(b); +} diff --git a/crates/labstream-net/tests/recover_live.rs b/crates/labstream-net/tests/recover_live.rs new file mode 100644 index 0000000..cef4898 --- /dev/null +++ b/crates/labstream-net/tests/recover_live.rs @@ -0,0 +1,213 @@ +//! Recovery after a lost connection, measured against a live outlet. +//! +//! These tests open real sockets on the loopback interface, so they belong to +//! the reporting tier and not to the merge gate. A failure here can come from +//! the machine. +//! +//! A proxy sits between the inlet and the outlet. The test closes the proxied +//! connection while the outlet continues to run. This is what a network fault +//! does to an inlet on another machine. liblsl connects again to the same +//! endpoint (`src/data_receiver.cpp:331-345`). + +use labstream_net::{Inlet, Outlet, StreamInfo}; +use labstream_wire::{Format, Sample, Value}; +use std::io::{Read, Write}; +use std::net::{Shutdown, TcpListener, TcpStream}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +/// A TCP proxy that forwards each connection to `target`. +/// +/// The proxy keeps a copy of each accepted socket, so the test can close it. +struct Proxy { + port: u16, + accepted: Arc>>, +} + +impl Proxy { + fn start(target: u16) -> Option { + let listener = TcpListener::bind("127.0.0.1:0").ok()?; + let port = listener.local_addr().ok()?.port(); + let accepted = Arc::new(Mutex::new(Vec::new())); + let list = Arc::clone(&accepted); + std::thread::spawn(move || { + for client in listener.incoming() { + let client = match client { + Ok(c) => c, + Err(_) => continue, + }; + let server = match TcpStream::connect(("127.0.0.1", target)) { + Ok(s) => s, + Err(_) => continue, + }; + list.lock().unwrap().push(client.try_clone().unwrap()); + pipe(client.try_clone().unwrap(), server.try_clone().unwrap()); + pipe(server, client); + } + }); + Some(Proxy { port, accepted }) + } + + /// Close every connection that the proxy accepted until now. + fn cut(&self) { + for s in self.accepted.lock().unwrap().drain(..) { + let _ = s.shutdown(Shutdown::Both); + } + } + + fn connections(&self) -> usize { + self.accepted.lock().unwrap().len() + } +} + +/// Copy bytes from one socket to the other until one side closes. +fn pipe(mut from: TcpStream, mut to: TcpStream) { + std::thread::spawn(move || { + let mut buf = [0u8; 16384]; + loop { + match from.read(&mut buf) { + Ok(0) | Err(_) => break, + Ok(n) => { + if to.write_all(&buf[..n]).is_err() { + break; + } + } + } + } + let _ = to.shutdown(Shutdown::Both); + let _ = from.shutdown(Shutdown::Both); + }); +} + +fn sample(k: usize) -> Sample { + Sample { + timestamp: 0.0, + values: vec![Value::F32(k as f32)], + } +} + +fn value(s: &Sample) -> f32 { + match s.values[0] { + Value::F32(v) => v, + _ => panic!("wrong format"), + } +} + +#[test] +fn a_cut_connection_to_a_running_outlet_connects_again() { + let mut info = StreamInfo::new("RecoverLive1", "RecoverTest", 1, Format::Float32, 100.0); + info.source_id = "RecoverLive1_src".to_string(); + let outlet = match Outlet::new(info) { + Ok(o) => o, + Err(_) => { + eprintln!("no local network; skipping"); + return; + } + }; + let real = outlet.info().clone(); + let proxy = match Proxy::start(real.v4data_port) { + Some(p) => p, + None => { + eprintln!("no local network; skipping"); + return; + } + }; + + // The inlet knows the outlet only through the proxy port. + let mut via_proxy = real.clone(); + via_proxy.v4address = "127.0.0.1".to_string(); + via_proxy.v4data_port = proxy.port; + let mut inlet = Inlet::open_recovering(&via_proxy, Duration::from_secs(5), 4096) + .expect("the inlet connects through the proxy"); + assert!(outlet.wait_for_consumers(Duration::from_secs(5))); + + // Samples before the cut. + let mut k = 0usize; + let mut got = Vec::new(); + let end = Instant::now() + Duration::from_secs(5); + while got.is_empty() && Instant::now() < end { + outlet.push(&sample(k)); + k += 1; + let _ = inlet.pull_chunk(&mut got, 1000, Duration::from_millis(20)); + } + assert!(!got.is_empty(), "samples arrive before the cut"); + + proxy.cut(); + let cut_at = k; + + // Push until samples from after the cut arrive. The stale consumer can + // still be registered in the outlet, so a wait for consumers proves + // nothing here. + let mut after = Vec::new(); + let end = Instant::now() + Duration::from_secs(10); + while Instant::now() < end { + outlet.push(&sample(k)); + k += 1; + let mut block = Vec::new(); + let _ = inlet.pull_chunk(&mut block, 1000, Duration::from_millis(20)); + after.extend(block.into_iter().filter(|s| value(s) as usize > cut_at)); + if after.len() >= 10 { + break; + } + } + assert!( + after.len() >= 10, + "samples flow again after the cut, got {}", + after.len() + ); + assert!(inlet.recoveries() >= 1, "the inlet counts the recovery"); + assert!(inlet.is_connected()); + assert_eq!(inlet.current_info().uid, real.uid, "the same outlet"); + assert!( + proxy.connections() >= 1, + "the new connection went through the proxy" + ); +} + +#[test] +fn a_restarted_outlet_is_found_and_the_clock_follows_it() { + let make = || { + let mut info = StreamInfo::new("RecoverLive2", "RecoverTest", 1, Format::Float32, 100.0); + info.source_id = "RecoverLive2_src".to_string(); + Outlet::new(info) + }; + let first = match make() { + Ok(o) => o, + Err(_) => { + eprintln!("no local network; skipping"); + return; + } + }; + let old = first.info().clone(); + let mut inlet = + Inlet::open_recovering(&old, Duration::from_secs(5), 4096).expect("the inlet connects"); + assert!(first.wait_for_consumers(Duration::from_secs(5))); + assert!( + inlet + .wait_for_time_correction(Duration::from_secs(5)) + .is_some(), + "the first source gives an offset" + ); + drop(first); + + let second = make().expect("the second outlet opens"); + let new_uid = second.info().uid.clone(); + assert_ne!(new_uid, old.uid); + + let mut got = Vec::new(); + let end = Instant::now() + Duration::from_secs(20); + let mut k = 0usize; + while got.is_empty() && Instant::now() < end { + second.push(&sample(k)); + k += 1; + let _ = inlet.pull_chunk(&mut got, 1000, Duration::from_millis(20)); + } + assert!(!got.is_empty(), "samples of the second outlet arrive"); + assert_eq!(inlet.current_info().uid, new_uid); + assert!( + inlet + .wait_for_time_correction(Duration::from_secs(5)) + .is_some(), + "the probes reach the service port of the second outlet" + ); +}