From 9b6c19e2509db5dac84ef7b24a8d955f01cdffe6 Mon Sep 17 00:00:00 2001 From: rednayan Date: Mon, 16 Nov 2026 14:47:40 +0100 Subject: [PATCH 1/7] Measure the latency from a push to a pull oracle/latency.c is one C program for the C ABI. It runs as an outlet or as an inlet, and reads its own CLOCK_MONOTONIC. oracle/latency.py links it to each library, runs each pair more than once, and makes a table. Each row names the library that the loader picked. With --control, a known delay of 1000 us must show in the table. docs/latency.md gives the method and the first results for liblsl, main, fix/network-robustness and perf/quick-wins. At 4000 Hz x 64 channels in chunks of 32, main uses 9 to 11 % CPU on each side. The fix branch uses 2 to 3 %, near liblsl. --- conformance/oracle/latency.c | 253 ++++++++++++++++++++++++++++++++++ conformance/oracle/latency.py | 205 +++++++++++++++++++++++++++ docs/latency.md | 119 ++++++++++++++++ 3 files changed, 577 insertions(+) create mode 100644 conformance/oracle/latency.c create mode 100755 conformance/oracle/latency.py create mode 100644 docs/latency.md diff --git a/conformance/oracle/latency.c b/conformance/oracle/latency.c new file mode 100644 index 0000000..bd55098 --- /dev/null +++ b/conformance/oracle/latency.c @@ -0,0 +1,253 @@ +/* Measure the time from a push in one process to a pull in another. + * + * The program runs as an outlet or as an inlet. Link it once against real + * liblsl and once against the Rust library. The same program then measures + * both libraries, and the two can also measure each other. + * + * The clock is CLOCK_MONOTONIC of this program, not the clock of the + * library. The outlet passes the push time as the timestamp of each sample. + * The inlet does no post-processing, so it reads the push time back + * unchanged. Both processes run on one machine and read the same clock. + * + * Channel 0 carries the sample number. The inlet uses it to count lost, + * repeated, and reordered samples. + * + * Each process writes one JSON line to stdout when it ends. + * `oracle/latency.py` runs the pairs and makes the table. + * + * Build: + * cc -O2 -I .build/install/include oracle/latency.c -o BIN -lm -ldl \ + * -L DIR -llsl -Wl,-rpath,DIR + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +#include + +static double now(void) { + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + return (double)t.tv_sec + (double)t.tv_nsec * 1e-9; +} + +static void sleep_until(double t) { + struct timespec ts; + ts.tv_sec = (time_t)t; + ts.tv_nsec = (long)((t - (double)ts.tv_sec) * 1e9); + while (clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &ts, NULL) != 0) { + } +} + +/* User plus system CPU time of this process, in seconds. */ +static double cpu_seconds(void) { + struct rusage r; + getrusage(RUSAGE_SELF, &r); + return (double)r.ru_utime.tv_sec + (double)r.ru_utime.tv_usec * 1e-6 + + (double)r.ru_stime.tv_sec + (double)r.ru_stime.tv_usec * 1e-6; +} + +/* The file of the library that the loader picked. An earlier run used the + * wrong library through a stale RUNPATH, so each row names its library. */ +static const char *library_path(void) { + Dl_info d; + if (dladdr((void *)lsl_library_info, &d) && d.dli_fname) return d.dli_fname; + return "unknown"; +} + +static int cmp_double(const void *a, const void *b) { + double x = *(const double *)a, y = *(const double *)b; + return (x > y) - (x < y); +} + +/* The value at fraction q of a sorted array. */ +static double quantile(const double *v, long n, double q) { + if (n <= 0) return NAN; + long k = (long)(q * (double)(n - 1) + 0.5); + return v[k]; +} + +static void print_library(void) { + printf("\"library\": \"%s\", \"library_info\": \"%s\"", library_path(), lsl_library_info()); +} + +static int run_outlet(const char *name, int channels, double rate, int chunk, double seconds, + double offset_us) { + lsl_streaminfo info = + lsl_create_streaminfo(name, "Latency", channels, rate, cft_float32, name); + lsl_outlet out = lsl_create_outlet(info, 0, 360); + if (!out) { + fprintf(stderr, "ERROR could not create the outlet\n"); + return 1; + } + fprintf(stderr, "READY\n"); + if (!lsl_wait_for_consumers(out, 20.0)) { + fprintf(stderr, "ERROR no consumer in 20 s\n"); + return 1; + } + + long total = (long)(rate * seconds); + long pushes = (total + chunk - 1) / chunk; + float *data = calloc((size_t)chunk * (size_t)channels, sizeof(float)); + double *stamps = calloc((size_t)chunk, sizeof(double)); + double *lag = calloc((size_t)pushes, sizeof(double)); + + /* The inlet starts its reader at the first pull, so the first samples + * wait in the outlet for a short time. Give it time to start. */ + sleep_until(now() + 0.5); + + double cpu0 = cpu_seconds(); + double t0 = now(); + long k = 0; + for (long p = 0; p < pushes; p++) { + long n = total - k < chunk ? total - k : chunk; + double due = t0 + (double)(k + n) / rate; + sleep_until(due); + for (long i = 0; i < n; i++) { + data[i * channels] = (float)(k + i); + for (int c = 1; c < channels; c++) data[i * channels + c] = (float)c; + } + double t = now(); + lag[p] = t - due; + for (long i = 0; i < n; i++) stamps[i] = t - offset_us * 1e-6; + if (n == 1) + lsl_push_sample_ft(out, data, stamps[0]); + else + lsl_push_chunk_ftn(out, data, (unsigned long)(n * channels), stamps); + k += n; + } + double wall = now() - t0; + double cpu = cpu_seconds() - cpu0; + + /* Let the last samples leave before the outlet closes. */ + sleep_until(now() + 1.0); + + qsort(lag, (size_t)pushes, sizeof(double), cmp_double); + printf("{\"role\": \"outlet\", "); + print_library(); + printf(", \"sent\": %ld, \"pushes\": %ld, \"wall_s\": %.3f, \"cpu_pct\": %.3f, " + "\"lag_p50_us\": %.1f, \"lag_p99_us\": %.1f, \"lag_max_us\": %.1f}\n", + total, pushes, wall, 100.0 * cpu / wall, quantile(lag, pushes, 0.5) * 1e6, + quantile(lag, pushes, 0.99) * 1e6, lag[pushes - 1] * 1e6); + fflush(stdout); + lsl_destroy_outlet(out); + return 0; +} + +static int run_inlet(const char *name, int channels, double rate, double seconds, + double warmup) { + lsl_streaminfo found[1]; + if (lsl_resolve_byprop(found, 1, "name", name, 1, 20.0) < 1) { + fprintf(stderr, "ERROR the stream %s was not found\n", name); + return 1; + } + lsl_inlet in = lsl_create_inlet(found[0], 360, 0, 1); + int32_t ec = 0; + lsl_open_stream(in, 20.0, &ec); + if (ec) { + fprintf(stderr, "ERROR open_stream gave %d\n", ec); + return 1; + } + fprintf(stderr, "READY\n"); + + long total = (long)(rate * seconds); + long skip = (long)(rate * warmup); + double *lat = calloc((size_t)total + 1, sizeof(double)); + unsigned char *seen = calloc((size_t)total + 1, 1); + float *buf = calloc((size_t)channels, sizeof(float)); + long got = 0, kept = 0, repeated = 0, reordered = 0, alien = 0; + long last = -1; + double cpu0 = 0, t0 = 0; + + for (;;) { + double ts = lsl_pull_sample_f(in, buf, channels, 3.0, &ec); + double t = now(); + if (ts == 0.0 || ec) break; /* no sample in 3 s: the outlet is gone */ + if (got == 0) { + cpu0 = cpu_seconds(); + t0 = t; + } + got++; + long idx = (long)buf[0]; + if (idx < 0 || idx >= total) { + alien++; + continue; + } + if (seen[idx]) { + repeated++; + continue; + } + seen[idx] = 1; + if (idx < last) reordered++; + last = idx; + if (idx >= skip) lat[kept++] = t - ts; + if (idx == total - 1) break; + } + double wall = now() - t0; + double cpu = cpu_seconds() - cpu0; + + long lost = 0; + for (long i = 0; i < total; i++) + if (!seen[i]) lost++; + + qsort(lat, (size_t)kept, sizeof(double), cmp_double); + printf("{\"role\": \"inlet\", "); + print_library(); + printf(", \"expected\": %ld, \"received\": %ld, \"lost\": %ld, \"repeated\": %ld, " + "\"reordered\": %ld, \"alien\": %ld, \"measured\": %ld, \"wall_s\": %.3f, " + "\"cpu_pct\": %.3f, \"p50_us\": %.1f, \"p90_us\": %.1f, \"p99_us\": %.1f, " + "\"p999_us\": %.1f, \"max_us\": %.1f, \"min_us\": %.1f}\n", + total, got, lost, repeated, reordered, alien, kept, wall, + wall > 0 ? 100.0 * cpu / wall : 0.0, quantile(lat, kept, 0.5) * 1e6, + quantile(lat, kept, 0.9) * 1e6, quantile(lat, kept, 0.99) * 1e6, + quantile(lat, kept, 0.999) * 1e6, kept ? lat[kept - 1] * 1e6 : NAN, + kept ? lat[0] * 1e6 : NAN); + fflush(stdout); + lsl_destroy_inlet(in); + return 0; +} + +int main(int argc, char **argv) { + const char *role = NULL; + const char *name = "LatencyBench"; + int channels = 8; + double rate = 512.0; + int chunk = 1; + double seconds = 10.0; + double warmup = 1.0; + double offset_us = 0.0; + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "outlet") || !strcmp(argv[i], "inlet")) { + role = argv[i]; + continue; + } + if (i + 1 >= argc) { + fprintf(stderr, "the option %s has no value\n", argv[i]); + return 2; + } + if (!strcmp(argv[i], "--name")) name = argv[++i]; + else if (!strcmp(argv[i], "--channels")) channels = atoi(argv[++i]); + else if (!strcmp(argv[i], "--rate")) rate = atof(argv[++i]); + else if (!strcmp(argv[i], "--chunk")) chunk = atoi(argv[++i]); + else if (!strcmp(argv[i], "--seconds")) seconds = atof(argv[++i]); + else if (!strcmp(argv[i], "--warmup")) warmup = atof(argv[++i]); + else if (!strcmp(argv[i], "--offset-us")) offset_us = atof(argv[++i]); + else { + fprintf(stderr, "unknown option %s\n", argv[i]); + return 2; + } + } + if (!role || channels < 1 || rate <= 0 || chunk < 1 || seconds <= 0) { + fprintf(stderr, "usage: latency outlet|inlet [--name N] [--channels C] [--rate R] " + "[--chunk K] [--seconds S] [--warmup S] [--offset-us U]\n"); + return 2; + } + if (!strcmp(role, "outlet")) return run_outlet(name, channels, rate, chunk, seconds, offset_us); + return run_inlet(name, channels, rate, seconds, warmup); +} diff --git a/conformance/oracle/latency.py b/conformance/oracle/latency.py new file mode 100755 index 0000000..48d4897 --- /dev/null +++ b/conformance/oracle/latency.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Measure the push-to-pull latency of each library, and of each mixed pair. + +The script builds `oracle/latency.c` once for each library, runs an outlet +process and an inlet process for each cell, and writes one table. Each cell +runs more than once, because one run of a timing test proves little. + +The control row is liblsl to liblsl. With `--control`, the script also runs +that pair with a known delay in the timestamps, and the table must show the +same delay. If it does not, the instrument is wrong, not the library. + +Usage: + oracle/latency.py + oracle/latency.py --lib oracle=.build/install/lib --lib main=/path/to/dir + oracle/latency.py --loads 512x8x1,4000x64x32 --seconds 10 --repeat 3 + oracle/latency.py --control + +A load `RATExCHANNELSxCHUNK` pushes CHUNK samples in each push call. The +first `--lib` is the reference. The script runs each library against itself, +and the reference against each other library, in both directions. +""" + +import argparse +import json +import os +import statistics +import subprocess +import sys +import tempfile +import threading +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + +DEFAULT_LOADS = "512x8x1,512x8x16,4000x64x1,4000x64x32" + + +def build(source: Path, lib_dir: Path, out: Path) -> None: + cmd = [ + "cc", "-O2", "-I", str(ROOT / ".build" / "install" / "include"), str(source), + "-o", str(out), "-lm", "-ldl", "-L", str(lib_dir), "-llsl", + f"-Wl,-rpath,{lib_dir}", + ] + subprocess.run(cmd, check=True) + + +def wait_ready(proc: subprocess.Popen, what: str, lines: list) -> None: + """Read stderr until the READY line. Keep the rest for an error report.""" + for raw in proc.stderr: + line = raw.decode(errors="replace").rstrip() + lines.append(line) + if line == "READY": + # Drain the rest in the background, so the pipe never fills. + threading.Thread(target=lambda: [lines.append(r.decode(errors="replace").rstrip()) + for r in proc.stderr], daemon=True).start() + return + raise RuntimeError(f"{what}: the peer stopped before READY: {lines[-5:]}") + + +def last_json(text: str) -> dict: + for line in reversed(text.splitlines()): + line = line.strip() + if line.startswith("{"): + return json.loads(line) + raise RuntimeError("no JSON line in the output") + + +def run_pair(out_bin: Path, in_bin: Path, name: str, rate: float, channels: int, chunk: int, + seconds: float, offset_us: float) -> tuple: + common = ["--name", name, "--rate", str(rate), "--channels", str(channels), + "--seconds", str(seconds)] + out_err, in_err = [], [] + outlet = subprocess.Popen([str(out_bin), "outlet", *common, "--chunk", str(chunk), + "--offset-us", str(offset_us)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + wait_ready(outlet, "outlet", out_err) + inlet = subprocess.Popen([str(in_bin), "inlet", *common], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + wait_ready(inlet, "inlet", in_err) + in_out = inlet.stdout.read().decode() + inlet.wait(timeout=seconds + 30) + finally: + if inlet.poll() is None: + inlet.kill() + out_out = outlet.stdout.read().decode() + outlet.wait(timeout=30) + finally: + if outlet.poll() is None: + outlet.kill() + return last_json(out_out), last_json(in_out) + + +def spread(values: list) -> str: + if len(values) == 1: + return f"{values[0]:.0f}" + return f"{statistics.median(values):.0f} ({min(values):.0f}-{max(values):.0f})" + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--lib", action="append", default=None, metavar="NAME=DIR", + help="a library directory that holds liblsl.so (repeat it)") + ap.add_argument("--loads", default=DEFAULT_LOADS) + ap.add_argument("--seconds", type=float, default=10.0) + ap.add_argument("--repeat", type=int, default=3) + ap.add_argument("--no-mixed", action="store_true", + help="run each library only against itself") + ap.add_argument("--control", action="store_true", + help="also measure a known delay of 1000 us with the reference pair") + ap.add_argument("--json", default=None, help="write every run to this file") + args = ap.parse_args() + + libs = args.lib or [f"oracle={ROOT / '.build' / 'install' / 'lib'}", + f"rust={ROOT / '.build' / 'rustlib'}"] + named = [] + for item in libs: + name, _, path = item.partition("=") + named.append((name, Path(path).resolve())) + + work = Path(tempfile.mkdtemp(prefix="latency-")) + bins = {} + for name, path in named: + bins[name] = work / f"latency_{name}" + build(ROOT / "oracle" / "latency.c", path, bins[name]) + + ref = named[0][0] + pairs = [(n, n) for n, _ in named] + if not args.no_mixed: + for n, _ in named[1:]: + pairs += [(ref, n), (n, ref)] + + loads = [] + for spec in args.loads.split(","): + rate, channels, chunk = spec.split("x") + loads.append((float(rate), int(channels), int(chunk))) + + runs = [] + counter = 0 + + def one(out_name, in_name, rate, channels, chunk, offset_us=0.0): + nonlocal counter + counter += 1 + stream = f"Lat{os.getpid()}_{counter}" + o, i = run_pair(bins[out_name], bins[in_name], stream, rate, channels, chunk, + args.seconds, offset_us) + row = {"outlet": out_name, "inlet": in_name, "rate": rate, "channels": channels, + "chunk": chunk, "offset_us": offset_us, "outlet_run": o, "inlet_run": i} + runs.append(row) + print(f" {out_name:>10} -> {in_name:<10} {rate:g} Hz x {channels} ch, chunk {chunk}: " + f"p50 {i['p50_us']:.0f} us, p99 {i['p99_us']:.0f} us, lost {i['lost']}", + file=sys.stderr) + return row + + for rate, channels, chunk in loads: + for out_name, in_name in pairs: + for _ in range(args.repeat): + one(out_name, in_name, rate, channels, chunk) + + control_ok = None + if args.control: + base = [one(ref, ref, 512.0, 8, 1)["inlet_run"]["p50_us"] for _ in range(args.repeat)] + late = [one(ref, ref, 512.0, 8, 1, 1000.0)["inlet_run"]["p50_us"] + for _ in range(args.repeat)] + shift = statistics.median(late) - statistics.median(base) + control_ok = abs(shift - 1000.0) < 100.0 + print(f"\ncontrol: a delay of 1000 us shows as {shift:.0f} us " + f"({'pass' if control_ok else 'FAIL'})") + + print("\n| outlet -> inlet | load | p50 us | p99 us | p99.9 us | max us | lost | " + "CPU outlet % | CPU inlet % |") + print("|---|---|---|---|---|---|---|---|---|") + for rate, channels, chunk in loads: + for out_name, in_name in pairs: + cell = [r for r in runs if r["outlet"] == out_name and r["inlet"] == in_name + and r["rate"] == rate and r["channels"] == channels + and r["chunk"] == chunk and r["offset_us"] == 0.0] + if not cell: + continue + ins = [r["inlet_run"] for r in cell] + outs = [r["outlet_run"] for r in cell] + print(f"| {out_name} -> {in_name} | {rate:g} Hz x {channels}, chunk {chunk} | " + f"{spread([x['p50_us'] for x in ins])} | {spread([x['p99_us'] for x in ins])} | " + f"{spread([x['p999_us'] for x in ins])} | {max(x['max_us'] for x in ins):.0f} | " + f"{sum(x['lost'] + x['repeated'] + x['alien'] for x in ins)} | " + f"{statistics.median(x['cpu_pct'] for x in outs):.1f} | " + f"{statistics.median(x['cpu_pct'] for x in ins):.1f} |") + print(f"\nEach cell: median of {args.repeat} runs, with the range in brackets. " + f"Each run: {args.seconds:g} s. The first second is not measured.") + print("Libraries:") + for name, path in named: + seen = {r["inlet_run"]["library"] for r in runs if r["inlet"] == name} + print(f" {name}: {path} (loaded: {', '.join(sorted(seen))})") + + if args.json: + Path(args.json).write_text(json.dumps(runs, indent=1)) + if control_ok is False: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/latency.md b/docs/latency.md new file mode 100644 index 0000000..5a5e33e --- /dev/null +++ b/docs/latency.md @@ -0,0 +1,119 @@ +# Latency and CPU + +This page gives how to measure the time from a push to a pull, and the +results of the first measurement. Run the measurement again before you claim +that a change makes the library faster. + +## The instrument + +`conformance/oracle/latency.c` is one C program for the C ABI. It runs as an +outlet or as an inlet. The runner `conformance/oracle/latency.py` links it +once to each library, then runs an outlet process and an inlet process for +each cell. + +- **The clock.** The program reads its own `CLOCK_MONOTONIC`, not the clock + of the library. The outlet gives the push time as the timestamp of each + sample. The inlet does no post-processing, and records the pull time minus + the timestamp. +- **The chunk.** All samples of one push call get the push time. The latency + of a chunk is thus the latency of its last sample to leave. +- **The pull.** The inlet calls `lsl_pull_sample_f` with a timeout of 3 s. The + call returns when a sample arrives, so no poll interval adds latency. +- **Loss.** Channel 0 carries the sample number. The inlet counts lost, + repeated, and reordered samples. +- **The library.** Each process reports the file of the library that the + loader picked. A stale `RUNPATH` once made a test use the wrong library. +- **CPU.** Each process reports its user and system time, as a percentage of + one core, over the time that it sends or receives. +- **The schedule.** The outlet waits on an absolute schedule, and reports how + late each push is. This lag is not part of the latency. + +## The control + +The control pair is liblsl to liblsl. With `--control`, the runner also runs +that pair with timestamps that are 1000 µs early. The table must show a delay +that is 1000 µs larger. On the run below, the delay showed as 966 µs. + +## How to run it + +Run these commands from `conformance/`. The oracle build must exist +(`conformance/oracle/README.md`). + +```bash +oracle/latency.py --control +``` + +To compare branches, build the C library of each branch in its own worktree +and target directory. Then give each directory to the runner: + +```bash +oracle/latency.py --lib oracle=.build/install/lib --lib main=DIR1 --lib fix=DIR2 --no-mixed --control +``` + +Do not copy a branch build into `.build/rustlib`. Other scripts read that +directory. + +## The first results + +Machine: AMD Ryzen 5 PRO 3500U, 8 threads, Linux 6.16, loopback, no other +load on purpose. Each cell is the median of 3 runs of 10 s, with the range of +the 3 runs in brackets. The first second of each run is not measured. + +Libraries: + +- `oracle`: liblsl at `e651023c`. +- `main`: labstream `a912cef`. +- `fix`: branch `fix/network-robustness` at `e01e998` (pull request #9). +- `quick`: branch `perf/quick-wins` at `d6880e4` (pull request #10). + +| outlet -> inlet | load | p50 µs | p99 µs | p99.9 µs | max µs | lost | CPU outlet % | CPU inlet % | +|---|---|---|---|---|---|---|---|---| +| oracle -> oracle | 512 Hz x 8, chunk 1 | 102 (93-142) | 327 (201-501) | 419 (231-624) | 2150 | 0 | 5.7 | 3.4 | +| main -> main | 512 Hz x 8, chunk 1 | 96 (94-96) | 212 (204-217) | 465 (260-1808) | 2932 | 0 | 4.2 | 3.2 | +| fix -> fix | 512 Hz x 8, chunk 1 | 96 (94-97) | 204 (200-235) | 254 (246-1011) | 3218 | 0 | 4.2 | 3.2 | +| quick -> quick | 512 Hz x 8, chunk 1 | 94 (92-98) | 204 (194-239) | 419 (308-557) | 2382 | 0 | 4.1 | 3.2 | +| oracle -> oracle | 512 Hz x 8, chunk 16 | 246 (224-247) | 646 (599-648) | 742 (729-779) | 780 | 0 | 0.7 | 0.4 | +| main -> main | 512 Hz x 8, chunk 16 | 290 (287-297) | 667 (614-1054) | 1025 (703-2150) | 2151 | 0 | 2.0 | 1.5 | +| fix -> fix | 512 Hz x 8, chunk 16 | 220 (218-238) | 514 (464-514) | 617 (480-624) | 626 | 0 | 1.1 | 0.5 | +| quick -> quick | 512 Hz x 8, chunk 16 | 220 (214-236) | 531 (495-536) | 696 (676-754) | 756 | 0 | 1.1 | 0.5 | +| oracle -> oracle | 4000 Hz x 64, chunk 1 | 71 (70-72) | 116 (114-117) | 152 (152-164) | 2803 | 0 | 30.4 | 18.4 | +| main -> main | 4000 Hz x 64, chunk 1 | 73 (73-75) | 116 (114-119) | 154 (149-172) | 755 | 0 | 21.2 | 18.4 | +| fix -> fix | 4000 Hz x 64, chunk 1 | 75 (74-76) | 121 (116-165) | 168 (159-2487) | 5199 | 0 | 21.5 | 18.2 | +| quick -> quick | 4000 Hz x 64, chunk 1 | 73 (71-74) | 125 (105-145) | 224 (157-1681) | 3985 | 0 | 20.5 | 17.5 | +| oracle -> oracle | 4000 Hz x 64, chunk 32 | 318 (317-320) | 651 (646-667) | 768 (764-844) | 876 | 0 | 3.3 | 1.7 | +| main -> main | 4000 Hz x 64, chunk 32 | 382 (381-406) | 795 (756-852) | 1108 (892-1309) | 2916 | 0 | 9.3 | 10.8 | +| fix -> fix | 4000 Hz x 64, chunk 32 | 343 (310-355) | 664 (626-699) | 810 (796-1063) | 3097 | 0 | 3.1 | 2.2 | +| quick -> quick | 4000 Hz x 64, chunk 32 | 328 (322-339) | 658 (657-671) | 829 (797-832) | 892 | 0 | 3.2 | 2.2 | + +### What the results show + +- **A chunk push on `main` costs 3 to 5 times the CPU of liblsl.** At + 4000 Hz x 64 channels in chunks of 32, `main` uses 9.3 % and 10.8 %. The + `fix` branch uses 3.1 % and 2.2 %, near liblsl. The `fix` branch writes each chunk + with one write, and locks the inlet queue once for each read. +- **A chunk push on `main` is also slower.** The median is 382 µs on `main`, + 343 µs on `fix`, and 318 µs on liblsl. +- **A push of one sample gives the same latency in all four builds.** The + median is 70 to 100 µs. At 4000 Hz x 64 channels, the labstream outlet uses + about 21 % of one core, and the liblsl outlet uses 30 %. +- **No build lost a sample.** +- **The maximum is noise.** A single value of 2 to 5 ms comes from the + scheduler of the operating system. The p99.9 column is a better measure of + the tail. +- **The `quick` branch does not change the latency of the data path.** It + changes the start of a connection and the end of a dead connection. This + benchmark measures neither. + +## What this benchmark does not measure + +- **Two machines.** All results come from the loopback interface. +- **A slow or lossy network.** Do not add a `tc netem` rule to the `lo` + interface of a shared machine. The rule delays every program on the machine. + Use a network namespace instead. Make two namespaces with a `veth` pair, + add the `netem` rule inside one of them, and run the outlet and the inlet in + different namespaces. Discovery then needs the namespace addresses in + `lab.KnownPeers` of the configuration. This procedure was not run yet. +- **The application.** Phasic and other programs pull in their own loop. Their + poll interval adds to these numbers. +- **The first offset and the recovery.** The live tests in + `crates/labstream-net/tests/` measure these. From c4eb99b257ea33f7d14e57218bb4c2d05b1d0594 Mon Sep 17 00:00:00 2001 From: rednayan Date: Mon, 16 Nov 2026 14:47:40 +0100 Subject: [PATCH 2/7] Propose a feature header between two labstream peers docs/extensions.md proposes the request header Labstream-Features. A labstream outlet answers it, and liblsl ignores it on both sides. The page gives the syntax, the rules, the byte where a feature can change the framing, the first features (seq and shm), the rejected alternatives, and the conformance plan. Nothing is implemented. oracle/headerprobe.py sends a feed request with and without the header to a real outlet. The answer headers and the test pattern must be the same. It passes for liblsl and for labstream main. A probe with a known header (Native-Byte-Order) shows a difference, so the probe can fail. --- CHANGELOG.md | 7 ++ conformance/oracle/headerprobe.py | 152 ++++++++++++++++++++++ docs/extensions.md | 202 ++++++++++++++++++++++++++++++ docs/repository.md | 2 + 4 files changed, 363 insertions(+) create mode 100755 conformance/oracle/headerprobe.py create mode 100644 docs/extensions.md diff --git a/CHANGELOG.md b/CHANGELOG.md index e2c1ac9..f565f2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added +- `docs/latency.md` and `conformance/oracle/latency.py`, with + `conformance/oracle/latency.c`. They measure the time from a push to a pull, + and the CPU of each side, for liblsl and labstream. +- `docs/extensions.md`, a proposal for a feature header between two labstream + peers. Nothing in it is implemented. +- `conformance/oracle/headerprobe.py`. It makes sure that an outlet ignores + the proposed header. - `docs/repository.md`. It gives the layout of the repository, how the workspace joins the crates, the daily commands, CI, and the dependency updates. diff --git a/conformance/oracle/headerprobe.py b/conformance/oracle/headerprobe.py new file mode 100755 index 0000000..2f2cbcf --- /dev/null +++ b/conformance/oracle/headerprobe.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Make sure that an outlet ignores the `Labstream-Features` request header. + +`docs/extensions.md` proposes this header. The proposal depends on one fact: +an outlet that does not know the header gives the same answer with it and +without it. This probe measures that fact on a real outlet. + +The probe starts `oracle/publish.c`, linked to one library. It opens two feed +connections by hand. One request has the extra header, and one does not. +The answer headers and the two test pattern samples must be the same, byte +for byte. + +Usage: + oracle/headerprobe.py # liblsl and the Rust library + oracle/headerprobe.py --lib oracle=.build/install/lib +""" + +import argparse +import os +import re +import socket +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +CHANNELS = 8 + +# The request that liblsl sends (`src/data_receiver.cpp:170-190`), with fixed +# values. +BASE_HEADERS = [ + "Native-Byte-Order: 1234", + "Endian-Performance: 0", + "Has-IEEE754-Floats: 1", + "Supports-Subnormals: 1", + "Value-Size: 4", + "Data-Protocol-Version: 110", + "Max-Buffer-Length: 360", + "Max-Chunk-Length: 0", + "Hostname: probe", + "Source-Id: ", + "Session-Id: default", +] +EXTRA = "Labstream-Features: seq/1 shm/1=0a1b2c" + + +def build(lib_dir: Path, out: Path) -> None: + subprocess.run(["cc", "-I", str(ROOT / ".build" / "install" / "include"), + str(ROOT / "oracle" / "publish.c"), "-o", str(out), "-lm", + "-L", str(lib_dir), "-llsl", f"-Wl,-rpath,{lib_dir}"], check=True) + + +def find_stream(lib_dir: Path, name: str) -> tuple: + """Resolve the stream with pylsl and give its uid and IPv4 data port.""" + code = ( + "import pylsl,sys\n" + f"r=pylsl.resolve_byprop('name','{name}',1,10)\n" + "print(r[0].as_xml() if r else '')\n" + ) + lib = next(p for p in sorted(lib_dir.glob("liblsl.so*"))) + env = dict(os.environ, PYLSL_LIB=str(lib)) + xml = subprocess.run([sys.executable, "-c", code], env=env, capture_output=True, + text=True, timeout=30).stdout + uid = re.search(r"([^<]*)", xml) + port = re.search(r"(\d+)", xml) + if not uid or not port: + raise RuntimeError("the stream was not found") + return uid.group(1), int(port.group(1)) + + +def read_exact(sock: socket.socket, n: int) -> bytes: + out = b"" + while len(out) < n: + part = sock.recv(n - len(out)) + if not part: + raise RuntimeError(f"the connection closed after {len(out)} bytes") + out += part + return out + + +def feed(port: int, uid: str, extra: bool) -> tuple: + """Send one feed request. Give the answer headers and the test pattern.""" + lines = [f"LSL:streamfeed/110 {uid}", *BASE_HEADERS] + if extra: + lines.append(EXTRA) + request = "\r\n".join(lines) + "\r\n\r\n" + with socket.create_connection(("127.0.0.1", port), timeout=5) as s: + s.sendall(request.encode()) + head = b"" + while b"\r\n\r\n" not in head: + part = s.recv(1) + if not part: + raise RuntimeError("the connection closed in the headers") + head += part + # Each test pattern sample: a tag, a timestamp, and the values. + # SPEC.md 6.4. + pattern = read_exact(s, 2 * (1 + 8 + 4 * CHANNELS)) + return head.decode(errors="replace"), pattern + + +def probe(name: str, lib_dir: Path, work: Path) -> bool: + binary = work / f"publish_{name}" + build(lib_dir, binary) + stream = f"HeaderProbe{os.getpid()}{name}" + pub = subprocess.Popen([str(binary), "--name", stream, "--channels", str(CHANNELS), + "--rate", "100", "--seconds", "20"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + time.sleep(1.0) + uid, port = find_stream(lib_dir, stream) + plain_head, plain_pattern = feed(port, uid, extra=False) + with_head, with_pattern = feed(port, uid, extra=True) + finally: + pub.kill() + pub.wait() + same = plain_head == with_head and plain_pattern == with_pattern + leaked = "labstream-features" in with_head.lower() + ok = same and not leaked + print(f" {name:<8} {'pass' if ok else 'FAIL'} answer headers " + f"{'same' if plain_head == with_head else 'DIFFERENT'}, test pattern " + f"{'same' if plain_pattern == with_pattern else 'DIFFERENT'}, " + f"{'a Labstream-Features answer' if leaked else 'no Labstream-Features answer'}") + if not ok: + print(" without:", plain_head.replace("\r\n", " | ")) + print(" with: ", with_head.replace("\r\n", " | ")) + return ok + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--lib", action="append", default=None, metavar="NAME=DIR") + args = ap.parse_args() + libs = args.lib or [f"oracle={ROOT / '.build' / 'install' / 'lib'}", + f"rust={ROOT / '.build' / 'rustlib'}"] + work = Path(tempfile.mkdtemp(prefix="headerprobe-")) + results = [] + for item in libs: + name, _, path = item.partition("=") + results.append(probe(name, Path(path).resolve(), work)) + print() + if all(results): + print("each outlet ignores the Labstream-Features header") + return 0 + print("an outlet reacts to the Labstream-Features header") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/extensions.md b/docs/extensions.md new file mode 100644 index 0000000..24f83d3 --- /dev/null +++ b/docs/extensions.md @@ -0,0 +1,202 @@ +# Proposal: feature negotiation between two labstream peers + +**Status:** proposal. Nothing on this page is implemented. The wire format does +not change until a feature section below is accepted and SPEC.md gets a +section for it. + +## The purpose + +Some improvements need a change to the data channel. Examples are sequence +numbers that make sample loss visible, and a shared-memory transport for two +processes on one machine. liblsl does not know these changes. + +This page defines how a labstream inlet and a labstream outlet agree to use +such a feature. The agreement must leave every other pair unchanged: + +| Outlet | Inlet | Result | +|---|---|---| +| liblsl | liblsl | plain LSL, as today | +| liblsl | labstream | plain LSL | +| labstream | liblsl | plain LSL | +| labstream 0.1.2 or earlier | labstream | plain LSL | +| labstream | labstream | plain LSL, plus each feature that both ends accept | + +## What liblsl does with an extra header + +These facts come from `conformance/liblsl/` at `e651023c`. + +1. The feed server reads each request header line, and acts only on the keys + that it knows. An unknown key has no effect + (`src/tcp_server.cpp:605-633`). +2. The feed client reads each answer header line the same way, and acts only + on the keys that it knows (`src/data_receiver.cpp:215-253`). +3. Both sides cut a header line at the first `;`, and convert the line to + lowercase before they compare the key (`src/tcp_server.cpp:610-613`, + `src/data_receiver.cpp:220-223`). +4. Both sides read a header line into a buffer of 16384 bytes + (`src/tcp_server.cpp:604`, `src/data_receiver.cpp:193`). +5. The labstream parser cuts and converts the same way + (`crates/labstream-proto/src/header.rs:103-107`). + +As a result, a liblsl outlet ignores an extra request header, and a liblsl +inlet ignores an extra answer header. Item 3 limits the syntax of the header. +A value must not contain `;`, and a value must not depend on letter case. + +## The header + +### The request + +A labstream inlet adds one header line to its feed request: + +``` +Labstream-Features: seq/1 shm/1 +``` + +The value is a list of tokens with one space between two tokens. Each token +has this form: + +``` +token = name "/" version [ "=" param ] +name = 1*( a-z / 0-9 / "-" ) +version = 1*( 0-9 ) +param = 1*( 0-9 / a-f ) +``` + +A parameter is hexadecimal, because the header line loses its letter case. +The whole line must stay below 16384 bytes. An inlet can offer more than one +version of a feature, for example `seq/1 seq/2`. + +### The answer + +A labstream outlet that reads the request header adds one line to its answer: + +``` +Labstream-Features: seq/1 +``` + +The answer holds only tokens from the request. For each name, it holds at +most one version. + +### The rules + +1. An inlet sends the request header only when it supports one or more + features. +2. An outlet sends the answer header only when the request held one. +3. An outlet ignores a token that it does not know, or that does not match + the syntax. +4. An inlet ignores an answer token that it did not offer. +5. A feature is on only when the inlet offered it and the outlet answered it. +6. If the answer has no `Labstream-Features` line, no feature is on. A liblsl + outlet and an older labstream outlet give this result. +7. A configuration key can remove a feature from the offer and from the + answer. The name of the key is an open question. + +## Where the framing changes + +After the answer headers, the outlet sends two test pattern samples +(SPEC.md 6). These two samples always use the plain framing of SPEC.md 7. +The inlet compares them exactly as today. + +A feature can change the framing only from the first byte after the second +test pattern sample. Thus the test pattern stays a check of the codec, and a +fault in a feature cannot hide a fault in the codec. + +## The scope of an agreement + +An agreement holds for one connection. A recovery opens a new connection with +a new request, so the set of features can change. An outlet that restarts as +a liblsl program gives an example. + +The inlet must report the features of the current connection, and must show +when the set changes. This works the same way as the clock reset after a +recovery. The API names are an open question. + +## The first features + +Each feature below gets its own section and its own conformance checks +before any code. The text here gives only the requirements that affect the +negotiation. + +### `seq`: sequence numbers + +The outlet gives each sample a number when the application pushes it. One +counter serves the whole outlet, not one counter for each connection. + +This rule is necessary. The two real causes of loss are a full consumer +queue (SPEC.md 8.7) and the gap between a lost connection and the next one. +A counter for each connection starts again after a recovery, so it shows +neither cause. + +The inlet reports the number of samples that it did not receive. The counter +also shows a repeated sample and a sample out of order. + +### `shm`: shared memory on one machine + +The hostname does not prove that the two processes share a machine. Two +machines can have the same name, and a container can have the name of its +host. Thus `shm` needs one more step after the headers: + +1. The inlet offers `shm/1`. +2. The outlet makes a memory segment. It writes a random value into the + segment. It answers `shm/1=`. +3. The inlet opens the segment and reads the random value. +4. The inlet writes one line on the TCP connection: `shm-ok ` or + `shm-no`. +5. If the value matches, the outlet sends the samples through the segment. + Otherwise, or after 1 s with no line, the outlet sends the samples on TCP. + +The outlet does not send data before step 5. The TCP connection stays open +in both cases. It carries the end of the stream and the recovery. + +The segment must allow access only by the user that owns the outlet. The +wake-up method and the code for each platform are not part of this page. + +### A clock model with drift + +A fit of offset and drift is a choice of the inlet alone. The outlet answers +the same time probes as today. Thus this feature needs no negotiation, and +it does not use this header. + +## Alternatives that this proposal rejects + +**A higher `Data-Protocol-Version`, for example 120.** liblsl uses the lower +of the two versions (`src/tcp_server.cpp:641`). Thus a liblsl peer falls back +correctly. But the version numbers belong to liblsl. A future liblsl release +can use the same number for a different format, and the two ends then agree +on different things. A version is also one scale, and the features are +independent. + +**A new request line, for example `LSL:labstream-feed`.** A liblsl server +sends no answer to a request line that it does not know +(`src/tcp_server.cpp:502-531`). The inlet then needs a timeout and a second +connection for each liblsl outlet. That adds a delay to each start and to +each recovery. + +**A field in the stream description.** The description goes to every +consumer, and LabRecorder writes it into each XDF file. A field there changes +the stored data, and it does not describe one connection. + +## The conformance plan + +1. **liblsl outlet, raw request.** Send a feed request with the header to the + oracle outlet. Make sure that the answer and the data bytes are the same + as for the request without the header. `oracle/headerprobe.py` does this + now. It passes for liblsl and for labstream `main`. +2. **liblsl inlet.** Make sure that a labstream outlet sends no + `Labstream-Features` line when the request has none. +3. **Released labstream 0.1.2.** Run it as a peer in both directions. It + knows nothing of the header, so each run must give plain LSL. +4. **Two labstream peers.** Run each feature on and off, and each mix of + offer and answer. +5. **Syntax.** Send tokens with uppercase letters, with `;`, and with a bad + form. Make sure that the outlet ignores each bad token, and keeps the + good ones. +6. **The full interop matrix.** `oracle/interop.py` must give the same result + as before, with the header on. + +## Open questions + +- The final name of the header. +- The name of the configuration key from rule 7. +- The API that reports the features of the current connection. +- The exact bytes of the `seq` framing. This belongs to the `seq` section. diff --git a/docs/repository.md b/docs/repository.md index 7a4ffc4..68e1846 100644 --- a/docs/repository.md +++ b/docs/repository.md @@ -291,6 +291,8 @@ release of a new crate therefore goes by hand, with a token. | `docs/releasing.md` | how to release to crates.io and to GitHub, and how to recover | | `docs/versioning.md` | what a version number means, and the rule for each change | | `docs/conformance.md` | what was measured against liblsl, how, and the result | +| `docs/latency.md` | how to measure the latency and the CPU, and the results | +| `docs/extensions.md` | a proposal for features between two labstream peers | | `SPEC.md` | the protocol, with a citation for every claim | | `CONTRIBUTING.md` | the rules for a change, and the style of the documents | | `CHANGELOG.md` | what changed in each version | From 5caab3bc8c450e10519b6219b872621fe48bcd16 Mon Sep 17 00:00:00 2001 From: rednayan Date: Mon, 16 Nov 2026 15:18:29 +0100 Subject: [PATCH 3/7] Define the seq feature and the feature header in labstream-proto docs/extensions.md now gives the seq/1 feature in full: one counter for the whole outlet, a 9-byte numbering record (tag 0xFE, little-endian number), the count rules on the inlet, and what the inlet reports. The configuration key labstream.Features turns a feature off. labstream-proto gets two pure modules. features reads, writes, offers, answers and agrees the Labstream-Features tokens. seq writes and reads the record, decides when the outlet writes one, and counts the lost samples. FeedParams::to_wire_with, FeedClient::start_with, FeedClient::features and Accepted::to_wire_with add the header to the handshake. With no tokens, each gives the same bytes as before. --- crates/labstream-proto/src/features.rs | 207 +++++++++++++++++++++ crates/labstream-proto/src/feed.rs | 125 ++++++++++++- crates/labstream-proto/src/lib.rs | 2 + crates/labstream-proto/src/negotiate.rs | 20 ++ crates/labstream-proto/src/seq.rs | 233 ++++++++++++++++++++++++ docs/extensions.md | 96 ++++++++-- 6 files changed, 666 insertions(+), 17 deletions(-) create mode 100644 crates/labstream-proto/src/features.rs create mode 100644 crates/labstream-proto/src/seq.rs diff --git a/crates/labstream-proto/src/features.rs b/crates/labstream-proto/src/features.rs new file mode 100644 index 0000000..c9ce1f1 --- /dev/null +++ b/crates/labstream-proto/src/features.rs @@ -0,0 +1,207 @@ +//! The feature header between two labstream peers. `docs/extensions.md`. +//! +//! A labstream inlet offers features in the request header +//! `Labstream-Features`, and a labstream outlet answers the ones that it +//! accepts. liblsl reads only the header keys that it knows, on both sides +//! (`src/tcp_server.cpp:605-633`, `src/data_receiver.cpp:215-253`), so a +//! liblsl peer ignores the header and the connection stays plain LSL. +//! +//! Both parsers cut a header line at the first `;` and convert it to +//! lowercase (`src/tcp_server.cpp:610-613`, `crate::header`). A token is +//! therefore lowercase, and a parameter is hexadecimal. + +/// The header name, as it goes on the wire. +pub const HEADER: &str = "Labstream-Features"; + +/// The header name, as [`crate::Headers`] stores it. +pub const HEADER_KEY: &str = "labstream-features"; + +/// The name of the sequence number feature. `crate::seq`. +pub const SEQ: &str = "seq"; + +/// The features that this library implements, each with its versions. +pub const SUPPORTED: &[(&str, &[u32])] = &[(SEQ, &[1])]; + +/// One feature with a version, and an optional parameter. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Token { + /// The feature name: `a-z`, `0-9`, and `-`. + pub name: String, + /// The version of the feature. + pub version: u32, + /// A hexadecimal parameter, in lowercase. + pub param: Option, +} + +impl Token { + /// A token with no parameter. + pub fn new(name: &str, version: u32) -> Self { + Token { + name: name.to_string(), + version, + param: None, + } + } + + /// Read one token. The result is `None` if the token does not match the + /// syntax. + pub fn parse(text: &str) -> Option { + let (head, param) = match text.split_once('=') { + Some((h, p)) => (h, Some(p)), + None => (text, None), + }; + let (name, version) = head.split_once('/')?; + let name_ok = !name.is_empty() + && name + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'); + let version_ok = !version.is_empty() && version.bytes().all(|b| b.is_ascii_digit()); + if !name_ok || !version_ok { + return None; + } + let param = match param { + Some(p) if !p.is_empty() && p.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) => { + Some(p.to_string()) + } + Some(_) => return None, + None => None, + }; + Some(Token { + name: name.to_string(), + version: version.parse().ok()?, + param, + }) + } + + /// Write the token as it goes on the wire. + pub fn to_wire(&self) -> String { + match &self.param { + Some(p) => format!("{}/{}={}", self.name, self.version, p), + None => format!("{}/{}", self.name, self.version), + } + } +} + +/// Read a header value. A token with a bad form is left out. +pub fn parse(value: &str) -> Vec { + value.split_ascii_whitespace().filter_map(Token::parse).collect() +} + +/// Write a whole header line, with its line end. +pub fn header_line(tokens: &[Token]) -> String { + let list: Vec = tokens.iter().map(Token::to_wire).collect(); + format!("{HEADER}: {}\r\n", list.join(" ")) +} + +/// The tokens that this process offers: each supported feature that the +/// configuration allows, at each supported version. +pub fn offer(allowed: &[String]) -> Vec { + SUPPORTED + .iter() + .filter(|(name, _)| allowed.iter().any(|a| a == name)) + .flat_map(|(name, versions)| versions.iter().map(|v| Token::new(name, *v))) + .collect() +} + +/// The outlet's answer to an offer. +/// +/// The answer holds only offered tokens. For each name it holds at most one +/// version: the highest version that both sides support. A feature that the +/// configuration does not allow is left out. +pub fn answer(offered: &[Token], allowed: &[String]) -> Vec { + let mut out: Vec = Vec::new(); + for (name, versions) in SUPPORTED { + if !allowed.iter().any(|a| a == name) { + continue; + } + let best = offered + .iter() + .filter(|t| t.name == *name && versions.contains(&t.version) && t.param.is_none()) + .max_by_key(|t| t.version); + if let Some(t) = best { + out.push(t.clone()); + } + } + out +} + +/// The features that are on, from what the inlet offered and what the outlet +/// answered. +/// +/// An answer token that was not offered is left out. A feature is on only +/// when both sides named it. +pub fn agreed(offered: &[Token], answered: &[Token]) -> Vec { + let mut out: Vec = Vec::new(); + for t in answered { + let was_offered = offered + .iter() + .any(|o| o.name == t.name && o.version == t.version); + if was_offered && !out.iter().any(|x| x.name == t.name) { + out.push(t.clone()); + } + } + out +} + +/// True when the list holds the feature at this version. +pub fn has(tokens: &[Token], name: &str, version: u32) -> bool { + tokens.iter().any(|t| t.name == name && t.version == version) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn all() -> Vec { + vec![SEQ.to_string()] + } + + #[test] + fn a_token_reads_back_as_it_was_written() { + for text in ["seq/1", "shm/1=0a1b2c", "x-y2/10"] { + assert_eq!(Token::parse(text).unwrap().to_wire(), text); + } + } + + #[test] + fn a_bad_token_is_left_out() { + let got = parse("seq/1 SEQ/1 seq/ /1 seq/x shm/1=XYZ shm/1= seq/2"); + assert_eq!(got, vec![Token::new("seq", 1), Token::new("seq", 2)]); + } + + #[test] + fn the_header_line_is_lowercase_safe_and_has_no_semicolon() { + let line = header_line(&[Token::new("seq", 1)]); + assert_eq!(line, "Labstream-Features: seq/1\r\n"); + // Both parsers cut the line at `;` and convert it to lowercase. The + // value must read the same after that. + let value = line.split_once(':').unwrap().1.trim(); + assert!(!value.contains(';')); + assert_eq!(parse(&value.to_ascii_lowercase()), parse(value)); + } + + #[test] + fn the_answer_holds_only_offered_and_allowed_features() { + let offered = parse("seq/1 seq/7 shm/1=00ff"); + assert_eq!(answer(&offered, &all()), vec![Token::new("seq", 1)]); + assert!(answer(&offered, &[]).is_empty()); + assert!(answer(&parse("shm/1"), &all()).is_empty()); + assert!(answer(&parse("seq/9"), &all()).is_empty()); + } + + #[test] + fn a_feature_is_on_only_when_both_sides_name_it() { + let offered = offer(&all()); + assert_eq!(offered, vec![Token::new("seq", 1)]); + assert!(has(&agreed(&offered, &parse("seq/1")), SEQ, 1)); + assert!(agreed(&offered, &parse("seq/2")).is_empty()); + assert!(agreed(&offered, &[]).is_empty()); + assert!(agreed(&[], &parse("seq/1")).is_empty()); + assert_eq!(agreed(&offered, &parse("seq/1 seq/1")).len(), 1); + } + + #[test] + fn the_configuration_can_turn_every_feature_off() { + assert!(offer(&[]).is_empty()); + } +} diff --git a/crates/labstream-proto/src/feed.rs b/crates/labstream-proto/src/feed.rs index 14a46e7..8ac972b 100644 --- a/crates/labstream-proto/src/feed.rs +++ b/crates/labstream-proto/src/feed.rs @@ -3,6 +3,7 @@ //! The machine takes bytes and returns bytes. It opens no socket and reads no //! clock, so a test drives every state with a byte slice. +use crate::features::{self, Token}; use crate::header::{parse_block, Error, Headers, StatusLine}; use crate::negotiate::{BIG, LITTLE}; use labstream_wire::{ByteOrder, Codec, Format, Sample, TEST_PATTERN_OFFSETS}; @@ -99,6 +100,20 @@ impl FeedParams { s.push_str("\r\n"); s.into_bytes() } + + /// Write the request with a `Labstream-Features` line, after the fields + /// of liblsl. With no tokens, the request is the same as + /// [`FeedParams::to_wire`]. `docs/extensions.md`. + pub fn to_wire_with(&self, features: &[Token]) -> Vec { + let mut bytes = self.to_wire(); + if !features.is_empty() { + // Put the line before the empty line that ends the block. + bytes.truncate(bytes.len() - 2); + bytes.extend_from_slice(features::header_line(features).as_bytes()); + bytes.extend_from_slice(b"\r\n"); + } + bytes + } } /// The parameters that the answer settled. @@ -135,12 +150,29 @@ pub struct FeedClient { state: State, agreement: Option, codec: Option, + /// The features that the request offered. + offer: Vec, + /// The features that are on, once the headers arrive. + features: Vec, } impl FeedClient { /// Build a client and return the bytes to send. pub fn start(params: FeedParams, format: Format, channels: usize) -> (Self, Vec) { - let bytes = params.to_wire(); + Self::start_with(params, format, channels, Vec::new()) + } + + /// Build a client that offers features, and return the bytes to send. + /// + /// With no tokens, this is the same as [`FeedClient::start`]. + /// `docs/extensions.md`. + pub fn start_with( + params: FeedParams, + format: Format, + channels: usize, + offer: Vec, + ) -> (Self, Vec) { + let bytes = params.to_wire_with(&offer); ( FeedClient { params, @@ -149,11 +181,19 @@ impl FeedClient { state: State::AwaitingHeaders, agreement: None, codec: None, + offer, + features: Vec::new(), }, bytes, ) } + /// The features that are on for this connection. Empty until the headers + /// arrive, and empty when the outlet gave no answer for them. + pub fn features(&self) -> &[Token] { + &self.features + } + /// The current state. pub fn state(&self) -> State { self.state @@ -201,6 +241,13 @@ impl FeedClient { }; let agreement = self.read_answer(&headers)?; + // An outlet that does not know the header sends no answer for it, so + // no feature is on. + let answered = headers + .get(features::HEADER_KEY) + .map(features::parse) + .unwrap_or_default(); + self.features = features::agreed(&self.offer, &answered); let order = agreement.order; self.codec = Some( Codec::new(self.format, self.channels, order) @@ -353,7 +400,7 @@ pub enum TestPatternError { mod tests { use super::*; use crate::header::parse_block; - use crate::negotiate::{negotiate, FeedRequest, Outcome, ServerStream}; + use crate::negotiate::{negotiate, Accepted, FeedRequest, Outcome, ServerStream}; fn handshake( format: Format, @@ -418,6 +465,80 @@ mod tests { assert!(text.ends_with("\r\n\r\n")); } + /// Run the feature exchange through both header parsers, as the outlet + /// and the inlet do. `answer_features` is the server side decision. + fn feature_handshake(offer: Vec, allowed: &[String]) -> FeedClient { + let p = FeedParams::new("uid-1", Format::Float32); + let (mut client, request) = FeedClient::start_with(p, Format::Float32, 2, offer); + let block_start = request.iter().position(|&b| b == b'\n').unwrap() + 1; + let (headers, _) = parse_block(&request[block_start..]).expect("a block"); + let srv = ServerStream::new("uid-1", Format::Float32, LITTLE, 1000.0); + let req = FeedRequest::from_headers(110, "uid-1", &headers, srv.channel_bytes); + let accepted = match negotiate(&req, &srv) { + Outcome::Accept(a) => a, + Outcome::Refuse(s) => panic!("refused: {}", s.code), + }; + let answered = headers + .get(features::HEADER_KEY) + .map(|v| features::answer(&features::parse(v), allowed)); + let answer = accepted.to_wire_with(110, "uid-1", answered.as_deref()); + client.on_headers(answer.as_bytes()).expect("headers"); + client + } + + #[test] + fn a_request_with_no_offer_is_the_plain_request() { + let p = FeedParams::new("abc", Format::Float32); + assert_eq!(p.to_wire_with(&[]), p.to_wire()); + } + + #[test] + fn the_offer_goes_last_in_the_block() { + let p = FeedParams::new("abc", Format::Float32); + let text = String::from_utf8(p.to_wire_with(&[Token::new("seq", 1)])).unwrap(); + assert!(text.ends_with("Session-Id: default\r\nLabstream-Features: seq/1\r\n\r\n")); + assert!(text.starts_with(&String::from_utf8(p.to_wire()).unwrap()[..40])); + } + + #[test] + fn two_labstream_peers_agree_on_seq() { + let all = vec![features::SEQ.to_string()]; + let c = feature_handshake(features::offer(&all), &all); + assert!(features::has(c.features(), features::SEQ, 1)); + } + + #[test] + fn an_outlet_that_allows_nothing_turns_seq_off() { + let all = vec![features::SEQ.to_string()]; + let c = feature_handshake(features::offer(&all), &[]); + assert!(c.features().is_empty()); + } + + #[test] + fn an_answer_with_no_feature_line_turns_seq_off() { + // A liblsl outlet and an older labstream outlet answer this way. + let all = vec![features::SEQ.to_string()]; + let p = FeedParams::new("uid-1", Format::Float32); + let (mut c, _) = FeedClient::start_with(p, Format::Float32, 2, features::offer(&all)); + c.on_headers(b"LSL/110 200 OK\r\nUID: uid-1\r\nByte-Order: 1234\r\nSuppress-Subnormals: 0\r\nData-Protocol-Version: 110\r\n\r\n") + .expect("headers"); + assert!(c.features().is_empty()); + } + + #[test] + fn an_answer_with_no_request_line_has_no_feature_line() { + let a = Accepted { + data_protocol_version: 110, + byte_order: LITTLE, + reverse_byte_order: false, + suppress_subnormals: false, + max_buffer_length: 360, + max_chunk_length: 0, + }; + assert_eq!(a.to_wire_with(110, "u", None), a.to_wire(110, "u")); + assert!(a.to_wire_with(110, "u", Some(&[])).contains("Labstream-Features: \r\n")); + } + #[test] fn a_full_handshake_reaches_the_streaming_state() { let c = handshake(Format::Float32, 8, LITTLE, 0.0); diff --git a/crates/labstream-proto/src/lib.rs b/crates/labstream-proto/src/lib.rs index 3bc9a8d..d579a9e 100644 --- a/crates/labstream-proto/src/lib.rs +++ b/crates/labstream-proto/src/lib.rs @@ -24,9 +24,11 @@ pub mod canon; pub mod discovery; +pub mod features; pub mod feed; pub mod header; pub mod negotiate; +pub mod seq; pub mod timesync; pub use header::{Error, Headers, StatusAction, StatusLine}; diff --git a/crates/labstream-proto/src/negotiate.rs b/crates/labstream-proto/src/negotiate.rs index 922f4ca..dbc8b9f 100644 --- a/crates/labstream-proto/src/negotiate.rs +++ b/crates/labstream-proto/src/negotiate.rs @@ -176,6 +176,26 @@ impl Accepted { s.push_str("\r\n"); s } + + /// Write the answer with a `Labstream-Features` line. + /// + /// `features` is `None` when the request had no such header. The answer + /// is then the same as [`Accepted::to_wire`]. A request with the header + /// gets the line, also when no feature is on. `docs/extensions.md`. + pub fn to_wire_with( + &self, + configured_version: i32, + uid: &str, + features: Option<&[crate::features::Token]>, + ) -> String { + let mut s = self.to_wire(configured_version, uid); + if let Some(tokens) = features { + s.truncate(s.len() - 2); + s.push_str(&crate::features::header_line(tokens)); + s.push_str("\r\n"); + } + s + } } /// True when the format carries a subnormal value. diff --git a/crates/labstream-proto/src/seq.rs b/crates/labstream-proto/src/seq.rs new file mode 100644 index 0000000..aca7b4d --- /dev/null +++ b/crates/labstream-proto/src/seq.rs @@ -0,0 +1,233 @@ +//! The sequence numbers of the `seq/1` feature. `docs/extensions.md`. +//! +//! A numbering record can stand between two samples. It gives the number of +//! the next sample. The outlet writes one record at the start of each +//! connection, and one more before each sample whose number is not one more +//! than the number of the sample before it. The inlet counts the numbers +//! that never arrived. + +/// The first byte of a numbering record. +/// +/// The sample tags are 1 and 2 (`src/sample.h:19-20`, SPEC.md 7.1). This +/// value is neither, so the first byte tells a record from a sample. +pub const TAG: u8 = 0xFE; + +/// The length of a numbering record in bytes. +pub const RECORD_LEN: usize = 9; + +/// Append a record that gives `number` to the next sample. +/// +/// The number is always little-endian. The byte order of the connection does +/// not apply to it. +pub fn write_record(out: &mut Vec, number: u64) { + out.push(TAG); + out.extend_from_slice(&number.to_le_bytes()); +} + +/// What starts at a position in the sample stream. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Next { + /// A complete record, with its number. + Record(u64), + /// A record that is cut. The caller reads more bytes. + Incomplete, + /// No record. The bytes start a sample, or nothing. + Sample, +} + +/// Look at the start of `buf` for a record. +pub fn peek(buf: &[u8]) -> Next { + match buf.first() { + Some(&TAG) if buf.len() >= RECORD_LEN => { + let mut b = [0u8; 8]; + b.copy_from_slice(&buf[1..RECORD_LEN]); + Next::Record(u64::from_le_bytes(b)) + } + Some(&TAG) => Next::Incomplete, + _ => Next::Sample, + } +} + +/// The count of lost samples on the inlet side. +/// +/// The counter keeps the number that it expects next. A record above that +/// number adds the difference to the loss. Each sample moves the expected +/// number by one. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Counter { + expected: Option, + lost: u64, + backwards: u64, +} + +impl Counter { + /// A counter with no expected number yet. + pub fn new() -> Self { + Counter::default() + } + + /// Read a record. + /// + /// The first record sets the expected number and counts no loss. A number + /// below the expected number counts no loss: it adds one to + /// [`Counter::backwards`], and the counter takes the new number. + pub fn record(&mut self, number: u64) { + if let Some(e) = self.expected { + if number > e { + self.lost = self.lost.saturating_add(number - e); + } else if number < e { + self.backwards = self.backwards.saturating_add(1); + } + } + self.expected = Some(number); + } + + /// Count one sample. + pub fn sample(&mut self) { + if let Some(e) = self.expected { + self.expected = Some(e.saturating_add(1)); + } + } + + /// Forget the expected number. A new outlet has a new counter. + pub fn restart(&mut self) { + self.expected = None; + } + + /// The samples that never arrived, since the counter was made. + pub fn lost(&self) -> u64 { + self.lost + } + + /// How many records gave a number below the expected one. + pub fn backwards(&self) -> u64 { + self.backwards + } + + /// The number that the counter expects next, if it has one. + pub fn expected(&self) -> Option { + self.expected + } +} + +/// The record decision on the outlet side, for one connection. +/// +/// The writer calls [`Numbering::before`] with the number of each sample that +/// it is about to write. The result tells whether a record goes first. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Numbering { + next: Option, +} + +impl Numbering { + /// A new connection, which starts with a record. + pub fn new() -> Self { + Numbering::default() + } + + /// Write a record into `out` if the sample with `number` needs one, then + /// expect the number after it. + pub fn before(&mut self, number: u64, out: &mut Vec) { + if self.next != Some(number) { + write_record(out, number); + } + self.next = Some(number.wrapping_add(1)); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_record_reads_back() { + for n in [0u64, 1, 255, 1 << 40, u64::MAX] { + let mut out = Vec::new(); + write_record(&mut out, n); + assert_eq!(out.len(), RECORD_LEN); + assert_eq!(peek(&out), Next::Record(n)); + } + } + + #[test] + fn the_number_is_little_endian() { + let mut out = Vec::new(); + write_record(&mut out, 0x0102_0304_0506_0708); + assert_eq!(out, [0xFE, 8, 7, 6, 5, 4, 3, 2, 1]); + } + + #[test] + fn a_cut_record_asks_for_more_bytes() { + let mut out = Vec::new(); + write_record(&mut out, 42); + for cut in 1..RECORD_LEN { + assert_eq!(peek(&out[..cut]), Next::Incomplete); + } + assert_eq!(peek(&[]), Next::Sample); + assert_eq!(peek(&[1, 0, 0]), Next::Sample); + assert_eq!(peek(&[2]), Next::Sample); + } + + #[test] + fn the_first_record_counts_no_loss() { + let mut c = Counter::new(); + c.record(1000); + assert_eq!(c.lost(), 0); + assert_eq!(c.expected(), Some(1000)); + } + + #[test] + fn a_gap_counts_the_missing_numbers() { + let mut c = Counter::new(); + c.record(10); + c.sample(); // 10 + c.sample(); // 11 + c.record(15); // 12, 13, 14 are lost + c.sample(); + assert_eq!(c.lost(), 3); + assert_eq!(c.expected(), Some(16)); + } + + #[test] + fn a_number_that_goes_back_counts_no_loss_and_does_not_underflow() { + let mut c = Counter::new(); + c.record(100); + c.sample(); + c.record(5); + assert_eq!(c.lost(), 0); + assert_eq!(c.backwards(), 1); + assert_eq!(c.expected(), Some(5)); + } + + #[test] + fn a_restart_takes_a_new_baseline() { + let mut c = Counter::new(); + c.record(50); + c.restart(); + c.record(0); + c.record(3); + assert_eq!(c.lost(), 3); + assert_eq!(c.backwards(), 0); + } + + #[test] + fn the_writer_and_the_counter_agree() { + // The numbers that a writer sends, with gaps from dropped samples. + let numbers = [7u64, 8, 9, 12, 13, 20, 21, 22]; + let mut w = Numbering::new(); + let mut c = Counter::new(); + let mut records = 0; + for n in numbers { + let mut out = Vec::new(); + w.before(n, &mut out); + if let Next::Record(r) = peek(&out) { + records += 1; + c.record(r); + } + c.sample(); + } + assert_eq!(records, 3, "one at the start and one at each gap"); + assert_eq!(c.lost(), 2 + 6); + assert_eq!(c.expected(), Some(23)); + } +} diff --git a/docs/extensions.md b/docs/extensions.md index 24f83d3..3a5de76 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -1,8 +1,8 @@ # Proposal: feature negotiation between two labstream peers -**Status:** proposal. Nothing on this page is implemented. The wire format does -not change until a feature section below is accepted and SPEC.md gets a -section for it. +**Status:** proposal. The header and the `seq` feature are implemented on +the branch `feat/seq`. The other features are not implemented. SPEC.md gets +a section for a feature only after the feature is accepted. ## The purpose @@ -88,8 +88,11 @@ most one version. 5. A feature is on only when the inlet offered it and the outlet answered it. 6. If the answer has no `Labstream-Features` line, no feature is on. A liblsl outlet and an older labstream outlet give this result. -7. A configuration key can remove a feature from the offer and from the - answer. The name of the key is an open question. +7. The configuration key `labstream.Features` lists the features that this + process can use, for example `{seq}`. The default lists every feature that + the library has. An inlet offers only the listed features, and an outlet + answers only the listed features. The value `{}` turns every feature off. + liblsl does not read this key. ## Where the framing changes @@ -119,16 +122,81 @@ negotiation. ### `seq`: sequence numbers -The outlet gives each sample a number when the application pushes it. One -counter serves the whole outlet, not one counter for each connection. +**Status:** implemented on the branch `feat/seq` +(`crates/labstream-proto/src/seq.rs`). The token is `seq/1`. -This rule is necessary. The two real causes of loss are a full consumer -queue (SPEC.md 8.7) and the gap between a lost connection and the next one. -A counter for each connection starts again after a recovery, so it shows -neither cause. +#### The purpose -The inlet reports the number of samples that it did not receive. The counter -also shows a repeated sample and a sample out of order. +LSL has no sign of a lost sample (SPEC.md 8.7). An application can only +guess a loss from a gap in the timestamps. `seq` gives each sample a number, +so the inlet can count the samples that it did not receive. + +#### The numbers + +1. The outlet gives each pushed sample the next number of one counter. The + counter starts at 0 when the outlet opens. +2. One counter serves the whole outlet, not one counter for each connection. + A sample that the outlet pushes with no consumer also takes a number. +3. The two real causes of loss are a full consumer queue in the outlet + (SPEC.md 8.7) and the gap between a lost connection and the next one. A + counter for each connection starts again after a recovery, so it shows + neither cause. That is the reason for rule 2. + +#### The record + +When `seq/1` is on, a numbering record can stand between two samples. It +has 9 bytes: + +``` +u8 tag 0xFE +u64 number little-endian, the number of the next sample +``` + +1. The tag values 1 and 2 start a sample (`src/sample.h:19-20`, SPEC.md + 7.1). The tag `0xFE` is not a sample tag, so a reader can tell a record + from a sample by its first byte. +2. The number is always little-endian. The `Byte-Order` of the connection + does not apply to it. +3. On each connection, the first record comes directly after the second test + pattern sample. Each connection thus starts with an absolute number. +4. After that, the outlet writes a record only before a sample whose number + is not one more than the number of the sample before it. A connection + with no loss carries one record and no more. +5. A record is not a sample. It does not count toward `Max-Chunk-Length`. +6. A record that is cut at the end of a read is not an error. The reader + waits for the rest of the bytes, as for a cut sample. + +#### The count + +The inlet keeps the number that it expects next. + +1. The first record after the inlet opens sets the expected number, and + counts no loss. The inlet has no number from before that point. +2. A later record with a number above the expected number adds the + difference to the count of lost samples. +3. A later record with a number below the expected number counts no loss. + The inlet adds one to a separate count of numbers that went back, and + takes the new number. The arithmetic must not underflow. +4. Each sample adds one to the expected number. +5. A recovery to the same outlet (the same `uid`) keeps the expected number. + The first record of the new connection then shows the gap. +6. A recovery to a new outlet (a new `uid`) starts again at rule 1, because + the new outlet has a new counter. + +#### What the inlet reports + +- `Inlet::samples_lost()` in `labstream-net` and `Inlet::lost()` in + `labstream` give the count of lost samples since the inlet opened. +- The value is `None` when the current connection does not use `seq`. A + liblsl outlet gives this result. `None` means "unknown". It does not mean + "no loss". +- Version 1 gives a count only. It does not give the position of each gap. + An application that must place a gap still uses the timestamps. +- The inlet queue also drops the oldest sample when the application reads + too slowly. This drop happens after the count, so `seq` does not see it. + `Inlet::samples_dropped()` in `labstream-net` and `Inlet::dropped()` in + `labstream` report it, as before. The total loss is the sum + of the two values. ### `shm`: shared memory on one machine @@ -197,6 +265,4 @@ the stored data, and it does not describe one connection. ## Open questions - The final name of the header. -- The name of the configuration key from rule 7. - The API that reports the features of the current connection. -- The exact bytes of the `seq` framing. This belongs to the `seq` section. From 557aad2fc22b61bf860c50eee2919f4f1f2b8492 Mon Sep 17 00:00:00 2001 From: rednayan Date: Mon, 16 Nov 2026 15:25:53 +0100 Subject: [PATCH 4/7] Number the samples between two labstream peers (seq/1) An inlet offers seq/1 in the Labstream-Features header. A labstream outlet answers it, and liblsl ignores it on both sides. The outlet numbers each pushed sample with one counter. Each consumer queue keeps the number after its last sample, so the samples in it always have consecutive numbers. The writer puts a numbering record before the first sample of a connection, and before each sample after a gap. A blocking outlet does the same. The inlet reads the records and counts the numbers that never arrived. The count stays across a reconnect to the same outlet, and starts again for a new outlet. Inlet::samples_lost in labstream-net and Inlet::lost in labstream give it. The value is None when the connection does not use seq/1. The configuration key labstream.Features turns the feature off. Tests: a model test of the queue numbers, records cut at each byte in both byte orders, the records of the writer, and three live tests. In the live tests, the received, lost and dropped samples add up to the pushed samples, for a full outlet queue and for a lost connection. --- crates/labstream-net/src/config.rs | 5 + crates/labstream-net/src/inlet.rs | 148 ++++++++++++- crates/labstream-net/src/outlet.rs | 147 ++++++++++--- crates/labstream-net/src/queue.rs | 208 +++++++++++++++++- crates/labstream-net/tests/seq_live.rs | 282 +++++++++++++++++++++++++ crates/labstream-proto/src/features.rs | 13 +- crates/labstream-proto/src/feed.rs | 4 +- crates/labstream/src/inlet.rs | 14 ++ 8 files changed, 777 insertions(+), 44 deletions(-) create mode 100644 crates/labstream-net/tests/seq_live.rs diff --git a/crates/labstream-net/src/config.rs b/crates/labstream-net/src/config.rs index 8f33b91..4371694 100644 --- a/crates/labstream-net/src/config.rs +++ b/crates/labstream-net/src/config.rs @@ -188,6 +188,10 @@ pub struct Config { pub allow_ipv4: bool, /// True when an outlet binds an IPv6 stack. `ports.IPv6`. pub allow_ipv6: bool, + /// The labstream features that this process can use, for example `seq`. + /// `labstream.Features`. liblsl does not read this key. The value `{}` + /// turns every feature off. `docs/extensions.md`. + pub features: Vec, /// Where the values came from, for a report. pub source: String, } @@ -273,6 +277,7 @@ impl Config { port_range: ini.int("ports.PortRange", 32) as u16, session_id: ini.text("lab.SessionID", "default"), known_peers: ini.set("lab.KnownPeers", "{}"), + features: ini.set("labstream.Features", "{seq}"), multicast_addresses: addresses, multicast_ttl: ttl, watchdog_time_threshold: ini.float("tuning.WatchdogTimeThreshold", 15.0), diff --git a/crates/labstream-net/src/inlet.rs b/crates/labstream-net/src/inlet.rs index 0e65ceb..998998a 100644 --- a/crates/labstream-net/src/inlet.rs +++ b/crates/labstream-net/src/inlet.rs @@ -5,6 +5,7 @@ use crate::config; use crate::info::StreamInfo; use crate::queue::SampleQueue; use labstream_proto::discovery; +use labstream_proto::features; use labstream_proto::feed::{FeedClient, FeedParams, State, TestPatternError}; use labstream_time::{flags, PostProcessor}; use labstream_wire::Sample; @@ -365,6 +366,10 @@ struct ConnState { /// The description of the stream that the reader is attached to. A /// recovery replaces it, and the instance identifier changes. info: Mutex, + /// True when the current connection uses `seq/1`. `docs/extensions.md`. + numbered: AtomicBool, + /// The samples that never arrived, from the `seq/1` numbers. + lost: AtomicU64, } /// How long without data before the reader treats the source as gone. @@ -416,6 +421,7 @@ impl Inlet { ) -> std::io::Result { let (stream, client, buf) = handshake(info, timeout, buffer_samples)?; debug_assert_eq!(client.state(), State::Streaming); + let numbered = uses_seq(&client); // The queue holds what the application has not pulled yet. Its size is // the value that went out in `Max-Buffer-Length`, which is a count of @@ -438,6 +444,8 @@ impl Inlet { recoveries: AtomicU64::new(0), connected: AtomicBool::new(true), info: Mutex::new(info.clone()), + numbered: AtomicBool::new(numbered), + lost: AtomicU64::new(0), }); // One thread owns the connection for its whole life: it reads, and when @@ -586,6 +594,24 @@ impl Inlet { self.queue.dropped() } + /// How many samples never reached this inlet, since it opened. + /// + /// The count comes from the `seq/1` numbers (`docs/extensions.md`). It + /// shows a sample that the outlet dropped from a full queue, and the + /// samples between a lost connection and the next one. The value is + /// `None` when the current connection does not use `seq/1`, for example + /// with a liblsl outlet. `None` means unknown, not zero. + /// + /// The inlet queue drops after this count, so a full total adds + /// [`Inlet::samples_dropped`]. + pub fn samples_lost(&self) -> Option { + if self.conn.numbered.load(Ordering::SeqCst) { + Some(self.conn.lost.load(Ordering::SeqCst)) + } else { + None + } + } + /// The size of the queue, in samples. pub fn buffer_capacity(&self) -> usize { self.queue.capacity() @@ -632,6 +658,10 @@ fn read_and_recover( let srate = conn.info.lock().unwrap().nominal_srate; let mut deducer = labstream_wire::TimestampDeducer::new(srate); + // The `seq/1` count. It lives as long as the inlet, so the first record + // after a reconnect to the same outlet shows the gap. + let mut counter = labstream_proto::seq::Counter::new(); + loop { if queue.is_closed() { break; @@ -639,7 +669,15 @@ fn read_and_recover( // Decode everything already buffered before reading more. The whole // read goes into the queue with one lock. - let decoded = decode_buffered(&codec, &mut carry, &mut deducer, &mut batch); + let numbered = conn.numbered.load(Ordering::SeqCst); + let decoded = decode_buffered( + &codec, + &mut carry, + &mut deducer, + numbered.then_some(&mut counter), + &mut batch, + ); + conn.lost.store(counter.lost(), Ordering::SeqCst); if !batch.is_empty() { last_data = Instant::now(); queue.push_owned(&mut batch); @@ -725,6 +763,7 @@ fn read_and_recover( sock = s; carry = rest; codec = *c.codec().expect("the handshake set the codec"); + conn.numbered.store(uses_seq(&c), Ordering::SeqCst); conn.recoveries.fetch_add(1, Ordering::SeqCst); conn.connected.store(true, Ordering::SeqCst); last_data = Instant::now(); @@ -755,6 +794,9 @@ fn read_and_recover( } clock.reset_offset(); deducer = labstream_wire::TimestampDeducer::new(fresh.nominal_srate); + // A new outlet has a new counter. + counter.restart(); + conn.numbered.store(uses_seq(&c), Ordering::SeqCst); *conn.info.lock().unwrap() = *fresh; conn.recoveries.fetch_add(1, Ordering::SeqCst); conn.connected.store(true, Ordering::SeqCst); @@ -799,15 +841,33 @@ fn decode_buffered( codec: &labstream_wire::Codec, carry: &mut Vec, deducer: &mut labstream_wire::TimestampDeducer, + mut counter: Option<&mut labstream_proto::seq::Counter>, out: &mut Vec, ) -> Result<(), labstream_wire::Error> { + use labstream_proto::seq::{peek, Next, RECORD_LEN}; let mut pos = 0; let result = loop { + // With `seq/1`, a numbering record can stand before a sample. A cut + // record waits for more bytes, as a cut sample does. + if let Some(c) = counter.as_deref_mut() { + match peek(&carry[pos..]) { + Next::Record(n) => { + c.record(n); + pos += RECORD_LEN; + continue; + } + Next::Incomplete => break Ok(()), + Next::Sample => {} + } + } match codec.decode(&carry[pos..]) { Ok((mut sample, used)) => { pos += used; sample.timestamp = deducer.apply(sample.timestamp); out.push(sample); + if let Some(c) = counter.as_deref_mut() { + c.sample(); + } } Err(labstream_wire::Error::Truncated) => break Ok(()), Err(e) => break Err(e), @@ -895,6 +955,11 @@ pub fn read_fullinfo(known: &StreamInfo, timeout: Duration) -> std::io::Result bool { + features::has(client.features(), features::SEQ, 1) +} + /// Connect and run the whole handshake. SPEC.md 4 and 6. /// /// The reader thread calls this again after a recovery, so it lives outside @@ -916,7 +981,11 @@ fn handshake( params.session_id = info.session_id.clone(); params.max_buffer_length = buffer_samples.max(1); - let (mut client, request) = FeedClient::start(params, info.format, info.channel_count as usize); + // A labstream outlet answers the offer, and any other outlet ignores it. + // `docs/extensions.md`. + let offer = features::offer(&crate::config::get().features); + let (mut client, request) = + FeedClient::start_with(params, info.format, info.channel_count as usize, offer); stream.write_all(&request)?; stream.flush()?; @@ -1022,13 +1091,84 @@ mod tests { let mut from = 0; for &to in cuts.iter().chain(std::iter::once(&bytes.len())) { carry.extend_from_slice(&bytes[from..to]); - decode_buffered(codec, &mut carry, &mut deducer, &mut out).unwrap(); + decode_buffered(codec, &mut carry, &mut deducer, None, &mut out).unwrap(); from = to; } assert!(carry.is_empty(), "no byte stays behind at the end"); out } + /// Decode in reads, with a `seq/1` counter. Give the samples and the + /// counter at the end. + fn decode_numbered( + codec: &Codec, + bytes: &[u8], + cuts: &[usize], + ) -> (Vec, labstream_proto::seq::Counter) { + let mut carry = Vec::new(); + let mut deducer = TimestampDeducer::new(100.0); + let mut counter = labstream_proto::seq::Counter::new(); + let mut out = Vec::new(); + let mut from = 0; + for &to in cuts.iter().chain(std::iter::once(&bytes.len())) { + carry.extend_from_slice(&bytes[from..to]); + decode_buffered( + codec, + &mut carry, + &mut deducer, + Some(&mut counter), + &mut out, + ) + .unwrap(); + from = to; + } + assert!(carry.is_empty(), "no byte stays behind at the end"); + (out, counter) + } + + #[test] + fn a_numbered_stream_cut_at_any_byte_gives_the_same_samples_and_count() { + use labstream_proto::seq::write_record; + for order in [ByteOrder::Little, ByteOrder::Big] { + let codec = Codec::new(Format::Float32, 3, order); + let (samples, _) = stream(&codec, 12); + // Numbers 40..45, then a gap of 3, then 48..54. + let mut bytes = Vec::new(); + write_record(&mut bytes, 40); + for s in &samples[..5] { + codec.encode(s, &mut bytes).unwrap(); + } + write_record(&mut bytes, 48); + for s in &samples[5..] { + codec.encode(s, &mut bytes).unwrap(); + } + let (whole, c) = decode_numbered(&codec, &bytes, &[]); + assert_eq!(whole.len(), 12); + assert_eq!(c.lost(), 3); + assert_eq!(c.expected(), Some(55)); + for cut in 1..bytes.len() { + let (got, c) = decode_numbered(&codec, &bytes, &[cut]); + assert_eq!(got, whole, "a cut at {cut}"); + assert_eq!(c.lost(), 3, "a cut at {cut}"); + } + } + } + + #[test] + fn a_plain_stream_never_reads_a_record() { + // Without `seq/1`, the byte 0xFE is not a record. Here it can only + // stand inside a value, and the codec reads it as data. + let codec = Codec::new(Format::Int8, 1, ByteOrder::native()); + let s = Sample { + timestamp: 1.0, + values: vec![Value::I8(-2)], + }; + let mut bytes = Vec::new(); + codec.encode(&s, &mut bytes).unwrap(); + assert!(bytes.contains(&0xFE)); + assert_eq!(decode_in_reads(&codec, &bytes, &[]), vec![s]); + } + #[test] fn a_cut_at_any_byte_gives_the_same_samples() { let codec = Codec::new(Format::Float32, 3, ByteOrder::native()); @@ -1065,7 +1205,7 @@ mod tests { let mut carry = bytes; let mut deducer = TimestampDeducer::new(100.0); let mut out = Vec::new(); - assert!(decode_buffered(&codec, &mut carry, &mut deducer, &mut out).is_err()); + assert!(decode_buffered(&codec, &mut carry, &mut deducer, None, &mut out).is_err()); assert_eq!(out.len(), 2); } diff --git a/crates/labstream-net/src/outlet.rs b/crates/labstream-net/src/outlet.rs index 9f5ac98..32248ce 100644 --- a/crates/labstream-net/src/outlet.rs +++ b/crates/labstream-net/src/outlet.rs @@ -4,8 +4,10 @@ use crate::info::StreamInfo; use crate::queue::{Fanout, SampleQueue}; use crate::{clock, config}; use labstream_proto::discovery; +use labstream_proto::features; use labstream_proto::header::parse_block; use labstream_proto::negotiate::{negotiate, FeedRequest, Outcome, ServerStream, LITTLE}; +use labstream_proto::seq::Numbering; use labstream_proto::timesync; use labstream_wire::{Codec, Sample, TEST_PATTERN_OFFSETS}; use std::io::{Read, Write}; @@ -25,7 +27,7 @@ struct Shared { /// One queue per connected consumer. SPEC.md 8.8. fanout: Fanout, /// The consumers of a blocking outlet, which holds no queue at all. - sync: Mutex>, + sync: Mutex, /// True when this outlet writes each sample before the push returns. sync_mode: bool, stop: Mutex, @@ -38,6 +40,17 @@ struct Shared { struct SyncConsumer { stream: TcpStream, codec: Codec, + /// The record state, when the consumer agreed to `seq/1`. + numbering: Option, +} + +/// The consumers of a blocking outlet, and the number of the next sample. +/// +/// One lock holds both, so the numbers follow the order of the writes. +#[derive(Default)] +struct SyncSet { + list: Vec, + next: u64, } /// A stream outlet. @@ -139,7 +152,7 @@ impl Outlet { let shared = Arc::new(Shared { info: Mutex::new(info.clone()), fanout: Fanout::new(), - sync: Mutex::new(Vec::new()), + sync: Mutex::new(SyncSet::default()), sync_mode, stop: Mutex::new(false), }); @@ -193,7 +206,7 @@ impl Outlet { if self.shared.sync_mode { // A blocking outlet registers its consumers with the writer rather // than with the queue. `src/stream_outlet_impl.cpp:176-179`. - return self.shared.sync.lock().unwrap().len(); + return self.shared.sync.lock().unwrap().list.len(); } self.shared.fanout.count() } @@ -255,24 +268,21 @@ impl Outlet { /// A consumer whose socket fails is dropped. Nothing is queued, so a consumer /// that reads slowly holds up the caller. That is the whole point of the mode. fn write_blocking(shared: &Shared, sample: &Sample) { - let mut out = Vec::new(); - let mut consumers = shared.sync.lock().unwrap(); - consumers.retain_mut(|c| { - out.clear(); - if c.codec.encode(sample, &mut out).is_err() { - return false; - } - c.stream.write_all(&out).is_ok() && c.stream.flush().is_ok() - }); + write_blocking_many(shared, std::slice::from_ref(sample)); } /// Write several samples to every consumer, with one write for each consumer. fn write_blocking_many(shared: &Shared, samples: &[Sample]) { let mut out = Vec::new(); - let mut consumers = shared.sync.lock().unwrap(); - consumers.retain_mut(|c| { + let mut set = shared.sync.lock().unwrap(); + let first = set.next; + set.next = first.wrapping_add(samples.len() as u64); + set.list.retain_mut(|c| { out.clear(); - for s in samples { + for (i, s) in samples.iter().enumerate() { + if let Some(n) = c.numbering.as_mut() { + n.before(first.wrapping_add(i as u64), &mut out); + } if c.codec.encode(s, &mut out).is_err() { return false; } @@ -611,9 +621,17 @@ fn serve_client(mut stream: TcpStream, shared: Arc) -> std::io::Result<( return Ok(()); } + // A labstream inlet offers its features in one more header. An inlet + // that sent none gets the plain answer. `docs/extensions.md`. + let answered = headers + .get(features::HEADER_KEY) + .map(|v| features::answer(&features::parse(v), &config::get().features)); + let numbered = answered + .as_deref() + .is_some_and(|a| features::has(a, features::SEQ, 1)); stream.write_all( accepted - .to_wire(srv.configured_version, &info.uid) + .to_wire_with(srv.configured_version, &info.uid, answered.as_deref()) .as_bytes(), )?; @@ -657,11 +675,11 @@ fn serve_client(mut stream: TcpStream, shared: Arc) -> std::io::Result<( if trace { eprintln!("TRACE handing the socket to the blocking writer"); } - shared - .sync - .lock() - .unwrap() - .push(SyncConsumer { stream, codec }); + shared.sync.lock().unwrap().list.push(SyncConsumer { + stream, + codec, + numbering: numbered.then(Numbering::new), + }); return Ok(()); } @@ -681,7 +699,15 @@ fn serve_client(mut stream: TcpStream, shared: Arc) -> std::io::Result<( }; let stop = || *shared.stop.lock().unwrap(); - let sent = feed(&queue, &codec, &mut stream, max_chunk, &stop, trace); + let sent = feed( + &queue, + &codec, + &mut stream, + max_chunk, + numbered, + &stop, + trace, + ); if trace { eprintln!("TRACE session over after {sent} samples"); } @@ -702,15 +728,22 @@ fn serve_client(mut stream: TcpStream, shared: Arc) -> std::io::Result<( /// as the push-through flag. When the writer is slower than the source, one /// write can carry more than one push call. No sample waits for a later push. /// +/// When `numbered` is true, the consumer agreed to `seq/1`. A numbering +/// record then goes before the first sample, and before each sample whose +/// number does not follow the one before it. A record does not count toward +/// the chunk size. `docs/extensions.md`. +/// /// The result is the number of samples written. fn feed( queue: &SampleQueue, codec: &Codec, out: &mut W, max_chunk: usize, + numbered: bool, stop: &dyn Fn() -> bool, trace: bool, ) -> usize { + let mut numbering = numbered.then(Numbering::new); let mut batch: Vec = Vec::new(); let mut pending: Vec = Vec::new(); let mut count = 0usize; @@ -736,10 +769,15 @@ fn feed( break; } batch.clear(); - if queue.pop_many(&mut batch, usize::MAX, Duration::from_millis(200)) == 0 { + let (taken, first) = + queue.pop_many_numbered(&mut batch, usize::MAX, Duration::from_millis(200)); + if taken == 0 { continue; } - for s in &batch { + for (i, s) in batch.iter().enumerate() { + if let Some(n) = numbering.as_mut() { + n.before(first.wrapping_add(i as u64), &mut pending); + } if codec.encode(s, &mut pending).is_err() { return sent; } @@ -892,7 +930,7 @@ mod tests { let once = AtomicBool::new(false); // The first call to `stop` lets one round run. The second one stops. let stop = || once.swap(true, Ordering::SeqCst); - let sent = feed(queue, &codec, &mut out, max_chunk, &stop, false); + let sent = feed(queue, &codec, &mut out, max_chunk, false, &stop, false); (sent, out) } @@ -926,6 +964,65 @@ mod tests { assert_eq!(out.writes, 1); } + /// Feed a numbered queue once, and read the records and samples back. + fn run_numbered( + queue: &SampleQueue, + max_chunk: usize, + order: ByteOrder, + ) -> (Vec, Vec, Counting) { + use labstream_proto::seq::{peek, Next, RECORD_LEN}; + let codec = Codec::new(Format::Float32, 2, order); + let mut out = Counting::default(); + let once = AtomicBool::new(false); + let stop = || once.swap(true, Ordering::SeqCst); + feed(queue, &codec, &mut out, max_chunk, true, &stop, false); + let mut pos = 0; + let (mut got, mut records) = (Vec::new(), Vec::new()); + while pos < out.bytes.len() { + if let Next::Record(n) = peek(&out.bytes[pos..]) { + records.push(n); + pos += RECORD_LEN; + continue; + } + let (s, used) = codec.decode(&out.bytes[pos..]).unwrap(); + got.push(s); + pos += used; + } + (got, records, out) + } + + #[test] + fn a_numbered_feed_starts_with_a_record() { + for order in [ByteOrder::Little, ByteOrder::Big] { + let q = SampleQueue::with_first_number(1000, 500); + let sent = samples(20); + q.push_many(&sent); + let (got, records, _) = run_numbered(&q, usize::MAX, order); + assert_eq!(got, sent); + assert_eq!(records, vec![500], "one record, at the start"); + } + } + + #[test] + fn a_full_queue_shows_as_a_gap_in_the_numbers() { + // A queue of 4 that gets 10 samples keeps numbers 106 to 109. + let q = SampleQueue::with_first_number(4, 100); + q.push_many(&samples(10)); + let (got, records, _) = run_numbered(&q, usize::MAX, ByteOrder::native()); + assert_eq!(got.len(), 4); + assert_eq!(records, vec![106]); + } + + #[test] + fn a_record_does_not_count_toward_the_chunk_size() { + let q = SampleQueue::with_first_number(1000, 7); + q.push_many(&samples(32)); + let (got, records, out) = run_numbered(&q, 16, ByteOrder::native()); + assert_eq!(got.len(), 32); + assert_eq!(records, vec![7]); + assert_eq!(out.writes, 2, "16 samples in each write, the record extra"); + } + #[test] fn the_bytes_decode_to_the_same_samples() { let q = SampleQueue::new(1000); diff --git a/crates/labstream-net/src/queue.rs b/crates/labstream-net/src/queue.rs index e2bb66a..d0d93ee 100644 --- a/crates/labstream-net/src/queue.rs +++ b/crates/labstream-net/src/queue.rs @@ -16,10 +16,42 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Condvar, Mutex}; use std::time::{Duration, Instant}; +/// The samples of a queue, and the number after the last one. +/// +/// Each push adds one to `back`, also for a sample that the queue drops at +/// once. A removal does not change it. The samples in the queue thus always +/// have the numbers `back - len .. back`, with no gap: the queue drops only +/// from the front, and a push only adds at the back. `docs/extensions.md`. +#[derive(Debug)] +struct Ring { + q: VecDeque, + back: u64, +} + +impl std::ops::Deref for Ring { + type Target = VecDeque; + fn deref(&self) -> &VecDeque { + &self.q + } +} + +impl std::ops::DerefMut for Ring { + fn deref_mut(&mut self) -> &mut VecDeque { + &mut self.q + } +} + +impl Ring { + /// The number of the sample at the front. + fn front_number(&self) -> u64 { + self.back.wrapping_sub(self.q.len() as u64) + } +} + /// A queue that never blocks its writer. #[derive(Debug)] pub struct SampleQueue { - inner: Mutex>, + inner: Mutex, room: Condvar, capacity: usize, dropped: AtomicU64, @@ -32,9 +64,20 @@ impl SampleQueue { /// A capacity of zero would drop every sample, so the value floors at 1. /// liblsl floors the same way (`src/stream_info_impl.cpp:268`). pub fn new(capacity: usize) -> Self { + Self::with_first_number(capacity, 0) + } + + /// Build a queue whose first pushed sample gets the number `first`. + /// + /// The `seq` feature numbers the samples of an outlet with one counter. + /// A consumer that connects later starts at the current count. + pub fn with_first_number(capacity: usize, first: u64) -> Self { let capacity = capacity.max(1); SampleQueue { - inner: Mutex::new(VecDeque::with_capacity(capacity.min(4096))), + inner: Mutex::new(Ring { + q: VecDeque::with_capacity(capacity.min(4096)), + back: first, + }), room: Condvar::new(), capacity, dropped: AtomicU64::new(0), @@ -76,6 +119,7 @@ impl SampleQueue { self.dropped.fetch_add(1, Ordering::SeqCst); } q.push_back(s); + q.back = q.back.wrapping_add(1); drop(q); self.room.notify_one(); } @@ -102,6 +146,7 @@ impl SampleQueue { } q.push_back(s.clone()); } + q.back = q.back.wrapping_add(samples.len() as u64); drop(q); if lost > 0 { self.dropped.fetch_add(lost, Ordering::SeqCst); @@ -123,6 +168,8 @@ impl SampleQueue { let mut lost = 0u64; // Samples that the queue drops at once are never moved in. let skip = samples.len().saturating_sub(self.capacity); + // Each sample takes a number, also the ones that are never moved in. + q.back = q.back.wrapping_add(samples.len() as u64); for s in samples.drain(..).skip(skip) { if q.len() >= self.capacity { q.pop_front(); @@ -208,6 +255,38 @@ impl SampleQueue { } } + /// Like [`SampleQueue::pop_many`], and also give the number of the first + /// sample that it took. + /// + /// The samples that it took have consecutive numbers. With no sample, the + /// number is the one that the next sample gets. + pub fn pop_many_numbered( + &self, + out: &mut Vec, + max: usize, + timeout: Duration, + ) -> (usize, u64) { + if max == 0 { + return (0, self.inner.lock().unwrap().front_number()); + } + let end = Instant::now() + timeout; + let mut q = self.inner.lock().unwrap(); + loop { + if !q.is_empty() || Instant::now() >= end || *self.closed.lock().unwrap() { + let first = q.front_number(); + let take = q.len().min(max); + out.extend(q.drain(..take)); + return (take, first); + } + let left = end.saturating_duration_since(Instant::now()); + let (guard, _) = self + .room + .wait_timeout(q, left.min(Duration::from_millis(100))) + .unwrap(); + q = guard; + } + } + /// Wake every reader and stop the queue. pub fn close(&self) { *self.closed.lock().unwrap() = true; @@ -226,7 +305,17 @@ impl SampleQueue { /// its own queue, so a slow consumer loses only its own samples. SPEC.md 8.8. #[derive(Debug, Default)] pub struct Fanout { - consumers: Mutex>>, + consumers: Mutex, +} + +/// The queues of a fanout, and the number of the next pushed sample. +/// +/// One lock holds both, so a new queue starts at the number that the next +/// push gives, and every queue sees the same numbers. +#[derive(Debug, Default)] +struct Consumers { + list: Vec>, + next: u64, } impl Fanout { @@ -237,22 +326,29 @@ impl Fanout { /// Add a consumer with its own queue. pub fn add(&self, capacity: usize) -> Arc { - let q = Arc::new(SampleQueue::new(capacity)); - self.consumers.lock().unwrap().push(Arc::clone(&q)); + let mut set = self.consumers.lock().unwrap(); + let q = Arc::new(SampleQueue::with_first_number(capacity, set.next)); + set.list.push(Arc::clone(&q)); q } + /// The number that the next pushed sample gets. + pub fn next_number(&self) -> u64 { + self.consumers.lock().unwrap().next + } + /// Remove a consumer. pub fn remove(&self, q: &Arc) { self.consumers .lock() .unwrap() + .list .retain(|c| !Arc::ptr_eq(c, q)); } /// How many consumers are connected. pub fn count(&self) -> usize { - self.consumers.lock().unwrap().len() + self.consumers.lock().unwrap().list.len() } /// Give one sample to every consumer. @@ -261,8 +357,9 @@ impl Fanout { /// waits, whatever any consumer is doing. pub fn push(&self, s: &Sample) { let mut set = self.consumers.lock().unwrap(); - set.retain(|q| !q.is_closed()); - for q in set.iter() { + set.next = set.next.wrapping_add(1); + set.list.retain(|q| !q.is_closed()); + for q in set.list.iter() { q.push(s.clone()); } } @@ -276,15 +373,16 @@ impl Fanout { return; } let mut set = self.consumers.lock().unwrap(); - set.retain(|q| !q.is_closed()); - for q in set.iter() { + set.next = set.next.wrapping_add(samples.len() as u64); + set.list.retain(|q| !q.is_closed()); + for q in set.list.iter() { q.push_many(samples); } } /// Close every consumer queue. pub fn close(&self) { - for q in self.consumers.lock().unwrap().iter() { + for q in self.consumers.lock().unwrap().list.iter() { q.close(); } } @@ -538,4 +636,92 @@ mod tests { f.push(&sample(2)); assert_eq!(a.len(), 1, "a removed consumer must receive nothing more"); } + + /// A small generator, so the test runs the same steps on each run. + struct Steps(u64); + + impl Steps { + fn next(&mut self, below: u64) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 % below + } + } + + #[test] + fn the_numbers_follow_a_model_with_one_number_for_each_sample() { + // Each sample carries its own number as its value. The model keeps + // the numbers. Every way to push, drop and pop must agree with it. + for seed in 1..40u64 { + let mut steps = Steps(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15)); + let capacity = 1 + steps.next(12) as usize; + let first = steps.next(1000); + let q = SampleQueue::with_first_number(capacity, first); + let mut model: VecDeque = VecDeque::new(); + let mut next = first; + for _ in 0..400 { + match steps.next(5) { + 0 => { + q.push(sample(next as i32)); + model.push_back(next); + next += 1; + } + 1 | 2 => { + let k = steps.next(30) as usize; + let block: Vec = + (0..k).map(|i| sample((next + i as u64) as i32)).collect(); + if steps.next(2) == 0 { + q.push_many(&block); + } else { + let mut owned = block; + q.push_owned(&mut owned); + } + for i in 0..k as u64 { + model.push_back(next + i); + } + next += k as u64; + } + 3 => { + let got = q.pop(Duration::ZERO); + let want = model.pop_front(); + assert_eq!(got.map(|s| index(&s) as u64), want); + } + _ => { + let max = 1 + steps.next(20) as usize; + let mut out = Vec::new(); + let (n, number) = q.pop_many_numbered(&mut out, max, Duration::ZERO); + assert_eq!(n, out.len()); + let want_first = model.front().copied().unwrap_or(next); + assert_eq!(number, want_first, "seed {seed}"); + for (i, s) in out.iter().enumerate() { + assert_eq!(index(s) as u64, number + i as u64, "seed {seed}"); + assert_eq!(model.pop_front(), Some(number + i as u64)); + } + } + } + while model.len() > capacity { + model.pop_front(); + } + assert_eq!(q.len(), model.len(), "seed {seed}"); + } + } + } + + #[test] + fn a_late_consumer_starts_at_the_current_number() { + let f = Fanout::new(); + let a = f.add(100); + for i in 0..5 { + f.push(&sample(i)); + } + let b = f.add(100); + assert_eq!(f.next_number(), 5); + f.push_many(&[sample(5), sample(6)]); + let mut out = Vec::new(); + assert_eq!(a.pop_many_numbered(&mut out, 100, Duration::ZERO), (7, 0)); + out.clear(); + assert_eq!(b.pop_many_numbered(&mut out, 100, Duration::ZERO), (2, 5)); + assert_eq!(index(&out[0]), 5); + } } diff --git a/crates/labstream-net/tests/seq_live.rs b/crates/labstream-net/tests/seq_live.rs new file mode 100644 index 0000000..6b1225e --- /dev/null +++ b/crates/labstream-net/tests/seq_live.rs @@ -0,0 +1,282 @@ +//! The `seq/1` feature, measured against a live outlet. `docs/extensions.md`. +//! +//! These tests open real sockets on the loopback interface, so they belong to +//! the reporting tier and not to the merge gate. A failure here can come from +//! the machine. +//! +//! Each test makes sure that the counts add up: every pushed sample is +//! received, counted as lost, or counted as dropped by the inlet queue. + +use labstream_net::{Inlet, Outlet, StreamInfo}; +use labstream_proto::features::{self, Token}; +use labstream_proto::feed::{FeedClient, FeedParams, TestPatternError}; +use labstream_proto::seq; +use labstream_wire::{Format, Sample, Value}; +use std::io::{Read, Write}; +use std::net::{Shutdown, TcpListener, TcpStream}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +fn sample(k: usize) -> Sample { + Sample { + timestamp: 0.0, + values: vec![Value::F32(k as f32)], + } +} + +fn value(s: &Sample) -> usize { + match s.values[0] { + Value::F32(v) => v as usize, + _ => panic!("wrong format"), + } +} + +fn outlet(name: &str) -> Option { + let mut info = StreamInfo::new(name, "SeqTest", 1, Format::Float32, 1000.0); + info.source_id = format!("{name}_src"); + Outlet::new(info).ok() +} + +/// Pull until a sample with the value `last` arrives, or the time runs out. +fn pull_until(inlet: &mut Inlet, last: usize, within: Duration) -> Vec { + let mut got = Vec::new(); + let end = Instant::now() + within; + while Instant::now() < end { + let mut block = Vec::new(); + let _ = inlet.pull_chunk(&mut block, 100_000, Duration::from_millis(20)); + let done = block.iter().any(|s| value(s) == last); + got.extend(block); + if done { + break; + } + } + got +} + +#[test] +fn the_count_adds_up_when_the_outlet_queue_drops() { + let out = match outlet("SeqLive1") { + Some(o) => o, + None => { + eprintln!("no local network; skipping"); + return; + } + }; + // The inlet asks for 32 samples of buffer, so the outlet keeps a queue + // of 32 for it. + let mut inlet = + Inlet::open_recovering(out.info(), Duration::from_secs(5), 32).expect("the inlet opens"); + assert!(out.wait_for_consumers(Duration::from_secs(5))); + + // One sample sets the baseline of the count. + out.push(&sample(0)); + let first = pull_until(&mut inlet, 0, Duration::from_secs(5)); + assert_eq!(first.len(), 1, "the first sample arrives"); + assert_eq!(inlet.samples_lost(), Some(0), "two labstream peers use seq"); + + // One push of a large block. The queue of 32 keeps the last 32 samples. + let n = 100_000; + out.push_samples((1..=n).map(sample).collect()); + let got = pull_until(&mut inlet, n, Duration::from_secs(10)); + assert_eq!(got.last().map(value), Some(n), "the last sample arrives"); + + let lost = inlet.samples_lost().expect("a count"); + let dropped = inlet.samples_dropped(); + println!("received {}, lost {lost}, dropped {dropped}", got.len()); + assert_eq!(got.len() as u64 + lost + dropped, n as u64); + assert!( + lost >= (n - 64) as u64, + "the outlet dropped most of the block" + ); +} + +/// A TCP proxy that forwards each connection to `target`, and can cut them. +struct Proxy { + port: u16, + accepted: Arc>>, +} + +impl Proxy { + fn start(target: u16) -> Option { + let listener = TcpListener::bind("127.0.0.1:0").ok()?; + let port = listener.local_addr().ok()?.port(); + let accepted = Arc::new(Mutex::new(Vec::new())); + let list = Arc::clone(&accepted); + std::thread::spawn(move || { + for client in listener.incoming().flatten() { + let server = match TcpStream::connect(("127.0.0.1", target)) { + Ok(s) => s, + Err(_) => continue, + }; + list.lock().unwrap().push(client.try_clone().unwrap()); + pipe(client.try_clone().unwrap(), server.try_clone().unwrap()); + pipe(server, client); + } + }); + Some(Proxy { port, accepted }) + } + + fn cut(&self) { + for s in self.accepted.lock().unwrap().drain(..) { + let _ = s.shutdown(Shutdown::Both); + } + } +} + +fn pipe(mut from: TcpStream, mut to: TcpStream) { + std::thread::spawn(move || { + let mut buf = [0u8; 16384]; + while let Ok(n) = from.read(&mut buf) { + if n == 0 || to.write_all(&buf[..n]).is_err() { + break; + } + } + let _ = to.shutdown(Shutdown::Both); + let _ = from.shutdown(Shutdown::Both); + }); +} + +#[test] +fn the_count_shows_the_samples_of_a_lost_connection() { + let out = match outlet("SeqLive2") { + Some(o) => Arc::new(o), + None => { + eprintln!("no local network; skipping"); + return; + } + }; + let real = out.info().clone(); + let proxy = match Proxy::start(real.v4data_port) { + Some(p) => p, + None => { + eprintln!("no local network; skipping"); + return; + } + }; + let mut via_proxy = real.clone(); + via_proxy.v4address = "127.0.0.1".to_string(); + via_proxy.v4data_port = proxy.port; + let mut inlet = Inlet::open_recovering(&via_proxy, Duration::from_secs(5), 100_000) + .expect("the inlet connects through the proxy"); + assert!(out.wait_for_consumers(Duration::from_secs(5))); + + // A source thread pushes one sample each 200 us. + let pushed = Arc::new(AtomicUsize::new(0)); + let stop = Arc::new(AtomicBool::new(false)); + let source = { + let (out, pushed, stop) = (Arc::clone(&out), Arc::clone(&pushed), Arc::clone(&stop)); + std::thread::spawn(move || { + let mut k = 0; + while !stop.load(Ordering::SeqCst) { + out.push(&sample(k)); + k += 1; + pushed.store(k, Ordering::SeqCst); + std::thread::sleep(Duration::from_micros(200)); + } + }) + }; + + let mut got = pull_until(&mut inlet, 50, Duration::from_secs(5)); + assert!(!got.is_empty(), "samples arrive before the cut"); + proxy.cut(); + // Wait for samples from after the recovery. + let cut_at = pushed.load(Ordering::SeqCst); + let end = Instant::now() + Duration::from_secs(10); + while Instant::now() < end && got.last().map(value).unwrap_or(0) < cut_at + 200 { + let mut block = Vec::new(); + let _ = inlet.pull_chunk(&mut block, 100_000, Duration::from_millis(20)); + got.extend(block); + } + stop.store(true, Ordering::SeqCst); + source.join().unwrap(); + let last = pushed.load(Ordering::SeqCst) - 1; + if got.last().map(value) != Some(last) { + got.extend(pull_until(&mut inlet, last, Duration::from_secs(5))); + } + assert_eq!(got.last().map(value), Some(last), "the last sample arrives"); + + assert!(inlet.recoveries() >= 1, "the inlet connected again"); + let first = value(&got[0]); + let lost = inlet.samples_lost().expect("a count after the recovery"); + let dropped = inlet.samples_dropped(); + let expected = (last - first + 1) as u64; + println!( + "pushed {expected} after the first received, received {}, lost {lost}, dropped {dropped}", + got.len() + ); + assert_eq!(got.len() as u64 + lost + dropped, expected); +} + +/// Open a feed by hand, and give the answer features and the first byte after +/// the test pattern. +fn raw_feed(out: &Outlet, offer: Vec) -> (Vec, u8) { + let info = out.info(); + let mut stream = + TcpStream::connect(("127.0.0.1", info.v4data_port)).expect("the data port answers"); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let params = FeedParams::new(&info.uid, info.format); + let (mut client, request) = FeedClient::start_with(params, info.format, 1, offer); + stream.write_all(&request).unwrap(); + + let mut buf = Vec::new(); + let mut tmp = [0u8; 4096]; + loop { + match client.on_headers(&buf) { + Ok(used) => { + buf.drain(..used); + break; + } + Err(labstream_proto::header::Error::Incomplete) => {} + Err(e) => panic!("{e}"), + } + let n = stream.read(&mut tmp).unwrap(); + assert!(n > 0, "the outlet closed in the headers"); + buf.extend_from_slice(&tmp[..n]); + } + loop { + match client.on_test_pattern(&buf) { + Ok(used) => { + buf.drain(..used); + break; + } + Err(TestPatternError::Incomplete) => {} + Err(e) => panic!("{e:?}"), + } + let n = stream.read(&mut tmp).unwrap(); + buf.extend_from_slice(&tmp[..n]); + } + // The consumer exists now. One push gives the first data byte. + let end = Instant::now() + Duration::from_secs(5); + while buf.is_empty() && Instant::now() < end { + out.push(&sample(1)); + let _ = stream.set_read_timeout(Some(Duration::from_millis(50))); + if let Ok(n) = stream.read(&mut tmp) { + buf.extend_from_slice(&tmp[..n]); + } + } + (client.features().to_vec(), buf[0]) +} + +#[test] +fn a_request_without_the_header_gets_plain_samples() { + let out = match outlet("SeqLive3") { + Some(o) => o, + None => { + eprintln!("no local network; skipping"); + return; + } + }; + let (agreed, first) = raw_feed(&out, Vec::new()); + assert!(agreed.is_empty()); + assert!( + first == 1 || first == 2, + "a sample tag, not a record: {first}" + ); + + let (agreed, first) = raw_feed(&out, vec![Token::new(features::SEQ, 1)]); + assert!(features::has(&agreed, features::SEQ, 1)); + assert_eq!(first, seq::TAG, "a numbered feed starts with a record"); +} diff --git a/crates/labstream-proto/src/features.rs b/crates/labstream-proto/src/features.rs index c9ce1f1..096e9ce 100644 --- a/crates/labstream-proto/src/features.rs +++ b/crates/labstream-proto/src/features.rs @@ -60,7 +60,9 @@ impl Token { return None; } let param = match param { - Some(p) if !p.is_empty() && p.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) => { + Some(p) + if !p.is_empty() && p.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) => + { Some(p.to_string()) } Some(_) => return None, @@ -84,7 +86,10 @@ impl Token { /// Read a header value. A token with a bad form is left out. pub fn parse(value: &str) -> Vec { - value.split_ascii_whitespace().filter_map(Token::parse).collect() + value + .split_ascii_whitespace() + .filter_map(Token::parse) + .collect() } /// Write a whole header line, with its line end. @@ -145,7 +150,9 @@ pub fn agreed(offered: &[Token], answered: &[Token]) -> Vec { /// True when the list holds the feature at this version. pub fn has(tokens: &[Token], name: &str, version: u32) -> bool { - tokens.iter().any(|t| t.name == name && t.version == version) + tokens + .iter() + .any(|t| t.name == name && t.version == version) } #[cfg(test)] diff --git a/crates/labstream-proto/src/feed.rs b/crates/labstream-proto/src/feed.rs index 8ac972b..8c31c9e 100644 --- a/crates/labstream-proto/src/feed.rs +++ b/crates/labstream-proto/src/feed.rs @@ -536,7 +536,9 @@ mod tests { max_chunk_length: 0, }; assert_eq!(a.to_wire_with(110, "u", None), a.to_wire(110, "u")); - assert!(a.to_wire_with(110, "u", Some(&[])).contains("Labstream-Features: \r\n")); + assert!(a + .to_wire_with(110, "u", Some(&[])) + .contains("Labstream-Features: \r\n")); } #[test] diff --git a/crates/labstream/src/inlet.rs b/crates/labstream/src/inlet.rs index c9bf730..2a8331a 100644 --- a/crates/labstream/src/inlet.rs +++ b/crates/labstream/src/inlet.rs @@ -229,6 +229,20 @@ impl Inlet { self.inner.samples_dropped() } + /// How many samples never reached this inlet, since it opened. + /// + /// Two labstream peers number their samples (the `seq/1` feature of + /// `docs/extensions.md`). The count shows a sample that the outlet + /// dropped because this inlet read too slowly, and the samples that went + /// out while the connection was lost. [`Inlet::dropped`] counts the + /// samples that this inlet dropped itself. The total loss is the sum. + /// + /// The value is `None` when the source does not number its samples, for + /// example a liblsl outlet. `None` means unknown, not zero. + pub fn lost(&self) -> Option { + self.inner.samples_lost() + } + /// True when the source ended and the program read every sample. /// /// An inlet that recovers reconnects on its own, so this stays false while From 12431fac143411897f88ca5e89c1a04a3377ec16 Mon Sep 17 00:00:00 2001 From: rednayan Date: Mon, 16 Nov 2026 15:35:51 +0100 Subject: [PATCH 5/7] Record the seq conformance results and the changelog docs/extensions.md now gives the status of each item of the conformance plan. Each item passes on this branch: the header probe on liblsl, main and seq outlets, the interop matrix with seq on and off, and the unit and live tests. oracle/headerprobe.py now accepts one Labstream-Features answer line with offered tokens only. Without that line, the answer headers must be the same, and the test pattern must be the same. --- CHANGELOG.md | 14 +++++++++++ conformance/oracle/headerprobe.py | 41 ++++++++++++++++++++----------- docs/extensions.md | 27 ++++++++++++++------ 3 files changed, 60 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6d46c5..99cc2d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ and [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added +- The `seq/1` feature between two labstream peers (`docs/extensions.md`). A + labstream inlet offers it in the request header `Labstream-Features`, and + a labstream outlet answers it. liblsl ignores the header on both sides. + The outlet numbers its samples, and the inlet counts the samples that + never arrived: from a full outlet queue, and from a lost connection. +- `Inlet::lost` in `labstream` and `Inlet::samples_lost` in `labstream-net`. + The value is `None` when the connection does not use `seq/1`. +- The configuration key `labstream.Features`. The value `{}` turns every + labstream feature off. +- `labstream_proto::features` and `labstream_proto::seq`, with + `FeedParams::to_wire_with`, `FeedClient::start_with`, + `FeedClient::features` and `Accepted::to_wire_with`. +- `SampleQueue::with_first_number`, `SampleQueue::pop_many_numbered` and + `Fanout::next_number`. - `docs/latency.md` and `conformance/oracle/latency.py`, with `conformance/oracle/latency.c`. They measure the time from a push to a pull, and the CPU of each side, for liblsl and labstream. diff --git a/conformance/oracle/headerprobe.py b/conformance/oracle/headerprobe.py index 2f2cbcf..2080ef8 100755 --- a/conformance/oracle/headerprobe.py +++ b/conformance/oracle/headerprobe.py @@ -1,14 +1,18 @@ #!/usr/bin/env python3 -"""Make sure that an outlet ignores the `Labstream-Features` request header. +"""Make sure that the `Labstream-Features` request header changes nothing else. -`docs/extensions.md` proposes this header. The proposal depends on one fact: -an outlet that does not know the header gives the same answer with it and -without it. This probe measures that fact on a real outlet. +`docs/extensions.md` defines this header. It depends on two facts: + +- An outlet that does not know the header (liblsl) gives the same answer + with it and without it. +- An outlet that knows it (labstream) adds one answer line, with only + offered tokens, and changes nothing else in the answer or in the test + pattern. The probe starts `oracle/publish.c`, linked to one library. It opens two feed connections by hand. One request has the extra header, and one does not. -The answer headers and the two test pattern samples must be the same, byte -for byte. +Without the `Labstream-Features` line, the answer headers must be the same, +byte for byte. The two test pattern samples must be the same, byte for byte. Usage: oracle/headerprobe.py # liblsl and the Rust library @@ -115,13 +119,20 @@ def probe(name: str, lib_dir: Path, work: Path) -> bool: finally: pub.kill() pub.wait() - same = plain_head == with_head and plain_pattern == with_pattern - leaked = "labstream-features" in with_head.lower() - ok = same and not leaked - print(f" {name:<8} {'pass' if ok else 'FAIL'} answer headers " - f"{'same' if plain_head == with_head else 'DIFFERENT'}, test pattern " - f"{'same' if plain_pattern == with_pattern else 'DIFFERENT'}, " - f"{'a Labstream-Features answer' if leaked else 'no Labstream-Features answer'}") + lines = with_head.split("\r\n") + feature_lines = [x for x in lines if x.lower().startswith("labstream-features:")] + rest = "\r\n".join(x for x in lines if x not in feature_lines) + offered = set(EXTRA.split(":", 1)[1].split()) + answered = set(feature_lines[0].split(":", 1)[1].split()) if feature_lines else set() + heads_same = plain_head == rest + pattern_same = plain_pattern == with_pattern + answer_ok = len(feature_lines) <= 1 and answered <= offered + ok = heads_same and pattern_same and answer_ok + answer = (f"answer '{' '.join(sorted(answered))}'" if feature_lines + else "no Labstream-Features answer") + print(f" {name:<8} {'pass' if ok else 'FAIL'} other answer headers " + f"{'same' if heads_same else 'DIFFERENT'}, test pattern " + f"{'same' if pattern_same else 'DIFFERENT'}, {answer}") if not ok: print(" without:", plain_head.replace("\r\n", " | ")) print(" with: ", with_head.replace("\r\n", " | ")) @@ -142,9 +153,9 @@ def main() -> int: results.append(probe(name, Path(path).resolve(), work)) print() if all(results): - print("each outlet ignores the Labstream-Features header") + print("the Labstream-Features header changes nothing else on any outlet") return 0 - print("an outlet reacts to the Labstream-Features header") + print("the Labstream-Features header changed something else on an outlet") return 1 diff --git a/docs/extensions.md b/docs/extensions.md index 3a5de76..45fbac7 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -246,21 +246,34 @@ the stored data, and it does not describe one connection. ## The conformance plan +Each item gives its status on the branch `feat/seq`. + 1. **liblsl outlet, raw request.** Send a feed request with the header to the oracle outlet. Make sure that the answer and the data bytes are the same - as for the request without the header. `oracle/headerprobe.py` does this - now. It passes for liblsl and for labstream `main`. + as for the request without the header. **Done:** `oracle/headerprobe.py` + passes for liblsl and for labstream `main`. 2. **liblsl inlet.** Make sure that a labstream outlet sends no - `Labstream-Features` line when the request has none. + `Labstream-Features` line, and no record, when the request has none. + **Done:** `crates/labstream-net/tests/seq_live.rs`, + `a_request_without_the_header_gets_plain_samples`. The interop matrix + also runs liblsl inlets against the labstream outlet. 3. **Released labstream 0.1.2.** Run it as a peer in both directions. It - knows nothing of the header, so each run must give plain LSL. + knows nothing of the header, so each run must give plain LSL. **Done:** + below `crates/`, `main` differs from the tag `v0.1.2` only in one comment. + `oracle/headerprobe.py` on a `main` outlet shows no answer line. Item 2 + covers a 0.1.2 inlet, which sends no header. 4. **Two labstream peers.** Run each feature on and off, and each mix of - offer and answer. + offer and answer. **Done:** the unit tests in `feed.rs` and + `features.rs` cover each mix. `oracle/interop.py` and its self test pass + with `seq` on, and again with `labstream.Features = {}` in a + configuration file. `oracle/headerprobe.py` with the feature off gives an + empty answer and no other change. 5. **Syntax.** Send tokens with uppercase letters, with `;`, and with a bad form. Make sure that the outlet ignores each bad token, and keeps the - good ones. + good ones. **Done:** the unit tests in `features.rs`. 6. **The full interop matrix.** `oracle/interop.py` must give the same result - as before, with the header on. + as before, with the header on. **Done:** each mode gives the same count + as on `main`. ## Open questions From 68a1cc8f228d9b7193ed3f64eca08de29325601a Mon Sep 17 00:00:00 2001 From: rednayan Date: Mon, 16 Nov 2026 15:42:18 +0100 Subject: [PATCH 6/7] Show the effect of the run order and the seq result in docs/latency.md A run with seq last in each group showed a slower median at 512 Hz. With seq first, seq and quick gave the same numbers. At 4000 Hz x 64 channels, seq/1 does not change the latency or the CPU. --- docs/latency.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/latency.md b/docs/latency.md index 5a5e33e..0f3c576 100644 --- a/docs/latency.md +++ b/docs/latency.md @@ -104,6 +104,24 @@ Libraries: changes the start of a connection and the end of a dead connection. This benchmark measures neither. +## The order of the runs + +The order of the runs can move a result. In one run of `seq` against +`quick`, the `seq` rows ran last in each group. At 512 Hz x 8 channels, +`seq` then showed a median of 138 µs against 97 µs. With `seq` first, both +gave 91 µs, and the p99 values were 188 µs and 190 µs. A difference at a low +rate is thus not a result until it stays after a change of order. + +On the branch `feat/seq`, the `seq/1` feature does not change the latency +or the CPU at 4000 Hz x 64 channels: + +| outlet -> inlet | load | p50 µs | p99 µs | CPU outlet % | CPU inlet % | +|---|---|---|---|---|---| +| quick -> quick | 4000 Hz x 64, chunk 1 | 73 (72-73) | 103 (101-104) | 20.7 | 17.5 | +| seq -> seq | 4000 Hz x 64, chunk 1 | 73 (72-75) | 99 (97-113) | 20.5 | 17.7 | +| quick -> quick | 4000 Hz x 64, chunk 32 | 356 (348-366) | 711 (677-712) | 3.3 | 2.4 | +| seq -> seq | 4000 Hz x 64, chunk 32 | 350 (332-352) | 648 (609-654) | 3.2 | 2.3 | + ## What this benchmark does not measure - **Two machines.** All results come from the loopback interface. From 35011b6383e9b823053247539d11e97729eaf44b Mon Sep 17 00:00:00 2001 From: rednayan Date: Mon, 16 Nov 2026 15:47:45 +0100 Subject: [PATCH 7/7] Write the first seq record when a connection starts The outlet now writes the opening record at once, after the test pattern and before any sample, in both the queued and the blocking mode. A gap from a lost connection thus shows when the inlet connects again, also on a quiet stream such as a marker stream. Before, it showed only with the next sample. New live tests: a_gap_shows_at_once_on_a_quiet_stream and a_new_outlet_starts_a_new_count. Each fails without its fix (lost Some(0) instead of Some(20), and Some(1067) instead of Some(0)). oracle/headerprobe.py now also reads the first data byte. It must be a sample tag, or the record tag 0xFE when the answer holds seq/1. A test of the configuration default of labstream.Features is added. --- conformance/oracle/headerprobe.py | 17 +++-- crates/labstream-net/src/config.rs | 8 +++ crates/labstream-net/src/outlet.rs | 32 ++++++++- crates/labstream-net/tests/seq_live.rs | 99 +++++++++++++++++++++++++- crates/labstream-proto/src/seq.rs | 24 +++++++ docs/extensions.md | 15 ++-- 6 files changed, 182 insertions(+), 13 deletions(-) diff --git a/conformance/oracle/headerprobe.py b/conformance/oracle/headerprobe.py index 2080ef8..089d536 100755 --- a/conformance/oracle/headerprobe.py +++ b/conformance/oracle/headerprobe.py @@ -101,7 +101,9 @@ def feed(port: int, uid: str, extra: bool) -> tuple: # Each test pattern sample: a tag, a timestamp, and the values. # SPEC.md 6.4. pattern = read_exact(s, 2 * (1 + 8 + 4 * CHANNELS)) - return head.decode(errors="replace"), pattern + # The publisher pushes all the time, so the first data byte follows. + first = read_exact(s, 1)[0] + return head.decode(errors="replace"), pattern, first def probe(name: str, lib_dir: Path, work: Path) -> bool: @@ -114,8 +116,8 @@ def probe(name: str, lib_dir: Path, work: Path) -> bool: try: time.sleep(1.0) uid, port = find_stream(lib_dir, stream) - plain_head, plain_pattern = feed(port, uid, extra=False) - with_head, with_pattern = feed(port, uid, extra=True) + plain_head, plain_pattern, plain_first = feed(port, uid, extra=False) + with_head, with_pattern, with_first = feed(port, uid, extra=True) finally: pub.kill() pub.wait() @@ -127,12 +129,17 @@ def probe(name: str, lib_dir: Path, work: Path) -> bool: heads_same = plain_head == rest pattern_same = plain_pattern == with_pattern answer_ok = len(feature_lines) <= 1 and answered <= offered - ok = heads_same and pattern_same and answer_ok + # A sample starts with tag 1 or 2 (SPEC.md 7.1). With seq/1 on, the data + # starts with a numbering record, tag 0xFE. `docs/extensions.md`. + plain_data_ok = plain_first in (1, 2) + with_data_ok = with_first == 0xFE if "seq/1" in answered else with_first in (1, 2) + ok = heads_same and pattern_same and answer_ok and plain_data_ok and with_data_ok answer = (f"answer '{' '.join(sorted(answered))}'" if feature_lines else "no Labstream-Features answer") print(f" {name:<8} {'pass' if ok else 'FAIL'} other answer headers " f"{'same' if heads_same else 'DIFFERENT'}, test pattern " - f"{'same' if pattern_same else 'DIFFERENT'}, {answer}") + f"{'same' if pattern_same else 'DIFFERENT'}, {answer}, first data byte " + f"{plain_first:#04x} plain, {with_first:#04x} with the header") if not ok: print(" without:", plain_head.replace("\r\n", " | ")) print(" with: ", with_head.replace("\r\n", " | ")) diff --git a/crates/labstream-net/src/config.rs b/crates/labstream-net/src/config.rs index 4371694..e44c52f 100644 --- a/crates/labstream-net/src/config.rs +++ b/crates/labstream-net/src/config.rs @@ -441,6 +441,14 @@ mod tests { assert_eq!(c.use_protocol_version, 110); } + #[test] + fn the_labstream_features_are_on_by_default_and_a_file_turns_them_off() { + let c = Config::from_ini(&Ini::default(), "test"); + assert_eq!(c.features, vec!["seq".to_string()]); + let off = Ini::parse("[labstream]\nFeatures = {}\n").expect("a file"); + assert!(Config::from_ini(&off, "test").features.is_empty()); + } + #[test] fn the_scope_decides_the_groups_and_the_hop_count() { let of = |scope: &str| { diff --git a/crates/labstream-net/src/outlet.rs b/crates/labstream-net/src/outlet.rs index 32248ce..a7ea511 100644 --- a/crates/labstream-net/src/outlet.rs +++ b/crates/labstream-net/src/outlet.rs @@ -675,10 +675,23 @@ fn serve_client(mut stream: TcpStream, shared: Arc) -> std::io::Result<( if trace { eprintln!("TRACE handing the socket to the blocking writer"); } - shared.sync.lock().unwrap().list.push(SyncConsumer { + let mut set = shared.sync.lock().unwrap(); + // A numbered consumer gets its first record at once, under the lock + // that numbers the writes. `docs/extensions.md`. + let numbering = if numbered { + let mut n = Numbering::new(); + let mut record = Vec::new(); + n.start(set.next, &mut record); + stream.write_all(&record)?; + stream.flush()?; + Some(n) + } else { + None + }; + set.list.push(SyncConsumer { stream, codec, - numbering: numbered.then(Numbering::new), + numbering, }); return Ok(()); } @@ -744,6 +757,17 @@ fn feed( trace: bool, ) -> usize { let mut numbering = numbered.then(Numbering::new); + // The first record goes out at once, before any sample. A gap before + // this connection is then visible on a quiet stream too. + if let Some(n) = numbering.as_mut() { + let (_, next) = queue.pop_many_numbered(&mut Vec::new(), 0, Duration::ZERO); + let mut record = Vec::new(); + n.start(next, &mut record); + if out.write_all(&record).is_err() { + return 0; + } + let _ = out.flush(); + } let mut batch: Vec = Vec::new(); let mut pending: Vec = Vec::new(); let mut count = 0usize; @@ -1020,7 +1044,9 @@ mod tests { let (got, records, out) = run_numbered(&q, 16, ByteOrder::native()); assert_eq!(got.len(), 32); assert_eq!(records, vec![7]); - assert_eq!(out.writes, 2, "16 samples in each write, the record extra"); + // The opening record is a write of its own, at the start of the + // connection. Then 16 samples go in each write. + assert_eq!(out.writes, 3, "the record, then two chunks of 16"); } #[test] diff --git a/crates/labstream-net/tests/seq_live.rs b/crates/labstream-net/tests/seq_live.rs index 6b1225e..edf6917 100644 --- a/crates/labstream-net/tests/seq_live.rs +++ b/crates/labstream-net/tests/seq_live.rs @@ -95,6 +95,8 @@ fn the_count_adds_up_when_the_outlet_queue_drops() { struct Proxy { port: u16, accepted: Arc>>, + /// While true, the proxy closes each new connection at once. + paused: Arc, } impl Proxy { @@ -103,8 +105,14 @@ impl Proxy { let port = listener.local_addr().ok()?.port(); let accepted = Arc::new(Mutex::new(Vec::new())); let list = Arc::clone(&accepted); + let paused = Arc::new(AtomicBool::new(false)); + let hold = Arc::clone(&paused); std::thread::spawn(move || { for client in listener.incoming().flatten() { + if hold.load(Ordering::SeqCst) { + let _ = client.shutdown(Shutdown::Both); + continue; + } let server = match TcpStream::connect(("127.0.0.1", target)) { Ok(s) => s, Err(_) => continue, @@ -114,7 +122,11 @@ impl Proxy { pipe(server, client); } }); - Some(Proxy { port, accepted }) + Some(Proxy { + port, + accepted, + paused, + }) } fn cut(&self) { @@ -280,3 +292,88 @@ fn a_request_without_the_header_gets_plain_samples() { assert!(features::has(&agreed, features::SEQ, 1)); assert_eq!(first, seq::TAG, "a numbered feed starts with a record"); } + +#[test] +fn a_gap_shows_at_once_on_a_quiet_stream() { + // A marker stream can stay quiet for minutes. The loss of a lost + // connection must show when the inlet connects again, not at the next + // sample. + let out = match outlet("SeqLive4") { + Some(o) => o, + None => { + eprintln!("no local network; skipping"); + return; + } + }; + let real = out.info().clone(); + let proxy = match Proxy::start(real.v4data_port) { + Some(p) => p, + None => { + eprintln!("no local network; skipping"); + return; + } + }; + let mut via_proxy = real.clone(); + via_proxy.v4address = "127.0.0.1".to_string(); + via_proxy.v4data_port = proxy.port; + let mut inlet = Inlet::open_recovering(&via_proxy, Duration::from_secs(5), 1000) + .expect("the inlet connects through the proxy"); + assert!(out.wait_for_consumers(Duration::from_secs(5))); + out.push_samples((0..10).map(sample).collect()); + assert_eq!(pull_until(&mut inlet, 9, Duration::from_secs(5)).len(), 10); + + // Cut the connection, and hold the inlet off while 20 samples go out. + proxy.paused.store(true, Ordering::SeqCst); + proxy.cut(); + std::thread::sleep(Duration::from_millis(100)); + out.push_samples((10..30).map(sample).collect()); + std::thread::sleep(Duration::from_millis(300)); + proxy.paused.store(false, Ordering::SeqCst); + + // No sample goes out after this point. + let end = Instant::now() + Duration::from_secs(10); + while inlet.samples_lost() != Some(20) && Instant::now() < end { + std::thread::sleep(Duration::from_millis(20)); + } + assert!(inlet.recoveries() >= 1, "the inlet connected again"); + assert_eq!( + inlet.samples_lost(), + Some(20), + "the gap shows with no new sample" + ); +} + +#[test] +fn a_new_outlet_starts_a_new_count() { + // The second outlet has its own counter. It pushes 1000 samples before + // the inlet finds it. If the inlet kept the expected number of the first + // outlet, those 1000 would show as lost. + let make = || outlet("SeqLive5"); + let first = match make() { + Some(o) => o, + None => { + eprintln!("no local network; skipping"); + return; + } + }; + let mut inlet = Inlet::open_recovering(first.info(), Duration::from_secs(5), 100_000) + .expect("the inlet connects"); + assert!(first.wait_for_consumers(Duration::from_secs(5))); + first.push_samples((0..5).map(sample).collect()); + assert_eq!(pull_until(&mut inlet, 4, Duration::from_secs(5)).len(), 5); + drop(first); + + let second = make().expect("the second outlet opens"); + second.push_samples((0..1000).map(sample).collect()); + let mut k = 1000; + let mut got = Vec::new(); + let end = Instant::now() + Duration::from_secs(20); + while got.is_empty() && Instant::now() < end { + second.push(&sample(k)); + k += 1; + let _ = inlet.pull_chunk(&mut got, 100_000, Duration::from_millis(20)); + } + assert!(!got.is_empty(), "samples of the second outlet arrive"); + assert_eq!(inlet.current_info().uid, second.info().uid); + assert_eq!(inlet.samples_lost(), Some(0), "the count starts again"); +} diff --git a/crates/labstream-proto/src/seq.rs b/crates/labstream-proto/src/seq.rs index aca7b4d..8957f27 100644 --- a/crates/labstream-proto/src/seq.rs +++ b/crates/labstream-proto/src/seq.rs @@ -125,6 +125,17 @@ impl Numbering { Numbering::default() } + /// Write the record that opens a connection. `number` is the number + /// that the next sample will get. + /// + /// The record goes out at once, before any sample. A gap that happened + /// before this connection is then visible to the inlet at once, also on + /// a stream that sends no more samples for a long time. + pub fn start(&mut self, number: u64, out: &mut Vec) { + write_record(out, number); + self.next = Some(number); + } + /// Write a record into `out` if the sample with `number` needs one, then /// expect the number after it. pub fn before(&mut self, number: u64, out: &mut Vec) { @@ -210,6 +221,19 @@ mod tests { assert_eq!(c.backwards(), 0); } + #[test] + fn a_started_connection_needs_no_second_record_for_the_next_number() { + let mut w = Numbering::new(); + let mut out = Vec::new(); + w.start(30, &mut out); + assert_eq!(peek(&out), Next::Record(30)); + let mut more = Vec::new(); + w.before(30, &mut more); + assert!(more.is_empty(), "the next sample follows the record"); + w.before(35, &mut more); + assert_eq!(peek(&more), Next::Record(35)); + } + #[test] fn the_writer_and_the_counter_agree() { // The numbers that a writer sends, with gaps from dropped samples. diff --git a/docs/extensions.md b/docs/extensions.md index 45fbac7..e6e10e9 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -158,7 +158,10 @@ u64 number little-endian, the number of the next sample 2. The number is always little-endian. The `Byte-Order` of the connection does not apply to it. 3. On each connection, the first record comes directly after the second test - pattern sample. Each connection thus starts with an absolute number. + pattern sample. The outlet writes it at once, before any sample, with the + number that the next sample will get. Each connection thus starts with an + absolute number, and a gap before the connection shows at once, also on + a stream that sends no more samples. 4. After that, the outlet writes a record only before a sample whose number is not one more than the number of the sample before it. A connection with no loss carries one record and no more. @@ -179,9 +182,11 @@ The inlet keeps the number that it expects next. takes the new number. The arithmetic must not underflow. 4. Each sample adds one to the expected number. 5. A recovery to the same outlet (the same `uid`) keeps the expected number. - The first record of the new connection then shows the gap. + The first record of the new connection then shows the gap. Live test: + `a_gap_shows_at_once_on_a_quiet_stream`. 6. A recovery to a new outlet (a new `uid`) starts again at rule 1, because - the new outlet has a new counter. + the new outlet has a new counter. Live test: + `a_new_outlet_starts_a_new_count`. #### What the inlet reports @@ -251,7 +256,9 @@ Each item gives its status on the branch `feat/seq`. 1. **liblsl outlet, raw request.** Send a feed request with the header to the oracle outlet. Make sure that the answer and the data bytes are the same as for the request without the header. **Done:** `oracle/headerprobe.py` - passes for liblsl and for labstream `main`. + passes for liblsl and for labstream `main`. It also reads the first data + byte: a sample tag (1 or 2) for each outlet, and the record tag `0xFE` + only when the answer holds `seq/1`. 2. **liblsl inlet.** Make sure that a labstream outlet sends no `Labstream-Features` line, and no record, when the request has none. **Done:** `crates/labstream-net/tests/seq_live.rs`,