From 9b6c19e2509db5dac84ef7b24a8d955f01cdffe6 Mon Sep 17 00:00:00 2001 From: rednayan Date: Mon, 16 Nov 2026 14:47:40 +0100 Subject: [PATCH 1/2] Measure the latency from a push to a pull oracle/latency.c is one C program for the C ABI. It runs as an outlet or as an inlet, and reads its own CLOCK_MONOTONIC. oracle/latency.py links it to each library, runs each pair more than once, and makes a table. Each row names the library that the loader picked. With --control, a known delay of 1000 us must show in the table. docs/latency.md gives the method and the first results for liblsl, main, fix/network-robustness and perf/quick-wins. At 4000 Hz x 64 channels in chunks of 32, main uses 9 to 11 % CPU on each side. The fix branch uses 2 to 3 %, near liblsl. --- conformance/oracle/latency.c | 253 ++++++++++++++++++++++++++++++++++ conformance/oracle/latency.py | 205 +++++++++++++++++++++++++++ docs/latency.md | 119 ++++++++++++++++ 3 files changed, 577 insertions(+) create mode 100644 conformance/oracle/latency.c create mode 100755 conformance/oracle/latency.py create mode 100644 docs/latency.md diff --git a/conformance/oracle/latency.c b/conformance/oracle/latency.c new file mode 100644 index 0000000..bd55098 --- /dev/null +++ b/conformance/oracle/latency.c @@ -0,0 +1,253 @@ +/* Measure the time from a push in one process to a pull in another. + * + * The program runs as an outlet or as an inlet. Link it once against real + * liblsl and once against the Rust library. The same program then measures + * both libraries, and the two can also measure each other. + * + * The clock is CLOCK_MONOTONIC of this program, not the clock of the + * library. The outlet passes the push time as the timestamp of each sample. + * The inlet does no post-processing, so it reads the push time back + * unchanged. Both processes run on one machine and read the same clock. + * + * Channel 0 carries the sample number. The inlet uses it to count lost, + * repeated, and reordered samples. + * + * Each process writes one JSON line to stdout when it ends. + * `oracle/latency.py` runs the pairs and makes the table. + * + * Build: + * cc -O2 -I .build/install/include oracle/latency.c -o BIN -lm -ldl \ + * -L DIR -llsl -Wl,-rpath,DIR + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +#include + +static double now(void) { + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + return (double)t.tv_sec + (double)t.tv_nsec * 1e-9; +} + +static void sleep_until(double t) { + struct timespec ts; + ts.tv_sec = (time_t)t; + ts.tv_nsec = (long)((t - (double)ts.tv_sec) * 1e9); + while (clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &ts, NULL) != 0) { + } +} + +/* User plus system CPU time of this process, in seconds. */ +static double cpu_seconds(void) { + struct rusage r; + getrusage(RUSAGE_SELF, &r); + return (double)r.ru_utime.tv_sec + (double)r.ru_utime.tv_usec * 1e-6 + + (double)r.ru_stime.tv_sec + (double)r.ru_stime.tv_usec * 1e-6; +} + +/* The file of the library that the loader picked. An earlier run used the + * wrong library through a stale RUNPATH, so each row names its library. */ +static const char *library_path(void) { + Dl_info d; + if (dladdr((void *)lsl_library_info, &d) && d.dli_fname) return d.dli_fname; + return "unknown"; +} + +static int cmp_double(const void *a, const void *b) { + double x = *(const double *)a, y = *(const double *)b; + return (x > y) - (x < y); +} + +/* The value at fraction q of a sorted array. */ +static double quantile(const double *v, long n, double q) { + if (n <= 0) return NAN; + long k = (long)(q * (double)(n - 1) + 0.5); + return v[k]; +} + +static void print_library(void) { + printf("\"library\": \"%s\", \"library_info\": \"%s\"", library_path(), lsl_library_info()); +} + +static int run_outlet(const char *name, int channels, double rate, int chunk, double seconds, + double offset_us) { + lsl_streaminfo info = + lsl_create_streaminfo(name, "Latency", channels, rate, cft_float32, name); + lsl_outlet out = lsl_create_outlet(info, 0, 360); + if (!out) { + fprintf(stderr, "ERROR could not create the outlet\n"); + return 1; + } + fprintf(stderr, "READY\n"); + if (!lsl_wait_for_consumers(out, 20.0)) { + fprintf(stderr, "ERROR no consumer in 20 s\n"); + return 1; + } + + long total = (long)(rate * seconds); + long pushes = (total + chunk - 1) / chunk; + float *data = calloc((size_t)chunk * (size_t)channels, sizeof(float)); + double *stamps = calloc((size_t)chunk, sizeof(double)); + double *lag = calloc((size_t)pushes, sizeof(double)); + + /* The inlet starts its reader at the first pull, so the first samples + * wait in the outlet for a short time. Give it time to start. */ + sleep_until(now() + 0.5); + + double cpu0 = cpu_seconds(); + double t0 = now(); + long k = 0; + for (long p = 0; p < pushes; p++) { + long n = total - k < chunk ? total - k : chunk; + double due = t0 + (double)(k + n) / rate; + sleep_until(due); + for (long i = 0; i < n; i++) { + data[i * channels] = (float)(k + i); + for (int c = 1; c < channels; c++) data[i * channels + c] = (float)c; + } + double t = now(); + lag[p] = t - due; + for (long i = 0; i < n; i++) stamps[i] = t - offset_us * 1e-6; + if (n == 1) + lsl_push_sample_ft(out, data, stamps[0]); + else + lsl_push_chunk_ftn(out, data, (unsigned long)(n * channels), stamps); + k += n; + } + double wall = now() - t0; + double cpu = cpu_seconds() - cpu0; + + /* Let the last samples leave before the outlet closes. */ + sleep_until(now() + 1.0); + + qsort(lag, (size_t)pushes, sizeof(double), cmp_double); + printf("{\"role\": \"outlet\", "); + print_library(); + printf(", \"sent\": %ld, \"pushes\": %ld, \"wall_s\": %.3f, \"cpu_pct\": %.3f, " + "\"lag_p50_us\": %.1f, \"lag_p99_us\": %.1f, \"lag_max_us\": %.1f}\n", + total, pushes, wall, 100.0 * cpu / wall, quantile(lag, pushes, 0.5) * 1e6, + quantile(lag, pushes, 0.99) * 1e6, lag[pushes - 1] * 1e6); + fflush(stdout); + lsl_destroy_outlet(out); + return 0; +} + +static int run_inlet(const char *name, int channels, double rate, double seconds, + double warmup) { + lsl_streaminfo found[1]; + if (lsl_resolve_byprop(found, 1, "name", name, 1, 20.0) < 1) { + fprintf(stderr, "ERROR the stream %s was not found\n", name); + return 1; + } + lsl_inlet in = lsl_create_inlet(found[0], 360, 0, 1); + int32_t ec = 0; + lsl_open_stream(in, 20.0, &ec); + if (ec) { + fprintf(stderr, "ERROR open_stream gave %d\n", ec); + return 1; + } + fprintf(stderr, "READY\n"); + + long total = (long)(rate * seconds); + long skip = (long)(rate * warmup); + double *lat = calloc((size_t)total + 1, sizeof(double)); + unsigned char *seen = calloc((size_t)total + 1, 1); + float *buf = calloc((size_t)channels, sizeof(float)); + long got = 0, kept = 0, repeated = 0, reordered = 0, alien = 0; + long last = -1; + double cpu0 = 0, t0 = 0; + + for (;;) { + double ts = lsl_pull_sample_f(in, buf, channels, 3.0, &ec); + double t = now(); + if (ts == 0.0 || ec) break; /* no sample in 3 s: the outlet is gone */ + if (got == 0) { + cpu0 = cpu_seconds(); + t0 = t; + } + got++; + long idx = (long)buf[0]; + if (idx < 0 || idx >= total) { + alien++; + continue; + } + if (seen[idx]) { + repeated++; + continue; + } + seen[idx] = 1; + if (idx < last) reordered++; + last = idx; + if (idx >= skip) lat[kept++] = t - ts; + if (idx == total - 1) break; + } + double wall = now() - t0; + double cpu = cpu_seconds() - cpu0; + + long lost = 0; + for (long i = 0; i < total; i++) + if (!seen[i]) lost++; + + qsort(lat, (size_t)kept, sizeof(double), cmp_double); + printf("{\"role\": \"inlet\", "); + print_library(); + printf(", \"expected\": %ld, \"received\": %ld, \"lost\": %ld, \"repeated\": %ld, " + "\"reordered\": %ld, \"alien\": %ld, \"measured\": %ld, \"wall_s\": %.3f, " + "\"cpu_pct\": %.3f, \"p50_us\": %.1f, \"p90_us\": %.1f, \"p99_us\": %.1f, " + "\"p999_us\": %.1f, \"max_us\": %.1f, \"min_us\": %.1f}\n", + total, got, lost, repeated, reordered, alien, kept, wall, + wall > 0 ? 100.0 * cpu / wall : 0.0, quantile(lat, kept, 0.5) * 1e6, + quantile(lat, kept, 0.9) * 1e6, quantile(lat, kept, 0.99) * 1e6, + quantile(lat, kept, 0.999) * 1e6, kept ? lat[kept - 1] * 1e6 : NAN, + kept ? lat[0] * 1e6 : NAN); + fflush(stdout); + lsl_destroy_inlet(in); + return 0; +} + +int main(int argc, char **argv) { + const char *role = NULL; + const char *name = "LatencyBench"; + int channels = 8; + double rate = 512.0; + int chunk = 1; + double seconds = 10.0; + double warmup = 1.0; + double offset_us = 0.0; + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "outlet") || !strcmp(argv[i], "inlet")) { + role = argv[i]; + continue; + } + if (i + 1 >= argc) { + fprintf(stderr, "the option %s has no value\n", argv[i]); + return 2; + } + if (!strcmp(argv[i], "--name")) name = argv[++i]; + else if (!strcmp(argv[i], "--channels")) channels = atoi(argv[++i]); + else if (!strcmp(argv[i], "--rate")) rate = atof(argv[++i]); + else if (!strcmp(argv[i], "--chunk")) chunk = atoi(argv[++i]); + else if (!strcmp(argv[i], "--seconds")) seconds = atof(argv[++i]); + else if (!strcmp(argv[i], "--warmup")) warmup = atof(argv[++i]); + else if (!strcmp(argv[i], "--offset-us")) offset_us = atof(argv[++i]); + else { + fprintf(stderr, "unknown option %s\n", argv[i]); + return 2; + } + } + if (!role || channels < 1 || rate <= 0 || chunk < 1 || seconds <= 0) { + fprintf(stderr, "usage: latency outlet|inlet [--name N] [--channels C] [--rate R] " + "[--chunk K] [--seconds S] [--warmup S] [--offset-us U]\n"); + return 2; + } + if (!strcmp(role, "outlet")) return run_outlet(name, channels, rate, chunk, seconds, offset_us); + return run_inlet(name, channels, rate, seconds, warmup); +} diff --git a/conformance/oracle/latency.py b/conformance/oracle/latency.py new file mode 100755 index 0000000..48d4897 --- /dev/null +++ b/conformance/oracle/latency.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Measure the push-to-pull latency of each library, and of each mixed pair. + +The script builds `oracle/latency.c` once for each library, runs an outlet +process and an inlet process for each cell, and writes one table. Each cell +runs more than once, because one run of a timing test proves little. + +The control row is liblsl to liblsl. With `--control`, the script also runs +that pair with a known delay in the timestamps, and the table must show the +same delay. If it does not, the instrument is wrong, not the library. + +Usage: + oracle/latency.py + oracle/latency.py --lib oracle=.build/install/lib --lib main=/path/to/dir + oracle/latency.py --loads 512x8x1,4000x64x32 --seconds 10 --repeat 3 + oracle/latency.py --control + +A load `RATExCHANNELSxCHUNK` pushes CHUNK samples in each push call. The +first `--lib` is the reference. The script runs each library against itself, +and the reference against each other library, in both directions. +""" + +import argparse +import json +import os +import statistics +import subprocess +import sys +import tempfile +import threading +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + +DEFAULT_LOADS = "512x8x1,512x8x16,4000x64x1,4000x64x32" + + +def build(source: Path, lib_dir: Path, out: Path) -> None: + cmd = [ + "cc", "-O2", "-I", str(ROOT / ".build" / "install" / "include"), str(source), + "-o", str(out), "-lm", "-ldl", "-L", str(lib_dir), "-llsl", + f"-Wl,-rpath,{lib_dir}", + ] + subprocess.run(cmd, check=True) + + +def wait_ready(proc: subprocess.Popen, what: str, lines: list) -> None: + """Read stderr until the READY line. Keep the rest for an error report.""" + for raw in proc.stderr: + line = raw.decode(errors="replace").rstrip() + lines.append(line) + if line == "READY": + # Drain the rest in the background, so the pipe never fills. + threading.Thread(target=lambda: [lines.append(r.decode(errors="replace").rstrip()) + for r in proc.stderr], daemon=True).start() + return + raise RuntimeError(f"{what}: the peer stopped before READY: {lines[-5:]}") + + +def last_json(text: str) -> dict: + for line in reversed(text.splitlines()): + line = line.strip() + if line.startswith("{"): + return json.loads(line) + raise RuntimeError("no JSON line in the output") + + +def run_pair(out_bin: Path, in_bin: Path, name: str, rate: float, channels: int, chunk: int, + seconds: float, offset_us: float) -> tuple: + common = ["--name", name, "--rate", str(rate), "--channels", str(channels), + "--seconds", str(seconds)] + out_err, in_err = [], [] + outlet = subprocess.Popen([str(out_bin), "outlet", *common, "--chunk", str(chunk), + "--offset-us", str(offset_us)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + wait_ready(outlet, "outlet", out_err) + inlet = subprocess.Popen([str(in_bin), "inlet", *common], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + wait_ready(inlet, "inlet", in_err) + in_out = inlet.stdout.read().decode() + inlet.wait(timeout=seconds + 30) + finally: + if inlet.poll() is None: + inlet.kill() + out_out = outlet.stdout.read().decode() + outlet.wait(timeout=30) + finally: + if outlet.poll() is None: + outlet.kill() + return last_json(out_out), last_json(in_out) + + +def spread(values: list) -> str: + if len(values) == 1: + return f"{values[0]:.0f}" + return f"{statistics.median(values):.0f} ({min(values):.0f}-{max(values):.0f})" + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--lib", action="append", default=None, metavar="NAME=DIR", + help="a library directory that holds liblsl.so (repeat it)") + ap.add_argument("--loads", default=DEFAULT_LOADS) + ap.add_argument("--seconds", type=float, default=10.0) + ap.add_argument("--repeat", type=int, default=3) + ap.add_argument("--no-mixed", action="store_true", + help="run each library only against itself") + ap.add_argument("--control", action="store_true", + help="also measure a known delay of 1000 us with the reference pair") + ap.add_argument("--json", default=None, help="write every run to this file") + args = ap.parse_args() + + libs = args.lib or [f"oracle={ROOT / '.build' / 'install' / 'lib'}", + f"rust={ROOT / '.build' / 'rustlib'}"] + named = [] + for item in libs: + name, _, path = item.partition("=") + named.append((name, Path(path).resolve())) + + work = Path(tempfile.mkdtemp(prefix="latency-")) + bins = {} + for name, path in named: + bins[name] = work / f"latency_{name}" + build(ROOT / "oracle" / "latency.c", path, bins[name]) + + ref = named[0][0] + pairs = [(n, n) for n, _ in named] + if not args.no_mixed: + for n, _ in named[1:]: + pairs += [(ref, n), (n, ref)] + + loads = [] + for spec in args.loads.split(","): + rate, channels, chunk = spec.split("x") + loads.append((float(rate), int(channels), int(chunk))) + + runs = [] + counter = 0 + + def one(out_name, in_name, rate, channels, chunk, offset_us=0.0): + nonlocal counter + counter += 1 + stream = f"Lat{os.getpid()}_{counter}" + o, i = run_pair(bins[out_name], bins[in_name], stream, rate, channels, chunk, + args.seconds, offset_us) + row = {"outlet": out_name, "inlet": in_name, "rate": rate, "channels": channels, + "chunk": chunk, "offset_us": offset_us, "outlet_run": o, "inlet_run": i} + runs.append(row) + print(f" {out_name:>10} -> {in_name:<10} {rate:g} Hz x {channels} ch, chunk {chunk}: " + f"p50 {i['p50_us']:.0f} us, p99 {i['p99_us']:.0f} us, lost {i['lost']}", + file=sys.stderr) + return row + + for rate, channels, chunk in loads: + for out_name, in_name in pairs: + for _ in range(args.repeat): + one(out_name, in_name, rate, channels, chunk) + + control_ok = None + if args.control: + base = [one(ref, ref, 512.0, 8, 1)["inlet_run"]["p50_us"] for _ in range(args.repeat)] + late = [one(ref, ref, 512.0, 8, 1, 1000.0)["inlet_run"]["p50_us"] + for _ in range(args.repeat)] + shift = statistics.median(late) - statistics.median(base) + control_ok = abs(shift - 1000.0) < 100.0 + print(f"\ncontrol: a delay of 1000 us shows as {shift:.0f} us " + f"({'pass' if control_ok else 'FAIL'})") + + print("\n| outlet -> inlet | load | p50 us | p99 us | p99.9 us | max us | lost | " + "CPU outlet % | CPU inlet % |") + print("|---|---|---|---|---|---|---|---|---|") + for rate, channels, chunk in loads: + for out_name, in_name in pairs: + cell = [r for r in runs if r["outlet"] == out_name and r["inlet"] == in_name + and r["rate"] == rate and r["channels"] == channels + and r["chunk"] == chunk and r["offset_us"] == 0.0] + if not cell: + continue + ins = [r["inlet_run"] for r in cell] + outs = [r["outlet_run"] for r in cell] + print(f"| {out_name} -> {in_name} | {rate:g} Hz x {channels}, chunk {chunk} | " + f"{spread([x['p50_us'] for x in ins])} | {spread([x['p99_us'] for x in ins])} | " + f"{spread([x['p999_us'] for x in ins])} | {max(x['max_us'] for x in ins):.0f} | " + f"{sum(x['lost'] + x['repeated'] + x['alien'] for x in ins)} | " + f"{statistics.median(x['cpu_pct'] for x in outs):.1f} | " + f"{statistics.median(x['cpu_pct'] for x in ins):.1f} |") + print(f"\nEach cell: median of {args.repeat} runs, with the range in brackets. " + f"Each run: {args.seconds:g} s. The first second is not measured.") + print("Libraries:") + for name, path in named: + seen = {r["inlet_run"]["library"] for r in runs if r["inlet"] == name} + print(f" {name}: {path} (loaded: {', '.join(sorted(seen))})") + + if args.json: + Path(args.json).write_text(json.dumps(runs, indent=1)) + if control_ok is False: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/latency.md b/docs/latency.md new file mode 100644 index 0000000..5a5e33e --- /dev/null +++ b/docs/latency.md @@ -0,0 +1,119 @@ +# Latency and CPU + +This page gives how to measure the time from a push to a pull, and the +results of the first measurement. Run the measurement again before you claim +that a change makes the library faster. + +## The instrument + +`conformance/oracle/latency.c` is one C program for the C ABI. It runs as an +outlet or as an inlet. The runner `conformance/oracle/latency.py` links it +once to each library, then runs an outlet process and an inlet process for +each cell. + +- **The clock.** The program reads its own `CLOCK_MONOTONIC`, not the clock + of the library. The outlet gives the push time as the timestamp of each + sample. The inlet does no post-processing, and records the pull time minus + the timestamp. +- **The chunk.** All samples of one push call get the push time. The latency + of a chunk is thus the latency of its last sample to leave. +- **The pull.** The inlet calls `lsl_pull_sample_f` with a timeout of 3 s. The + call returns when a sample arrives, so no poll interval adds latency. +- **Loss.** Channel 0 carries the sample number. The inlet counts lost, + repeated, and reordered samples. +- **The library.** Each process reports the file of the library that the + loader picked. A stale `RUNPATH` once made a test use the wrong library. +- **CPU.** Each process reports its user and system time, as a percentage of + one core, over the time that it sends or receives. +- **The schedule.** The outlet waits on an absolute schedule, and reports how + late each push is. This lag is not part of the latency. + +## The control + +The control pair is liblsl to liblsl. With `--control`, the runner also runs +that pair with timestamps that are 1000 µs early. The table must show a delay +that is 1000 µs larger. On the run below, the delay showed as 966 µs. + +## How to run it + +Run these commands from `conformance/`. The oracle build must exist +(`conformance/oracle/README.md`). + +```bash +oracle/latency.py --control +``` + +To compare branches, build the C library of each branch in its own worktree +and target directory. Then give each directory to the runner: + +```bash +oracle/latency.py --lib oracle=.build/install/lib --lib main=DIR1 --lib fix=DIR2 --no-mixed --control +``` + +Do not copy a branch build into `.build/rustlib`. Other scripts read that +directory. + +## The first results + +Machine: AMD Ryzen 5 PRO 3500U, 8 threads, Linux 6.16, loopback, no other +load on purpose. Each cell is the median of 3 runs of 10 s, with the range of +the 3 runs in brackets. The first second of each run is not measured. + +Libraries: + +- `oracle`: liblsl at `e651023c`. +- `main`: labstream `a912cef`. +- `fix`: branch `fix/network-robustness` at `e01e998` (pull request #9). +- `quick`: branch `perf/quick-wins` at `d6880e4` (pull request #10). + +| outlet -> inlet | load | p50 µs | p99 µs | p99.9 µs | max µs | lost | CPU outlet % | CPU inlet % | +|---|---|---|---|---|---|---|---|---| +| oracle -> oracle | 512 Hz x 8, chunk 1 | 102 (93-142) | 327 (201-501) | 419 (231-624) | 2150 | 0 | 5.7 | 3.4 | +| main -> main | 512 Hz x 8, chunk 1 | 96 (94-96) | 212 (204-217) | 465 (260-1808) | 2932 | 0 | 4.2 | 3.2 | +| fix -> fix | 512 Hz x 8, chunk 1 | 96 (94-97) | 204 (200-235) | 254 (246-1011) | 3218 | 0 | 4.2 | 3.2 | +| quick -> quick | 512 Hz x 8, chunk 1 | 94 (92-98) | 204 (194-239) | 419 (308-557) | 2382 | 0 | 4.1 | 3.2 | +| oracle -> oracle | 512 Hz x 8, chunk 16 | 246 (224-247) | 646 (599-648) | 742 (729-779) | 780 | 0 | 0.7 | 0.4 | +| main -> main | 512 Hz x 8, chunk 16 | 290 (287-297) | 667 (614-1054) | 1025 (703-2150) | 2151 | 0 | 2.0 | 1.5 | +| fix -> fix | 512 Hz x 8, chunk 16 | 220 (218-238) | 514 (464-514) | 617 (480-624) | 626 | 0 | 1.1 | 0.5 | +| quick -> quick | 512 Hz x 8, chunk 16 | 220 (214-236) | 531 (495-536) | 696 (676-754) | 756 | 0 | 1.1 | 0.5 | +| oracle -> oracle | 4000 Hz x 64, chunk 1 | 71 (70-72) | 116 (114-117) | 152 (152-164) | 2803 | 0 | 30.4 | 18.4 | +| main -> main | 4000 Hz x 64, chunk 1 | 73 (73-75) | 116 (114-119) | 154 (149-172) | 755 | 0 | 21.2 | 18.4 | +| fix -> fix | 4000 Hz x 64, chunk 1 | 75 (74-76) | 121 (116-165) | 168 (159-2487) | 5199 | 0 | 21.5 | 18.2 | +| quick -> quick | 4000 Hz x 64, chunk 1 | 73 (71-74) | 125 (105-145) | 224 (157-1681) | 3985 | 0 | 20.5 | 17.5 | +| oracle -> oracle | 4000 Hz x 64, chunk 32 | 318 (317-320) | 651 (646-667) | 768 (764-844) | 876 | 0 | 3.3 | 1.7 | +| main -> main | 4000 Hz x 64, chunk 32 | 382 (381-406) | 795 (756-852) | 1108 (892-1309) | 2916 | 0 | 9.3 | 10.8 | +| fix -> fix | 4000 Hz x 64, chunk 32 | 343 (310-355) | 664 (626-699) | 810 (796-1063) | 3097 | 0 | 3.1 | 2.2 | +| quick -> quick | 4000 Hz x 64, chunk 32 | 328 (322-339) | 658 (657-671) | 829 (797-832) | 892 | 0 | 3.2 | 2.2 | + +### What the results show + +- **A chunk push on `main` costs 3 to 5 times the CPU of liblsl.** At + 4000 Hz x 64 channels in chunks of 32, `main` uses 9.3 % and 10.8 %. The + `fix` branch uses 3.1 % and 2.2 %, near liblsl. The `fix` branch writes each chunk + with one write, and locks the inlet queue once for each read. +- **A chunk push on `main` is also slower.** The median is 382 µs on `main`, + 343 µs on `fix`, and 318 µs on liblsl. +- **A push of one sample gives the same latency in all four builds.** The + median is 70 to 100 µs. At 4000 Hz x 64 channels, the labstream outlet uses + about 21 % of one core, and the liblsl outlet uses 30 %. +- **No build lost a sample.** +- **The maximum is noise.** A single value of 2 to 5 ms comes from the + scheduler of the operating system. The p99.9 column is a better measure of + the tail. +- **The `quick` branch does not change the latency of the data path.** It + changes the start of a connection and the end of a dead connection. This + benchmark measures neither. + +## What this benchmark does not measure + +- **Two machines.** All results come from the loopback interface. +- **A slow or lossy network.** Do not add a `tc netem` rule to the `lo` + interface of a shared machine. The rule delays every program on the machine. + Use a network namespace instead. Make two namespaces with a `veth` pair, + add the `netem` rule inside one of them, and run the outlet and the inlet in + different namespaces. Discovery then needs the namespace addresses in + `lab.KnownPeers` of the configuration. This procedure was not run yet. +- **The application.** Phasic and other programs pull in their own loop. Their + poll interval adds to these numbers. +- **The first offset and the recovery.** The live tests in + `crates/labstream-net/tests/` measure these. From c4eb99b257ea33f7d14e57218bb4c2d05b1d0594 Mon Sep 17 00:00:00 2001 From: rednayan Date: Mon, 16 Nov 2026 14:47:40 +0100 Subject: [PATCH 2/2] Propose a feature header between two labstream peers docs/extensions.md proposes the request header Labstream-Features. A labstream outlet answers it, and liblsl ignores it on both sides. The page gives the syntax, the rules, the byte where a feature can change the framing, the first features (seq and shm), the rejected alternatives, and the conformance plan. Nothing is implemented. oracle/headerprobe.py sends a feed request with and without the header to a real outlet. The answer headers and the test pattern must be the same. It passes for liblsl and for labstream main. A probe with a known header (Native-Byte-Order) shows a difference, so the probe can fail. --- CHANGELOG.md | 7 ++ conformance/oracle/headerprobe.py | 152 ++++++++++++++++++++++ docs/extensions.md | 202 ++++++++++++++++++++++++++++++ docs/repository.md | 2 + 4 files changed, 363 insertions(+) create mode 100755 conformance/oracle/headerprobe.py create mode 100644 docs/extensions.md diff --git a/CHANGELOG.md b/CHANGELOG.md index e2c1ac9..f565f2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added +- `docs/latency.md` and `conformance/oracle/latency.py`, with + `conformance/oracle/latency.c`. They measure the time from a push to a pull, + and the CPU of each side, for liblsl and labstream. +- `docs/extensions.md`, a proposal for a feature header between two labstream + peers. Nothing in it is implemented. +- `conformance/oracle/headerprobe.py`. It makes sure that an outlet ignores + the proposed header. - `docs/repository.md`. It gives the layout of the repository, how the workspace joins the crates, the daily commands, CI, and the dependency updates. diff --git a/conformance/oracle/headerprobe.py b/conformance/oracle/headerprobe.py new file mode 100755 index 0000000..2f2cbcf --- /dev/null +++ b/conformance/oracle/headerprobe.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Make sure that an outlet ignores the `Labstream-Features` request header. + +`docs/extensions.md` proposes this header. The proposal depends on one fact: +an outlet that does not know the header gives the same answer with it and +without it. This probe measures that fact on a real outlet. + +The probe starts `oracle/publish.c`, linked to one library. It opens two feed +connections by hand. One request has the extra header, and one does not. +The answer headers and the two test pattern samples must be the same, byte +for byte. + +Usage: + oracle/headerprobe.py # liblsl and the Rust library + oracle/headerprobe.py --lib oracle=.build/install/lib +""" + +import argparse +import os +import re +import socket +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +CHANNELS = 8 + +# The request that liblsl sends (`src/data_receiver.cpp:170-190`), with fixed +# values. +BASE_HEADERS = [ + "Native-Byte-Order: 1234", + "Endian-Performance: 0", + "Has-IEEE754-Floats: 1", + "Supports-Subnormals: 1", + "Value-Size: 4", + "Data-Protocol-Version: 110", + "Max-Buffer-Length: 360", + "Max-Chunk-Length: 0", + "Hostname: probe", + "Source-Id: ", + "Session-Id: default", +] +EXTRA = "Labstream-Features: seq/1 shm/1=0a1b2c" + + +def build(lib_dir: Path, out: Path) -> None: + subprocess.run(["cc", "-I", str(ROOT / ".build" / "install" / "include"), + str(ROOT / "oracle" / "publish.c"), "-o", str(out), "-lm", + "-L", str(lib_dir), "-llsl", f"-Wl,-rpath,{lib_dir}"], check=True) + + +def find_stream(lib_dir: Path, name: str) -> tuple: + """Resolve the stream with pylsl and give its uid and IPv4 data port.""" + code = ( + "import pylsl,sys\n" + f"r=pylsl.resolve_byprop('name','{name}',1,10)\n" + "print(r[0].as_xml() if r else '')\n" + ) + lib = next(p for p in sorted(lib_dir.glob("liblsl.so*"))) + env = dict(os.environ, PYLSL_LIB=str(lib)) + xml = subprocess.run([sys.executable, "-c", code], env=env, capture_output=True, + text=True, timeout=30).stdout + uid = re.search(r"([^<]*)", xml) + port = re.search(r"(\d+)", xml) + if not uid or not port: + raise RuntimeError("the stream was not found") + return uid.group(1), int(port.group(1)) + + +def read_exact(sock: socket.socket, n: int) -> bytes: + out = b"" + while len(out) < n: + part = sock.recv(n - len(out)) + if not part: + raise RuntimeError(f"the connection closed after {len(out)} bytes") + out += part + return out + + +def feed(port: int, uid: str, extra: bool) -> tuple: + """Send one feed request. Give the answer headers and the test pattern.""" + lines = [f"LSL:streamfeed/110 {uid}", *BASE_HEADERS] + if extra: + lines.append(EXTRA) + request = "\r\n".join(lines) + "\r\n\r\n" + with socket.create_connection(("127.0.0.1", port), timeout=5) as s: + s.sendall(request.encode()) + head = b"" + while b"\r\n\r\n" not in head: + part = s.recv(1) + if not part: + raise RuntimeError("the connection closed in the headers") + head += part + # Each test pattern sample: a tag, a timestamp, and the values. + # SPEC.md 6.4. + pattern = read_exact(s, 2 * (1 + 8 + 4 * CHANNELS)) + return head.decode(errors="replace"), pattern + + +def probe(name: str, lib_dir: Path, work: Path) -> bool: + binary = work / f"publish_{name}" + build(lib_dir, binary) + stream = f"HeaderProbe{os.getpid()}{name}" + pub = subprocess.Popen([str(binary), "--name", stream, "--channels", str(CHANNELS), + "--rate", "100", "--seconds", "20"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + time.sleep(1.0) + uid, port = find_stream(lib_dir, stream) + plain_head, plain_pattern = feed(port, uid, extra=False) + with_head, with_pattern = feed(port, uid, extra=True) + finally: + pub.kill() + pub.wait() + same = plain_head == with_head and plain_pattern == with_pattern + leaked = "labstream-features" in with_head.lower() + ok = same and not leaked + print(f" {name:<8} {'pass' if ok else 'FAIL'} answer headers " + f"{'same' if plain_head == with_head else 'DIFFERENT'}, test pattern " + f"{'same' if plain_pattern == with_pattern else 'DIFFERENT'}, " + f"{'a Labstream-Features answer' if leaked else 'no Labstream-Features answer'}") + if not ok: + print(" without:", plain_head.replace("\r\n", " | ")) + print(" with: ", with_head.replace("\r\n", " | ")) + return ok + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--lib", action="append", default=None, metavar="NAME=DIR") + args = ap.parse_args() + libs = args.lib or [f"oracle={ROOT / '.build' / 'install' / 'lib'}", + f"rust={ROOT / '.build' / 'rustlib'}"] + work = Path(tempfile.mkdtemp(prefix="headerprobe-")) + results = [] + for item in libs: + name, _, path = item.partition("=") + results.append(probe(name, Path(path).resolve(), work)) + print() + if all(results): + print("each outlet ignores the Labstream-Features header") + return 0 + print("an outlet reacts to the Labstream-Features header") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/extensions.md b/docs/extensions.md new file mode 100644 index 0000000..24f83d3 --- /dev/null +++ b/docs/extensions.md @@ -0,0 +1,202 @@ +# Proposal: feature negotiation between two labstream peers + +**Status:** proposal. Nothing on this page is implemented. The wire format does +not change until a feature section below is accepted and SPEC.md gets a +section for it. + +## The purpose + +Some improvements need a change to the data channel. Examples are sequence +numbers that make sample loss visible, and a shared-memory transport for two +processes on one machine. liblsl does not know these changes. + +This page defines how a labstream inlet and a labstream outlet agree to use +such a feature. The agreement must leave every other pair unchanged: + +| Outlet | Inlet | Result | +|---|---|---| +| liblsl | liblsl | plain LSL, as today | +| liblsl | labstream | plain LSL | +| labstream | liblsl | plain LSL | +| labstream 0.1.2 or earlier | labstream | plain LSL | +| labstream | labstream | plain LSL, plus each feature that both ends accept | + +## What liblsl does with an extra header + +These facts come from `conformance/liblsl/` at `e651023c`. + +1. The feed server reads each request header line, and acts only on the keys + that it knows. An unknown key has no effect + (`src/tcp_server.cpp:605-633`). +2. The feed client reads each answer header line the same way, and acts only + on the keys that it knows (`src/data_receiver.cpp:215-253`). +3. Both sides cut a header line at the first `;`, and convert the line to + lowercase before they compare the key (`src/tcp_server.cpp:610-613`, + `src/data_receiver.cpp:220-223`). +4. Both sides read a header line into a buffer of 16384 bytes + (`src/tcp_server.cpp:604`, `src/data_receiver.cpp:193`). +5. The labstream parser cuts and converts the same way + (`crates/labstream-proto/src/header.rs:103-107`). + +As a result, a liblsl outlet ignores an extra request header, and a liblsl +inlet ignores an extra answer header. Item 3 limits the syntax of the header. +A value must not contain `;`, and a value must not depend on letter case. + +## The header + +### The request + +A labstream inlet adds one header line to its feed request: + +``` +Labstream-Features: seq/1 shm/1 +``` + +The value is a list of tokens with one space between two tokens. Each token +has this form: + +``` +token = name "/" version [ "=" param ] +name = 1*( a-z / 0-9 / "-" ) +version = 1*( 0-9 ) +param = 1*( 0-9 / a-f ) +``` + +A parameter is hexadecimal, because the header line loses its letter case. +The whole line must stay below 16384 bytes. An inlet can offer more than one +version of a feature, for example `seq/1 seq/2`. + +### The answer + +A labstream outlet that reads the request header adds one line to its answer: + +``` +Labstream-Features: seq/1 +``` + +The answer holds only tokens from the request. For each name, it holds at +most one version. + +### The rules + +1. An inlet sends the request header only when it supports one or more + features. +2. An outlet sends the answer header only when the request held one. +3. An outlet ignores a token that it does not know, or that does not match + the syntax. +4. An inlet ignores an answer token that it did not offer. +5. A feature is on only when the inlet offered it and the outlet answered it. +6. If the answer has no `Labstream-Features` line, no feature is on. A liblsl + outlet and an older labstream outlet give this result. +7. A configuration key can remove a feature from the offer and from the + answer. The name of the key is an open question. + +## Where the framing changes + +After the answer headers, the outlet sends two test pattern samples +(SPEC.md 6). These two samples always use the plain framing of SPEC.md 7. +The inlet compares them exactly as today. + +A feature can change the framing only from the first byte after the second +test pattern sample. Thus the test pattern stays a check of the codec, and a +fault in a feature cannot hide a fault in the codec. + +## The scope of an agreement + +An agreement holds for one connection. A recovery opens a new connection with +a new request, so the set of features can change. An outlet that restarts as +a liblsl program gives an example. + +The inlet must report the features of the current connection, and must show +when the set changes. This works the same way as the clock reset after a +recovery. The API names are an open question. + +## The first features + +Each feature below gets its own section and its own conformance checks +before any code. The text here gives only the requirements that affect the +negotiation. + +### `seq`: sequence numbers + +The outlet gives each sample a number when the application pushes it. One +counter serves the whole outlet, not one counter for each connection. + +This rule is necessary. The two real causes of loss are a full consumer +queue (SPEC.md 8.7) and the gap between a lost connection and the next one. +A counter for each connection starts again after a recovery, so it shows +neither cause. + +The inlet reports the number of samples that it did not receive. The counter +also shows a repeated sample and a sample out of order. + +### `shm`: shared memory on one machine + +The hostname does not prove that the two processes share a machine. Two +machines can have the same name, and a container can have the name of its +host. Thus `shm` needs one more step after the headers: + +1. The inlet offers `shm/1`. +2. The outlet makes a memory segment. It writes a random value into the + segment. It answers `shm/1=`. +3. The inlet opens the segment and reads the random value. +4. The inlet writes one line on the TCP connection: `shm-ok ` or + `shm-no`. +5. If the value matches, the outlet sends the samples through the segment. + Otherwise, or after 1 s with no line, the outlet sends the samples on TCP. + +The outlet does not send data before step 5. The TCP connection stays open +in both cases. It carries the end of the stream and the recovery. + +The segment must allow access only by the user that owns the outlet. The +wake-up method and the code for each platform are not part of this page. + +### A clock model with drift + +A fit of offset and drift is a choice of the inlet alone. The outlet answers +the same time probes as today. Thus this feature needs no negotiation, and +it does not use this header. + +## Alternatives that this proposal rejects + +**A higher `Data-Protocol-Version`, for example 120.** liblsl uses the lower +of the two versions (`src/tcp_server.cpp:641`). Thus a liblsl peer falls back +correctly. But the version numbers belong to liblsl. A future liblsl release +can use the same number for a different format, and the two ends then agree +on different things. A version is also one scale, and the features are +independent. + +**A new request line, for example `LSL:labstream-feed`.** A liblsl server +sends no answer to a request line that it does not know +(`src/tcp_server.cpp:502-531`). The inlet then needs a timeout and a second +connection for each liblsl outlet. That adds a delay to each start and to +each recovery. + +**A field in the stream description.** The description goes to every +consumer, and LabRecorder writes it into each XDF file. A field there changes +the stored data, and it does not describe one connection. + +## The conformance plan + +1. **liblsl outlet, raw request.** Send a feed request with the header to the + oracle outlet. Make sure that the answer and the data bytes are the same + as for the request without the header. `oracle/headerprobe.py` does this + now. It passes for liblsl and for labstream `main`. +2. **liblsl inlet.** Make sure that a labstream outlet sends no + `Labstream-Features` line when the request has none. +3. **Released labstream 0.1.2.** Run it as a peer in both directions. It + knows nothing of the header, so each run must give plain LSL. +4. **Two labstream peers.** Run each feature on and off, and each mix of + offer and answer. +5. **Syntax.** Send tokens with uppercase letters, with `;`, and with a bad + form. Make sure that the outlet ignores each bad token, and keeps the + good ones. +6. **The full interop matrix.** `oracle/interop.py` must give the same result + as before, with the header on. + +## Open questions + +- The final name of the header. +- The name of the configuration key from rule 7. +- The API that reports the features of the current connection. +- The exact bytes of the `seq` framing. This belongs to the `seq` section. diff --git a/docs/repository.md b/docs/repository.md index 7a4ffc4..68e1846 100644 --- a/docs/repository.md +++ b/docs/repository.md @@ -291,6 +291,8 @@ release of a new crate therefore goes by hand, with a token. | `docs/releasing.md` | how to release to crates.io and to GitHub, and how to recover | | `docs/versioning.md` | what a version number means, and the rule for each change | | `docs/conformance.md` | what was measured against liblsl, how, and the result | +| `docs/latency.md` | how to measure the latency and the CPU, and the results | +| `docs/extensions.md` | a proposal for features between two labstream peers | | `SPEC.md` | the protocol, with a citation for every claim | | `CONTRIBUTING.md` | the rules for a change, and the style of the documents | | `CHANGELOG.md` | what changed in each version |