diff --git a/CHANGELOG.md b/CHANGELOG.md
index e2c1ac9..f565f2f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,13 @@ and [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
### Added
+- `docs/latency.md` and `conformance/oracle/latency.py`, with
+ `conformance/oracle/latency.c`. They measure the time from a push to a pull,
+ and the CPU of each side, for liblsl and labstream.
+- `docs/extensions.md`, a proposal for a feature header between two labstream
+ peers. Nothing in it is implemented.
+- `conformance/oracle/headerprobe.py`. It makes sure that an outlet ignores
+ the proposed header.
- `docs/repository.md`. It gives the layout of the repository, how the
workspace joins the crates, the daily commands, CI, and the dependency
updates.
diff --git a/conformance/oracle/headerprobe.py b/conformance/oracle/headerprobe.py
new file mode 100755
index 0000000..2f2cbcf
--- /dev/null
+++ b/conformance/oracle/headerprobe.py
@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""Make sure that an outlet ignores the `Labstream-Features` request header.
+
+`docs/extensions.md` proposes this header. The proposal depends on one fact:
+an outlet that does not know the header gives the same answer with it and
+without it. This probe measures that fact on a real outlet.
+
+The probe starts `oracle/publish.c`, linked to one library. It opens two feed
+connections by hand. One request has the extra header, and one does not.
+The answer headers and the two test pattern samples must be the same, byte
+for byte.
+
+Usage:
+ oracle/headerprobe.py # liblsl and the Rust library
+ oracle/headerprobe.py --lib oracle=.build/install/lib
+"""
+
+import argparse
+import os
+import re
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+CHANNELS = 8
+
+# The request that liblsl sends (`src/data_receiver.cpp:170-190`), with fixed
+# values.
+BASE_HEADERS = [
+ "Native-Byte-Order: 1234",
+ "Endian-Performance: 0",
+ "Has-IEEE754-Floats: 1",
+ "Supports-Subnormals: 1",
+ "Value-Size: 4",
+ "Data-Protocol-Version: 110",
+ "Max-Buffer-Length: 360",
+ "Max-Chunk-Length: 0",
+ "Hostname: probe",
+ "Source-Id: ",
+ "Session-Id: default",
+]
+EXTRA = "Labstream-Features: seq/1 shm/1=0a1b2c"
+
+
+def build(lib_dir: Path, out: Path) -> None:
+ subprocess.run(["cc", "-I", str(ROOT / ".build" / "install" / "include"),
+ str(ROOT / "oracle" / "publish.c"), "-o", str(out), "-lm",
+ "-L", str(lib_dir), "-llsl", f"-Wl,-rpath,{lib_dir}"], check=True)
+
+
+def find_stream(lib_dir: Path, name: str) -> tuple:
+ """Resolve the stream with pylsl and give its uid and IPv4 data port."""
+ code = (
+ "import pylsl,sys\n"
+ f"r=pylsl.resolve_byprop('name','{name}',1,10)\n"
+ "print(r[0].as_xml() if r else '')\n"
+ )
+ lib = next(p for p in sorted(lib_dir.glob("liblsl.so*")))
+ env = dict(os.environ, PYLSL_LIB=str(lib))
+ xml = subprocess.run([sys.executable, "-c", code], env=env, capture_output=True,
+ text=True, timeout=30).stdout
+ uid = re.search(r"([^<]*)", xml)
+ port = re.search(r"(\d+)", xml)
+ if not uid or not port:
+ raise RuntimeError("the stream was not found")
+ return uid.group(1), int(port.group(1))
+
+
+def read_exact(sock: socket.socket, n: int) -> bytes:
+ out = b""
+ while len(out) < n:
+ part = sock.recv(n - len(out))
+ if not part:
+ raise RuntimeError(f"the connection closed after {len(out)} bytes")
+ out += part
+ return out
+
+
+def feed(port: int, uid: str, extra: bool) -> tuple:
+ """Send one feed request. Give the answer headers and the test pattern."""
+ lines = [f"LSL:streamfeed/110 {uid}", *BASE_HEADERS]
+ if extra:
+ lines.append(EXTRA)
+ request = "\r\n".join(lines) + "\r\n\r\n"
+ with socket.create_connection(("127.0.0.1", port), timeout=5) as s:
+ s.sendall(request.encode())
+ head = b""
+ while b"\r\n\r\n" not in head:
+ part = s.recv(1)
+ if not part:
+ raise RuntimeError("the connection closed in the headers")
+ head += part
+ # Each test pattern sample: a tag, a timestamp, and the values.
+ # SPEC.md 6.4.
+ pattern = read_exact(s, 2 * (1 + 8 + 4 * CHANNELS))
+ return head.decode(errors="replace"), pattern
+
+
+def probe(name: str, lib_dir: Path, work: Path) -> bool:
+ binary = work / f"publish_{name}"
+ build(lib_dir, binary)
+ stream = f"HeaderProbe{os.getpid()}{name}"
+ pub = subprocess.Popen([str(binary), "--name", stream, "--channels", str(CHANNELS),
+ "--rate", "100", "--seconds", "20"],
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+ try:
+ time.sleep(1.0)
+ uid, port = find_stream(lib_dir, stream)
+ plain_head, plain_pattern = feed(port, uid, extra=False)
+ with_head, with_pattern = feed(port, uid, extra=True)
+ finally:
+ pub.kill()
+ pub.wait()
+ same = plain_head == with_head and plain_pattern == with_pattern
+ leaked = "labstream-features" in with_head.lower()
+ ok = same and not leaked
+ print(f" {name:<8} {'pass' if ok else 'FAIL'} answer headers "
+ f"{'same' if plain_head == with_head else 'DIFFERENT'}, test pattern "
+ f"{'same' if plain_pattern == with_pattern else 'DIFFERENT'}, "
+ f"{'a Labstream-Features answer' if leaked else 'no Labstream-Features answer'}")
+ if not ok:
+ print(" without:", plain_head.replace("\r\n", " | "))
+ print(" with: ", with_head.replace("\r\n", " | "))
+ return ok
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("--lib", action="append", default=None, metavar="NAME=DIR")
+ args = ap.parse_args()
+ libs = args.lib or [f"oracle={ROOT / '.build' / 'install' / 'lib'}",
+ f"rust={ROOT / '.build' / 'rustlib'}"]
+ work = Path(tempfile.mkdtemp(prefix="headerprobe-"))
+ results = []
+ for item in libs:
+ name, _, path = item.partition("=")
+ results.append(probe(name, Path(path).resolve(), work))
+ print()
+ if all(results):
+ print("each outlet ignores the Labstream-Features header")
+ return 0
+ print("an outlet reacts to the Labstream-Features header")
+ return 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/conformance/oracle/latency.c b/conformance/oracle/latency.c
new file mode 100644
index 0000000..bd55098
--- /dev/null
+++ b/conformance/oracle/latency.c
@@ -0,0 +1,253 @@
+/* Measure the time from a push in one process to a pull in another.
+ *
+ * The program runs as an outlet or as an inlet. Link it once against real
+ * liblsl and once against the Rust library. The same program then measures
+ * both libraries, and the two can also measure each other.
+ *
+ * The clock is CLOCK_MONOTONIC of this program, not the clock of the
+ * library. The outlet passes the push time as the timestamp of each sample.
+ * The inlet does no post-processing, so it reads the push time back
+ * unchanged. Both processes run on one machine and read the same clock.
+ *
+ * Channel 0 carries the sample number. The inlet uses it to count lost,
+ * repeated, and reordered samples.
+ *
+ * Each process writes one JSON line to stdout when it ends.
+ * `oracle/latency.py` runs the pairs and makes the table.
+ *
+ * Build:
+ * cc -O2 -I .build/install/include oracle/latency.c -o BIN -lm -ldl \
+ * -L DIR -llsl -Wl,-rpath,DIR
+ */
+
+#define _GNU_SOURCE
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include
+
+static double now(void) {
+ struct timespec t;
+ clock_gettime(CLOCK_MONOTONIC, &t);
+ return (double)t.tv_sec + (double)t.tv_nsec * 1e-9;
+}
+
+static void sleep_until(double t) {
+ struct timespec ts;
+ ts.tv_sec = (time_t)t;
+ ts.tv_nsec = (long)((t - (double)ts.tv_sec) * 1e9);
+ while (clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &ts, NULL) != 0) {
+ }
+}
+
+/* User plus system CPU time of this process, in seconds. */
+static double cpu_seconds(void) {
+ struct rusage r;
+ getrusage(RUSAGE_SELF, &r);
+ return (double)r.ru_utime.tv_sec + (double)r.ru_utime.tv_usec * 1e-6 +
+ (double)r.ru_stime.tv_sec + (double)r.ru_stime.tv_usec * 1e-6;
+}
+
+/* The file of the library that the loader picked. An earlier run used the
+ * wrong library through a stale RUNPATH, so each row names its library. */
+static const char *library_path(void) {
+ Dl_info d;
+ if (dladdr((void *)lsl_library_info, &d) && d.dli_fname) return d.dli_fname;
+ return "unknown";
+}
+
+static int cmp_double(const void *a, const void *b) {
+ double x = *(const double *)a, y = *(const double *)b;
+ return (x > y) - (x < y);
+}
+
+/* The value at fraction q of a sorted array. */
+static double quantile(const double *v, long n, double q) {
+ if (n <= 0) return NAN;
+ long k = (long)(q * (double)(n - 1) + 0.5);
+ return v[k];
+}
+
+static void print_library(void) {
+ printf("\"library\": \"%s\", \"library_info\": \"%s\"", library_path(), lsl_library_info());
+}
+
+static int run_outlet(const char *name, int channels, double rate, int chunk, double seconds,
+ double offset_us) {
+ lsl_streaminfo info =
+ lsl_create_streaminfo(name, "Latency", channels, rate, cft_float32, name);
+ lsl_outlet out = lsl_create_outlet(info, 0, 360);
+ if (!out) {
+ fprintf(stderr, "ERROR could not create the outlet\n");
+ return 1;
+ }
+ fprintf(stderr, "READY\n");
+ if (!lsl_wait_for_consumers(out, 20.0)) {
+ fprintf(stderr, "ERROR no consumer in 20 s\n");
+ return 1;
+ }
+
+ long total = (long)(rate * seconds);
+ long pushes = (total + chunk - 1) / chunk;
+ float *data = calloc((size_t)chunk * (size_t)channels, sizeof(float));
+ double *stamps = calloc((size_t)chunk, sizeof(double));
+ double *lag = calloc((size_t)pushes, sizeof(double));
+
+ /* The inlet starts its reader at the first pull, so the first samples
+ * wait in the outlet for a short time. Give it time to start. */
+ sleep_until(now() + 0.5);
+
+ double cpu0 = cpu_seconds();
+ double t0 = now();
+ long k = 0;
+ for (long p = 0; p < pushes; p++) {
+ long n = total - k < chunk ? total - k : chunk;
+ double due = t0 + (double)(k + n) / rate;
+ sleep_until(due);
+ for (long i = 0; i < n; i++) {
+ data[i * channels] = (float)(k + i);
+ for (int c = 1; c < channels; c++) data[i * channels + c] = (float)c;
+ }
+ double t = now();
+ lag[p] = t - due;
+ for (long i = 0; i < n; i++) stamps[i] = t - offset_us * 1e-6;
+ if (n == 1)
+ lsl_push_sample_ft(out, data, stamps[0]);
+ else
+ lsl_push_chunk_ftn(out, data, (unsigned long)(n * channels), stamps);
+ k += n;
+ }
+ double wall = now() - t0;
+ double cpu = cpu_seconds() - cpu0;
+
+ /* Let the last samples leave before the outlet closes. */
+ sleep_until(now() + 1.0);
+
+ qsort(lag, (size_t)pushes, sizeof(double), cmp_double);
+ printf("{\"role\": \"outlet\", ");
+ print_library();
+ printf(", \"sent\": %ld, \"pushes\": %ld, \"wall_s\": %.3f, \"cpu_pct\": %.3f, "
+ "\"lag_p50_us\": %.1f, \"lag_p99_us\": %.1f, \"lag_max_us\": %.1f}\n",
+ total, pushes, wall, 100.0 * cpu / wall, quantile(lag, pushes, 0.5) * 1e6,
+ quantile(lag, pushes, 0.99) * 1e6, lag[pushes - 1] * 1e6);
+ fflush(stdout);
+ lsl_destroy_outlet(out);
+ return 0;
+}
+
+static int run_inlet(const char *name, int channels, double rate, double seconds,
+ double warmup) {
+ lsl_streaminfo found[1];
+ if (lsl_resolve_byprop(found, 1, "name", name, 1, 20.0) < 1) {
+ fprintf(stderr, "ERROR the stream %s was not found\n", name);
+ return 1;
+ }
+ lsl_inlet in = lsl_create_inlet(found[0], 360, 0, 1);
+ int32_t ec = 0;
+ lsl_open_stream(in, 20.0, &ec);
+ if (ec) {
+ fprintf(stderr, "ERROR open_stream gave %d\n", ec);
+ return 1;
+ }
+ fprintf(stderr, "READY\n");
+
+ long total = (long)(rate * seconds);
+ long skip = (long)(rate * warmup);
+ double *lat = calloc((size_t)total + 1, sizeof(double));
+ unsigned char *seen = calloc((size_t)total + 1, 1);
+ float *buf = calloc((size_t)channels, sizeof(float));
+ long got = 0, kept = 0, repeated = 0, reordered = 0, alien = 0;
+ long last = -1;
+ double cpu0 = 0, t0 = 0;
+
+ for (;;) {
+ double ts = lsl_pull_sample_f(in, buf, channels, 3.0, &ec);
+ double t = now();
+ if (ts == 0.0 || ec) break; /* no sample in 3 s: the outlet is gone */
+ if (got == 0) {
+ cpu0 = cpu_seconds();
+ t0 = t;
+ }
+ got++;
+ long idx = (long)buf[0];
+ if (idx < 0 || idx >= total) {
+ alien++;
+ continue;
+ }
+ if (seen[idx]) {
+ repeated++;
+ continue;
+ }
+ seen[idx] = 1;
+ if (idx < last) reordered++;
+ last = idx;
+ if (idx >= skip) lat[kept++] = t - ts;
+ if (idx == total - 1) break;
+ }
+ double wall = now() - t0;
+ double cpu = cpu_seconds() - cpu0;
+
+ long lost = 0;
+ for (long i = 0; i < total; i++)
+ if (!seen[i]) lost++;
+
+ qsort(lat, (size_t)kept, sizeof(double), cmp_double);
+ printf("{\"role\": \"inlet\", ");
+ print_library();
+ printf(", \"expected\": %ld, \"received\": %ld, \"lost\": %ld, \"repeated\": %ld, "
+ "\"reordered\": %ld, \"alien\": %ld, \"measured\": %ld, \"wall_s\": %.3f, "
+ "\"cpu_pct\": %.3f, \"p50_us\": %.1f, \"p90_us\": %.1f, \"p99_us\": %.1f, "
+ "\"p999_us\": %.1f, \"max_us\": %.1f, \"min_us\": %.1f}\n",
+ total, got, lost, repeated, reordered, alien, kept, wall,
+ wall > 0 ? 100.0 * cpu / wall : 0.0, quantile(lat, kept, 0.5) * 1e6,
+ quantile(lat, kept, 0.9) * 1e6, quantile(lat, kept, 0.99) * 1e6,
+ quantile(lat, kept, 0.999) * 1e6, kept ? lat[kept - 1] * 1e6 : NAN,
+ kept ? lat[0] * 1e6 : NAN);
+ fflush(stdout);
+ lsl_destroy_inlet(in);
+ return 0;
+}
+
+int main(int argc, char **argv) {
+ const char *role = NULL;
+ const char *name = "LatencyBench";
+ int channels = 8;
+ double rate = 512.0;
+ int chunk = 1;
+ double seconds = 10.0;
+ double warmup = 1.0;
+ double offset_us = 0.0;
+ for (int i = 1; i < argc; i++) {
+ if (!strcmp(argv[i], "outlet") || !strcmp(argv[i], "inlet")) {
+ role = argv[i];
+ continue;
+ }
+ if (i + 1 >= argc) {
+ fprintf(stderr, "the option %s has no value\n", argv[i]);
+ return 2;
+ }
+ if (!strcmp(argv[i], "--name")) name = argv[++i];
+ else if (!strcmp(argv[i], "--channels")) channels = atoi(argv[++i]);
+ else if (!strcmp(argv[i], "--rate")) rate = atof(argv[++i]);
+ else if (!strcmp(argv[i], "--chunk")) chunk = atoi(argv[++i]);
+ else if (!strcmp(argv[i], "--seconds")) seconds = atof(argv[++i]);
+ else if (!strcmp(argv[i], "--warmup")) warmup = atof(argv[++i]);
+ else if (!strcmp(argv[i], "--offset-us")) offset_us = atof(argv[++i]);
+ else {
+ fprintf(stderr, "unknown option %s\n", argv[i]);
+ return 2;
+ }
+ }
+ if (!role || channels < 1 || rate <= 0 || chunk < 1 || seconds <= 0) {
+ fprintf(stderr, "usage: latency outlet|inlet [--name N] [--channels C] [--rate R] "
+ "[--chunk K] [--seconds S] [--warmup S] [--offset-us U]\n");
+ return 2;
+ }
+ if (!strcmp(role, "outlet")) return run_outlet(name, channels, rate, chunk, seconds, offset_us);
+ return run_inlet(name, channels, rate, seconds, warmup);
+}
diff --git a/conformance/oracle/latency.py b/conformance/oracle/latency.py
new file mode 100755
index 0000000..48d4897
--- /dev/null
+++ b/conformance/oracle/latency.py
@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+"""Measure the push-to-pull latency of each library, and of each mixed pair.
+
+The script builds `oracle/latency.c` once for each library, runs an outlet
+process and an inlet process for each cell, and writes one table. Each cell
+runs more than once, because one run of a timing test proves little.
+
+The control row is liblsl to liblsl. With `--control`, the script also runs
+that pair with a known delay in the timestamps, and the table must show the
+same delay. If it does not, the instrument is wrong, not the library.
+
+Usage:
+ oracle/latency.py
+ oracle/latency.py --lib oracle=.build/install/lib --lib main=/path/to/dir
+ oracle/latency.py --loads 512x8x1,4000x64x32 --seconds 10 --repeat 3
+ oracle/latency.py --control
+
+A load `RATExCHANNELSxCHUNK` pushes CHUNK samples in each push call. The
+first `--lib` is the reference. The script runs each library against itself,
+and the reference against each other library, in both directions.
+"""
+
+import argparse
+import json
+import os
+import statistics
+import subprocess
+import sys
+import tempfile
+import threading
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+
+DEFAULT_LOADS = "512x8x1,512x8x16,4000x64x1,4000x64x32"
+
+
+def build(source: Path, lib_dir: Path, out: Path) -> None:
+ cmd = [
+ "cc", "-O2", "-I", str(ROOT / ".build" / "install" / "include"), str(source),
+ "-o", str(out), "-lm", "-ldl", "-L", str(lib_dir), "-llsl",
+ f"-Wl,-rpath,{lib_dir}",
+ ]
+ subprocess.run(cmd, check=True)
+
+
+def wait_ready(proc: subprocess.Popen, what: str, lines: list) -> None:
+ """Read stderr until the READY line. Keep the rest for an error report."""
+ for raw in proc.stderr:
+ line = raw.decode(errors="replace").rstrip()
+ lines.append(line)
+ if line == "READY":
+ # Drain the rest in the background, so the pipe never fills.
+ threading.Thread(target=lambda: [lines.append(r.decode(errors="replace").rstrip())
+ for r in proc.stderr], daemon=True).start()
+ return
+ raise RuntimeError(f"{what}: the peer stopped before READY: {lines[-5:]}")
+
+
+def last_json(text: str) -> dict:
+ for line in reversed(text.splitlines()):
+ line = line.strip()
+ if line.startswith("{"):
+ return json.loads(line)
+ raise RuntimeError("no JSON line in the output")
+
+
+def run_pair(out_bin: Path, in_bin: Path, name: str, rate: float, channels: int, chunk: int,
+ seconds: float, offset_us: float) -> tuple:
+ common = ["--name", name, "--rate", str(rate), "--channels", str(channels),
+ "--seconds", str(seconds)]
+ out_err, in_err = [], []
+ outlet = subprocess.Popen([str(out_bin), "outlet", *common, "--chunk", str(chunk),
+ "--offset-us", str(offset_us)],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ try:
+ wait_ready(outlet, "outlet", out_err)
+ inlet = subprocess.Popen([str(in_bin), "inlet", *common],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ try:
+ wait_ready(inlet, "inlet", in_err)
+ in_out = inlet.stdout.read().decode()
+ inlet.wait(timeout=seconds + 30)
+ finally:
+ if inlet.poll() is None:
+ inlet.kill()
+ out_out = outlet.stdout.read().decode()
+ outlet.wait(timeout=30)
+ finally:
+ if outlet.poll() is None:
+ outlet.kill()
+ return last_json(out_out), last_json(in_out)
+
+
+def spread(values: list) -> str:
+ if len(values) == 1:
+ return f"{values[0]:.0f}"
+ return f"{statistics.median(values):.0f} ({min(values):.0f}-{max(values):.0f})"
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("--lib", action="append", default=None, metavar="NAME=DIR",
+ help="a library directory that holds liblsl.so (repeat it)")
+ ap.add_argument("--loads", default=DEFAULT_LOADS)
+ ap.add_argument("--seconds", type=float, default=10.0)
+ ap.add_argument("--repeat", type=int, default=3)
+ ap.add_argument("--no-mixed", action="store_true",
+ help="run each library only against itself")
+ ap.add_argument("--control", action="store_true",
+ help="also measure a known delay of 1000 us with the reference pair")
+ ap.add_argument("--json", default=None, help="write every run to this file")
+ args = ap.parse_args()
+
+ libs = args.lib or [f"oracle={ROOT / '.build' / 'install' / 'lib'}",
+ f"rust={ROOT / '.build' / 'rustlib'}"]
+ named = []
+ for item in libs:
+ name, _, path = item.partition("=")
+ named.append((name, Path(path).resolve()))
+
+ work = Path(tempfile.mkdtemp(prefix="latency-"))
+ bins = {}
+ for name, path in named:
+ bins[name] = work / f"latency_{name}"
+ build(ROOT / "oracle" / "latency.c", path, bins[name])
+
+ ref = named[0][0]
+ pairs = [(n, n) for n, _ in named]
+ if not args.no_mixed:
+ for n, _ in named[1:]:
+ pairs += [(ref, n), (n, ref)]
+
+ loads = []
+ for spec in args.loads.split(","):
+ rate, channels, chunk = spec.split("x")
+ loads.append((float(rate), int(channels), int(chunk)))
+
+ runs = []
+ counter = 0
+
+ def one(out_name, in_name, rate, channels, chunk, offset_us=0.0):
+ nonlocal counter
+ counter += 1
+ stream = f"Lat{os.getpid()}_{counter}"
+ o, i = run_pair(bins[out_name], bins[in_name], stream, rate, channels, chunk,
+ args.seconds, offset_us)
+ row = {"outlet": out_name, "inlet": in_name, "rate": rate, "channels": channels,
+ "chunk": chunk, "offset_us": offset_us, "outlet_run": o, "inlet_run": i}
+ runs.append(row)
+ print(f" {out_name:>10} -> {in_name:<10} {rate:g} Hz x {channels} ch, chunk {chunk}: "
+ f"p50 {i['p50_us']:.0f} us, p99 {i['p99_us']:.0f} us, lost {i['lost']}",
+ file=sys.stderr)
+ return row
+
+ for rate, channels, chunk in loads:
+ for out_name, in_name in pairs:
+ for _ in range(args.repeat):
+ one(out_name, in_name, rate, channels, chunk)
+
+ control_ok = None
+ if args.control:
+ base = [one(ref, ref, 512.0, 8, 1)["inlet_run"]["p50_us"] for _ in range(args.repeat)]
+ late = [one(ref, ref, 512.0, 8, 1, 1000.0)["inlet_run"]["p50_us"]
+ for _ in range(args.repeat)]
+ shift = statistics.median(late) - statistics.median(base)
+ control_ok = abs(shift - 1000.0) < 100.0
+ print(f"\ncontrol: a delay of 1000 us shows as {shift:.0f} us "
+ f"({'pass' if control_ok else 'FAIL'})")
+
+ print("\n| outlet -> inlet | load | p50 us | p99 us | p99.9 us | max us | lost | "
+ "CPU outlet % | CPU inlet % |")
+ print("|---|---|---|---|---|---|---|---|---|")
+ for rate, channels, chunk in loads:
+ for out_name, in_name in pairs:
+ cell = [r for r in runs if r["outlet"] == out_name and r["inlet"] == in_name
+ and r["rate"] == rate and r["channels"] == channels
+ and r["chunk"] == chunk and r["offset_us"] == 0.0]
+ if not cell:
+ continue
+ ins = [r["inlet_run"] for r in cell]
+ outs = [r["outlet_run"] for r in cell]
+ print(f"| {out_name} -> {in_name} | {rate:g} Hz x {channels}, chunk {chunk} | "
+ f"{spread([x['p50_us'] for x in ins])} | {spread([x['p99_us'] for x in ins])} | "
+ f"{spread([x['p999_us'] for x in ins])} | {max(x['max_us'] for x in ins):.0f} | "
+ f"{sum(x['lost'] + x['repeated'] + x['alien'] for x in ins)} | "
+ f"{statistics.median(x['cpu_pct'] for x in outs):.1f} | "
+ f"{statistics.median(x['cpu_pct'] for x in ins):.1f} |")
+ print(f"\nEach cell: median of {args.repeat} runs, with the range in brackets. "
+ f"Each run: {args.seconds:g} s. The first second is not measured.")
+ print("Libraries:")
+ for name, path in named:
+ seen = {r["inlet_run"]["library"] for r in runs if r["inlet"] == name}
+ print(f" {name}: {path} (loaded: {', '.join(sorted(seen))})")
+
+ if args.json:
+ Path(args.json).write_text(json.dumps(runs, indent=1))
+ if control_ok is False:
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/docs/extensions.md b/docs/extensions.md
new file mode 100644
index 0000000..24f83d3
--- /dev/null
+++ b/docs/extensions.md
@@ -0,0 +1,202 @@
+# Proposal: feature negotiation between two labstream peers
+
+**Status:** proposal. Nothing on this page is implemented. The wire format does
+not change until a feature section below is accepted and SPEC.md gets a
+section for it.
+
+## The purpose
+
+Some improvements need a change to the data channel. Examples are sequence
+numbers that make sample loss visible, and a shared-memory transport for two
+processes on one machine. liblsl does not know these changes.
+
+This page defines how a labstream inlet and a labstream outlet agree to use
+such a feature. The agreement must leave every other pair unchanged:
+
+| Outlet | Inlet | Result |
+|---|---|---|
+| liblsl | liblsl | plain LSL, as today |
+| liblsl | labstream | plain LSL |
+| labstream | liblsl | plain LSL |
+| labstream 0.1.2 or earlier | labstream | plain LSL |
+| labstream | labstream | plain LSL, plus each feature that both ends accept |
+
+## What liblsl does with an extra header
+
+These facts come from `conformance/liblsl/` at `e651023c`.
+
+1. The feed server reads each request header line, and acts only on the keys
+ that it knows. An unknown key has no effect
+ (`src/tcp_server.cpp:605-633`).
+2. The feed client reads each answer header line the same way, and acts only
+ on the keys that it knows (`src/data_receiver.cpp:215-253`).
+3. Both sides cut a header line at the first `;`, and convert the line to
+ lowercase before they compare the key (`src/tcp_server.cpp:610-613`,
+ `src/data_receiver.cpp:220-223`).
+4. Both sides read a header line into a buffer of 16384 bytes
+ (`src/tcp_server.cpp:604`, `src/data_receiver.cpp:193`).
+5. The labstream parser cuts and converts the same way
+ (`crates/labstream-proto/src/header.rs:103-107`).
+
+As a result, a liblsl outlet ignores an extra request header, and a liblsl
+inlet ignores an extra answer header. Item 3 limits the syntax of the header.
+A value must not contain `;`, and a value must not depend on letter case.
+
+## The header
+
+### The request
+
+A labstream inlet adds one header line to its feed request:
+
+```
+Labstream-Features: seq/1 shm/1
+```
+
+The value is a list of tokens with one space between two tokens. Each token
+has this form:
+
+```
+token = name "/" version [ "=" param ]
+name = 1*( a-z / 0-9 / "-" )
+version = 1*( 0-9 )
+param = 1*( 0-9 / a-f )
+```
+
+A parameter is hexadecimal, because the header line loses its letter case.
+The whole line must stay below 16384 bytes. An inlet can offer more than one
+version of a feature, for example `seq/1 seq/2`.
+
+### The answer
+
+A labstream outlet that reads the request header adds one line to its answer:
+
+```
+Labstream-Features: seq/1
+```
+
+The answer holds only tokens from the request. For each name, it holds at
+most one version.
+
+### The rules
+
+1. An inlet sends the request header only when it supports one or more
+ features.
+2. An outlet sends the answer header only when the request held one.
+3. An outlet ignores a token that it does not know, or that does not match
+ the syntax.
+4. An inlet ignores an answer token that it did not offer.
+5. A feature is on only when the inlet offered it and the outlet answered it.
+6. If the answer has no `Labstream-Features` line, no feature is on. A liblsl
+ outlet and an older labstream outlet give this result.
+7. A configuration key can remove a feature from the offer and from the
+ answer. The name of the key is an open question.
+
+## Where the framing changes
+
+After the answer headers, the outlet sends two test pattern samples
+(SPEC.md 6). These two samples always use the plain framing of SPEC.md 7.
+The inlet compares them exactly as today.
+
+A feature can change the framing only from the first byte after the second
+test pattern sample. Thus the test pattern stays a check of the codec, and a
+fault in a feature cannot hide a fault in the codec.
+
+## The scope of an agreement
+
+An agreement holds for one connection. A recovery opens a new connection with
+a new request, so the set of features can change. An outlet that restarts as
+a liblsl program gives an example.
+
+The inlet must report the features of the current connection, and must show
+when the set changes. This works the same way as the clock reset after a
+recovery. The API names are an open question.
+
+## The first features
+
+Each feature below gets its own section and its own conformance checks
+before any code. The text here gives only the requirements that affect the
+negotiation.
+
+### `seq`: sequence numbers
+
+The outlet gives each sample a number when the application pushes it. One
+counter serves the whole outlet, not one counter for each connection.
+
+This rule is necessary. The two real causes of loss are a full consumer
+queue (SPEC.md 8.7) and the gap between a lost connection and the next one.
+A counter for each connection starts again after a recovery, so it shows
+neither cause.
+
+The inlet reports the number of samples that it did not receive. The counter
+also shows a repeated sample and a sample out of order.
+
+### `shm`: shared memory on one machine
+
+The hostname does not prove that the two processes share a machine. Two
+machines can have the same name, and a container can have the name of its
+host. Thus `shm` needs one more step after the headers:
+
+1. The inlet offers `shm/1`.
+2. The outlet makes a memory segment. It writes a random value into the
+ segment. It answers `shm/1=`.
+3. The inlet opens the segment and reads the random value.
+4. The inlet writes one line on the TCP connection: `shm-ok ` or
+ `shm-no`.
+5. If the value matches, the outlet sends the samples through the segment.
+ Otherwise, or after 1 s with no line, the outlet sends the samples on TCP.
+
+The outlet does not send data before step 5. The TCP connection stays open
+in both cases. It carries the end of the stream and the recovery.
+
+The segment must allow access only by the user that owns the outlet. The
+wake-up method and the code for each platform are not part of this page.
+
+### A clock model with drift
+
+A fit of offset and drift is a choice of the inlet alone. The outlet answers
+the same time probes as today. Thus this feature needs no negotiation, and
+it does not use this header.
+
+## Alternatives that this proposal rejects
+
+**A higher `Data-Protocol-Version`, for example 120.** liblsl uses the lower
+of the two versions (`src/tcp_server.cpp:641`). Thus a liblsl peer falls back
+correctly. But the version numbers belong to liblsl. A future liblsl release
+can use the same number for a different format, and the two ends then agree
+on different things. A version is also one scale, and the features are
+independent.
+
+**A new request line, for example `LSL:labstream-feed`.** A liblsl server
+sends no answer to a request line that it does not know
+(`src/tcp_server.cpp:502-531`). The inlet then needs a timeout and a second
+connection for each liblsl outlet. That adds a delay to each start and to
+each recovery.
+
+**A field in the stream description.** The description goes to every
+consumer, and LabRecorder writes it into each XDF file. A field there changes
+the stored data, and it does not describe one connection.
+
+## The conformance plan
+
+1. **liblsl outlet, raw request.** Send a feed request with the header to the
+ oracle outlet. Make sure that the answer and the data bytes are the same
+ as for the request without the header. `oracle/headerprobe.py` does this
+ now. It passes for liblsl and for labstream `main`.
+2. **liblsl inlet.** Make sure that a labstream outlet sends no
+ `Labstream-Features` line when the request has none.
+3. **Released labstream 0.1.2.** Run it as a peer in both directions. It
+ knows nothing of the header, so each run must give plain LSL.
+4. **Two labstream peers.** Run each feature on and off, and each mix of
+ offer and answer.
+5. **Syntax.** Send tokens with uppercase letters, with `;`, and with a bad
+ form. Make sure that the outlet ignores each bad token, and keeps the
+ good ones.
+6. **The full interop matrix.** `oracle/interop.py` must give the same result
+ as before, with the header on.
+
+## Open questions
+
+- The final name of the header.
+- The name of the configuration key from rule 7.
+- The API that reports the features of the current connection.
+- The exact bytes of the `seq` framing. This belongs to the `seq` section.
diff --git a/docs/latency.md b/docs/latency.md
new file mode 100644
index 0000000..5a5e33e
--- /dev/null
+++ b/docs/latency.md
@@ -0,0 +1,119 @@
+# Latency and CPU
+
+This page gives how to measure the time from a push to a pull, and the
+results of the first measurement. Run the measurement again before you claim
+that a change makes the library faster.
+
+## The instrument
+
+`conformance/oracle/latency.c` is one C program for the C ABI. It runs as an
+outlet or as an inlet. The runner `conformance/oracle/latency.py` links it
+once to each library, then runs an outlet process and an inlet process for
+each cell.
+
+- **The clock.** The program reads its own `CLOCK_MONOTONIC`, not the clock
+ of the library. The outlet gives the push time as the timestamp of each
+ sample. The inlet does no post-processing, and records the pull time minus
+ the timestamp.
+- **The chunk.** All samples of one push call get the push time. The latency
+ of a chunk is thus the latency of its last sample to leave.
+- **The pull.** The inlet calls `lsl_pull_sample_f` with a timeout of 3 s. The
+ call returns when a sample arrives, so no poll interval adds latency.
+- **Loss.** Channel 0 carries the sample number. The inlet counts lost,
+ repeated, and reordered samples.
+- **The library.** Each process reports the file of the library that the
+ loader picked. A stale `RUNPATH` once made a test use the wrong library.
+- **CPU.** Each process reports its user and system time, as a percentage of
+ one core, over the time that it sends or receives.
+- **The schedule.** The outlet waits on an absolute schedule, and reports how
+ late each push is. This lag is not part of the latency.
+
+## The control
+
+The control pair is liblsl to liblsl. With `--control`, the runner also runs
+that pair with timestamps that are 1000 µs early. The table must show a delay
+that is 1000 µs larger. On the run below, the delay showed as 966 µs.
+
+## How to run it
+
+Run these commands from `conformance/`. The oracle build must exist
+(`conformance/oracle/README.md`).
+
+```bash
+oracle/latency.py --control
+```
+
+To compare branches, build the C library of each branch in its own worktree
+and target directory. Then give each directory to the runner:
+
+```bash
+oracle/latency.py --lib oracle=.build/install/lib --lib main=DIR1 --lib fix=DIR2 --no-mixed --control
+```
+
+Do not copy a branch build into `.build/rustlib`. Other scripts read that
+directory.
+
+## The first results
+
+Machine: AMD Ryzen 5 PRO 3500U, 8 threads, Linux 6.16, loopback, no other
+load on purpose. Each cell is the median of 3 runs of 10 s, with the range of
+the 3 runs in brackets. The first second of each run is not measured.
+
+Libraries:
+
+- `oracle`: liblsl at `e651023c`.
+- `main`: labstream `a912cef`.
+- `fix`: branch `fix/network-robustness` at `e01e998` (pull request #9).
+- `quick`: branch `perf/quick-wins` at `d6880e4` (pull request #10).
+
+| outlet -> inlet | load | p50 µs | p99 µs | p99.9 µs | max µs | lost | CPU outlet % | CPU inlet % |
+|---|---|---|---|---|---|---|---|---|
+| oracle -> oracle | 512 Hz x 8, chunk 1 | 102 (93-142) | 327 (201-501) | 419 (231-624) | 2150 | 0 | 5.7 | 3.4 |
+| main -> main | 512 Hz x 8, chunk 1 | 96 (94-96) | 212 (204-217) | 465 (260-1808) | 2932 | 0 | 4.2 | 3.2 |
+| fix -> fix | 512 Hz x 8, chunk 1 | 96 (94-97) | 204 (200-235) | 254 (246-1011) | 3218 | 0 | 4.2 | 3.2 |
+| quick -> quick | 512 Hz x 8, chunk 1 | 94 (92-98) | 204 (194-239) | 419 (308-557) | 2382 | 0 | 4.1 | 3.2 |
+| oracle -> oracle | 512 Hz x 8, chunk 16 | 246 (224-247) | 646 (599-648) | 742 (729-779) | 780 | 0 | 0.7 | 0.4 |
+| main -> main | 512 Hz x 8, chunk 16 | 290 (287-297) | 667 (614-1054) | 1025 (703-2150) | 2151 | 0 | 2.0 | 1.5 |
+| fix -> fix | 512 Hz x 8, chunk 16 | 220 (218-238) | 514 (464-514) | 617 (480-624) | 626 | 0 | 1.1 | 0.5 |
+| quick -> quick | 512 Hz x 8, chunk 16 | 220 (214-236) | 531 (495-536) | 696 (676-754) | 756 | 0 | 1.1 | 0.5 |
+| oracle -> oracle | 4000 Hz x 64, chunk 1 | 71 (70-72) | 116 (114-117) | 152 (152-164) | 2803 | 0 | 30.4 | 18.4 |
+| main -> main | 4000 Hz x 64, chunk 1 | 73 (73-75) | 116 (114-119) | 154 (149-172) | 755 | 0 | 21.2 | 18.4 |
+| fix -> fix | 4000 Hz x 64, chunk 1 | 75 (74-76) | 121 (116-165) | 168 (159-2487) | 5199 | 0 | 21.5 | 18.2 |
+| quick -> quick | 4000 Hz x 64, chunk 1 | 73 (71-74) | 125 (105-145) | 224 (157-1681) | 3985 | 0 | 20.5 | 17.5 |
+| oracle -> oracle | 4000 Hz x 64, chunk 32 | 318 (317-320) | 651 (646-667) | 768 (764-844) | 876 | 0 | 3.3 | 1.7 |
+| main -> main | 4000 Hz x 64, chunk 32 | 382 (381-406) | 795 (756-852) | 1108 (892-1309) | 2916 | 0 | 9.3 | 10.8 |
+| fix -> fix | 4000 Hz x 64, chunk 32 | 343 (310-355) | 664 (626-699) | 810 (796-1063) | 3097 | 0 | 3.1 | 2.2 |
+| quick -> quick | 4000 Hz x 64, chunk 32 | 328 (322-339) | 658 (657-671) | 829 (797-832) | 892 | 0 | 3.2 | 2.2 |
+
+### What the results show
+
+- **A chunk push on `main` costs 3 to 5 times the CPU of liblsl.** At
+ 4000 Hz x 64 channels in chunks of 32, `main` uses 9.3 % and 10.8 %. The
+ `fix` branch uses 3.1 % and 2.2 %, near liblsl. The `fix` branch writes each chunk
+ with one write, and locks the inlet queue once for each read.
+- **A chunk push on `main` is also slower.** The median is 382 µs on `main`,
+ 343 µs on `fix`, and 318 µs on liblsl.
+- **A push of one sample gives the same latency in all four builds.** The
+ median is 70 to 100 µs. At 4000 Hz x 64 channels, the labstream outlet uses
+ about 21 % of one core, and the liblsl outlet uses 30 %.
+- **No build lost a sample.**
+- **The maximum is noise.** A single value of 2 to 5 ms comes from the
+ scheduler of the operating system. The p99.9 column is a better measure of
+ the tail.
+- **The `quick` branch does not change the latency of the data path.** It
+ changes the start of a connection and the end of a dead connection. This
+ benchmark measures neither.
+
+## What this benchmark does not measure
+
+- **Two machines.** All results come from the loopback interface.
+- **A slow or lossy network.** Do not add a `tc netem` rule to the `lo`
+ interface of a shared machine. The rule delays every program on the machine.
+ Use a network namespace instead. Make two namespaces with a `veth` pair,
+ add the `netem` rule inside one of them, and run the outlet and the inlet in
+ different namespaces. Discovery then needs the namespace addresses in
+ `lab.KnownPeers` of the configuration. This procedure was not run yet.
+- **The application.** Phasic and other programs pull in their own loop. Their
+ poll interval adds to these numbers.
+- **The first offset and the recovery.** The live tests in
+ `crates/labstream-net/tests/` measure these.
diff --git a/docs/repository.md b/docs/repository.md
index 7a4ffc4..68e1846 100644
--- a/docs/repository.md
+++ b/docs/repository.md
@@ -291,6 +291,8 @@ release of a new crate therefore goes by hand, with a token.
| `docs/releasing.md` | how to release to crates.io and to GitHub, and how to recover |
| `docs/versioning.md` | what a version number means, and the rule for each change |
| `docs/conformance.md` | what was measured against liblsl, how, and the result |
+| `docs/latency.md` | how to measure the latency and the CPU, and the results |
+| `docs/extensions.md` | a proposal for features between two labstream peers |
| `SPEC.md` | the protocol, with a citation for every claim |
| `CONTRIBUTING.md` | the rules for a change, and the style of the documents |
| `CHANGELOG.md` | what changed in each version |