From 81ced747a4ef5a66fe17f31373ad09979f823395 Mon Sep 17 00:00:00 2001 From: Jonathan Stokes Date: Mon, 14 Sep 2026 15:22:02 -0500 Subject: [PATCH 1/4] fix: the burst recorder waits for the whole tail, not its first message Two hardware tests failed together on 2026-09-11 and passed on a re-run of the same commit: test_the_connect_burst_warms_the_cache and test_the_burst_delivered_every_entry_the_preset_surface_reads. The connect burst closes with four state messages - RecallPreset, SetlistPosition, PresetDirty, Scene - and they are a GROUP, not a sequence. Re-measured 2026-09-14 on 4.0.1 / d14e over three connections: they land at about 11.1 s, spread over 6.0, 3.6 and 5.8 ms, in that order every time. RecallPreset is the FIRST of them. The recorder stopped on RecallPreset alone, believing it was the tail, and polled for it at 100 ms. Its stop therefore landed at a uniformly random point in the 100 ms after RecallPreset arrived, and about 6 ms of that window is before the other three have landed. When it fell there the recorder came off the transport and the fixture took its burst_warmed snapshot of a cache that had not heard PresetDirty or Scene yet. The cache stays on the transport all run and got them milliseconds later, which is why only the two tests reading the snapshot failed and every test reading the live cache passed. Measured offline against the real recorder with the poll phase randomised the way the unit randomises it: 300 runs lost PresetDirty 20 times and Scene 30 times, 6.7% and 10.0%. After the change, 0 of 300 for both. The stop condition is now the postcondition rather than a proxy for it: wait until every message in BURST_TAIL has arrived. No timeout was widened - the patience is unchanged at 30 s, and a burst that genuinely does not finish now reports which messages never came, so the next failure of this kind says whether the unit stopped sending or the recorder stopped listening. Co-Authored-By: Claude Opus 5 --- docs/protocol.md | 10 ++ tests/hardware/conftest.py | 96 +++++++++++++---- tests/hardware/test_broadcast_listener.py | 5 +- tests/hardware/test_model_state.py | 15 ++- tests/hardware/test_preset_surface.py | 14 ++- tests/test_handshake_burst_recorder.py | 123 ++++++++++++++++++++-- 6 files changed, 223 insertions(+), 40 deletions(-) diff --git a/docs/protocol.md b/docs/protocol.md index 778ee71..1b5d6b1 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -2095,6 +2095,16 @@ the order `RecallPreset`, `SetlistPosition`, `PresetDirty`, `Scene` - and with n pushes, because nothing changed. The seed `RecallPreset` sets `action`, `preset` and `reason`. +Re-measured 2026-09-14 on 4.0.1 / d14e over three consecutive connections: the four landed +at 11.21 s, 11.08 s and 11.07 s, spread over 6.0 ms, 3.6 ms and 5.8 ms, in that same order +every time. **`RecallPreset` is the FIRST of the four, not the last**, which matters to +anything that treats it as the signal the burst is over: the other three are still a few +milliseconds out. The hardware suite's burst recorder did exactly that and lost +`PresetDirty` in roughly one run in fifteen - it polls at 100 ms, so its stop landed at a +uniformly random point inside the 100 ms after `RecallPreset`, and about 6 ms of that +window is before the group has finished arriving. It now waits for all four +(`BURST_TAIL` in `tests/hardware/conftest.py`). Wait for the group, not for its head. + **`read_preset()` RECALLS the slot it reads** - that was already documented - and the recall **resets the active scene to the preset's default, discards unsaved edits, and interrupts the audio**. diff --git a/tests/hardware/conftest.py b/tests/hardware/conftest.py index 3ab6279..2a5b576 100644 --- a/tests/hardware/conftest.py +++ b/tests/hardware/conftest.py @@ -212,6 +212,19 @@ class HandshakeBurst: Runs on the RX thread, so it does the least it can: append and return. """ + #: The four state messages that CLOSE the connect burst, in the order the + #: unit sends them. The recording is complete when all four have arrived. + #: + #: They are one group, not a sequence with a last member to watch for: + #: re-measured 2026-09-14 on d14e over three sessions, they land at about + #: 11.1 s inside 3.6, 5.8 and 6.0 ms respectively, always in this order + #: (``docs/protocol.md``, "Connect burst, measured"). ``RecallPreset`` is + #: the FIRST of them. Waiting for that one alone is what this used to do, + #: and it cost two hardware tests roughly one run in fifteen - see + #: ``tests/test_handshake_burst_recorder.py``. + BURST_TAIL = ("RecallPresetMessage", "SetlistPositionMessage", + "PresetDirtyMessage", "SceneMessage") + def __init__(self): self._lock = threading.Lock() self._names = [] @@ -221,6 +234,8 @@ def __init__(self): #: handshake. Shapes only - the values are not this recorder's business. self._versions = [] self._detach = None + self._sentinels = frozenset() + self._patience = None self.closed = False self.settled_in = None # seconds the burst took, or None if it timed out @@ -244,26 +259,73 @@ def versions(self): with self._lock: return list(self._versions) - def record_until(self, sentinel, patience): - """Record until a ``sentinel``-typed message arrives, then stop. - - The seed ``RecallPresetMessage`` is the tail of the burst - measured - 2026-08-12 on d14e: ModelRepo at 4.9 s, the folder listings and settings - at 5.1 s, the current preset at 10.1 s - so waiting for it means the whole - burst has been recorded, however long the unit takes about it. - - Stops on ``patience`` seconds regardless, so a unit that never sends it - cannot hang the run. ``settled_in`` says which of the two happened. + def record_until(self, sentinels, patience): + """Record until EVERY type in ``sentinels`` has arrived, then stop. + + :data:`BURST_TAIL` is what the fixture passes. All four of them, not the + first: the burst's shape is ModelRepo at 4.9 s, the folder listings and + settings at 5.1 s, then those four together at about 11 s, and they are a + group six milliseconds wide rather than a sequence. This loop polls at + 100 ms, so a stop condition naming only the first of the group stops at + a uniformly random point in the 100 ms after it - inside the group about + one time in fifteen. When that happened the recorder came off the + transport and the fixture snapshotted ``burst_warmed`` before the other + three reached the cache, and the two tests reading that snapshot failed + together while every test reading the live cache passed. + + Stops on ``patience`` seconds regardless, so a unit that never sends one + cannot hang the run. ``settled_in`` says which of the two happened, and + :meth:`unfinished` says it in words for a failure message. """ + if isinstance(sentinels, str): + raise TypeError( + "record_until takes a collection of type names, not one name - " + "the burst is over when all of them have arrived, and a single " + "name is the bug this signature replaced. A str is a collection " + "of letters, so this would otherwise wait for message types " + "called 'R', 'e' and 'c' and never settle.") + self._sentinels = frozenset(sentinels) + self._patience = patience started = time.monotonic() deadline = started + patience while time.monotonic() < deadline: - if self._recorded(sentinel): + if not self.missing(): self.settled_in = time.monotonic() - started break time.sleep(0.1) self.close() + def missing(self): + """The sentinel types :meth:`record_until` has not seen yet. + + Builds a set of the recording each time rather than scanning it once per + sentinel: the recording runs to several hundred names by the time the + burst closes, and this is polled ten times a second at the busiest + moment the link has. + """ + with self._lock: + return self._sentinels - set(self._names) + + def unfinished(self): + """Why the recording is short of the whole burst, or ``None`` if it is not. + + A test asserting on the recording puts this in front of its own + assertions, because without it the two failures read identically from + the recording alone: the unit sent no ``PresetDirty`` at all, which is a + finding about the unit, and the recorder stopped before it arrived, + which is a bug in here. + """ + if self.settled_in is not None: + return None + if not self._sentinels: + return "the burst was never recorded: record_until was not called" + return ( + f"the connect burst did not finish within {self._patience}s - " + f"{', '.join(sorted(self.missing()))} never arrived, of the " + f"{len(self._sentinels)} messages that close it. Everything below " + f"is a recording of a burst that was cut off, so read it as that " + f"rather than as a finding about the unit or the cache.") + def close(self): """Stop recording and come off the transport. Idempotent. @@ -279,16 +341,6 @@ def close(self): if not already and self._detach is not None: self._detach() - def _recorded(self, name): - """Whether a message of type ``name`` has been recorded. - - Scans in place rather than going through :meth:`names`, which would copy - the whole recording on every poll, briefly contending with the RX thread - at the busiest moment it has. - """ - with self._lock: - return name in self._names - def names(self): """A snapshot of what has been recorded, in arrival order.""" with self._lock: @@ -376,7 +428,7 @@ def subscribe(transport): cache.bind(client) # Read by pytest_terminal_summary, which has a config and no fixtures. request.config._profile = type(client) - burst.record_until("RecallPresetMessage", patience=30.0) + burst.record_until(HandshakeBurst.BURST_TAIL, patience=30.0) # Taken here, before any test can read through the cache, so "the burst # warmed this" cannot later be confused with "some test read it". warmed = {entry.name: cache.cached(entry.name) for entry in entries.ENTRIES} diff --git a/tests/hardware/test_broadcast_listener.py b/tests/hardware/test_broadcast_listener.py index c6ec342..a87fe0f 100644 --- a/tests/hardware/test_broadcast_listener.py +++ b/tests/hardware/test_broadcast_listener.py @@ -82,9 +82,8 @@ def test_a_listener_registered_before_connecting_sees_the_handshake_burst( f"{handshake_burst.settled_in}s: {counted}") assert handshake_burst.closed, "the recorder was still running - see conftest" - assert handshake_burst.settled_in is not None, ( - f"the seed preset never arrived, so the burst was cut off by the " - f"fixture's patience rather than by finishing - {report}") + unfinished = handshake_burst.unfinished() + assert unfinished is None, f"{unfinished} - {report}" assert len(names) >= 100, report assert len(counted) >= 15, f"too few distinct state types in the burst - {report}" # Nothing in the handshake REQUESTS these. The subscription is a burst of diff --git a/tests/hardware/test_model_state.py b/tests/hardware/test_model_state.py index db4698e..e4202d3 100644 --- a/tests/hardware/test_model_state.py +++ b/tests/hardware/test_model_state.py @@ -152,14 +152,23 @@ def test_the_connect_burst_warms_the_cache(burst_warmed, handshake_burst, Read from a snapshot the connection fixture took the moment the burst finished, so this says the BURST filled it rather than some earlier test. + + ``unfinished`` comes first because the snapshot is only evidence if the + recording it came from is whole. A burst that was cut off leaves the same + absence as a unit that never sent the message, and the two want opposite + responses - so the cut-off case says so in its own words rather than being + reported as a finding about the unit. """ + unfinished = handshake_burst.unfinished() + assert unfinished is None, unfinished + record_property("burst_settled_in_s", round(handshake_burst.settled_in, 3)) record_property("burst_warmed", {name: sorted(fields) for name, fields in burst_warmed.items()}) counted = collections.Counter(handshake_burst.names()) assert counted.get("PresetDirtyMessage"), ( - f"the burst carried no PresetDirty, so there was nothing to warm the " - f"cache with - it recorded {dict(counted)}") + f"the burst ran to completion and carried no PresetDirty, so there was " + f"nothing to warm the cache with - it recorded {dict(counted)}") assert "is_dirty" in burst_warmed["dirty"], ( f"the unit announced its unsaved-changes state during the burst and the " f"model did not keep it - the cache held {burst_warmed}") @@ -203,6 +212,8 @@ def test_the_burst_warms_identity_from_connects_own_version_read(burst_warmed, entry stays marked and the first read of ``device.firmware`` still goes to the unit (see ``identity`` in device/entries.py). """ + unfinished = handshake_burst.unfinished() + assert unfinished is None, unfinished shapes = handshake_burst.versions() full = [s for s in shapes if "device_serial_number" in s[1]] own_reads = [s for s in shapes diff --git a/tests/hardware/test_preset_surface.py b/tests/hardware/test_preset_surface.py index 71305db..40e9426 100644 --- a/tests/hardware/test_preset_surface.py +++ b/tests/hardware/test_preset_surface.py @@ -194,16 +194,24 @@ def test_an_inactive_scene_refuses_writes_and_reads_fine(device): # -- the connect burst leaves the cache warm ---------------------------------- -def test_the_burst_delivered_every_entry_the_preset_surface_reads(burst_warmed): +def test_the_burst_delivered_every_entry_the_preset_surface_reads(burst_warmed, + handshake_burst): """Measured 2026-08-15: the burst carries RecallPreset, SetlistPosition, PresetDirty and Scene at about 10 s, inside ten milliseconds. The unit never announces its own firmware; identity is in the cache anyway since ADR-0020, because ``connect()`` reads ``Version`` before the handshake and the state layer is listening by then (measured 2026-09-07 on 4.0.1 / d14e). Asserted with its two kept fields, to keep this from passing on a run where the burst - delivered nothing at all.""" + delivered nothing at all. + + The snapshot is read only once the recording it came from is known to be + whole: a burst cut off early leaves exactly the absence a unit that stopped + sending would, and only one of those two is a finding about the unit.""" + unfinished = handshake_burst.unfinished() + assert unfinished is None, unfinished for name in ("preset", "scene", "dirty", "loaded"): - assert burst_warmed[name], f"the burst delivered nothing for {name}" + assert burst_warmed[name], ( + f"the burst ran to completion and delivered nothing for {name}") assert set(burst_warmed["identity"]) == {"device_serial_number", "app_fw_version"}, ( f"connect's own Version read should have warmed identity with its two " f"kept fields; the cache held {burst_warmed['identity']}. If the unit's " diff --git a/tests/test_handshake_burst_recorder.py b/tests/test_handshake_burst_recorder.py index 82eade3..173d02e 100644 --- a/tests/test_handshake_burst_recorder.py +++ b/tests/test_handshake_burst_recorder.py @@ -10,12 +10,14 @@ deleting the wrong assertion: * The hardware burst test covers the WIRING. If the fixture stopped calling - ``record_until``, its ``assert handshake_burst.closed`` and ``settled_in is not + ``record_until``, its ``assert handshake_burst.closed`` and ``unfinished() is None`` fail loudly - neither is a floor, so contamination cannot satisfy them. Those two lines are load-bearing, not belt-and-braces. * This file covers the STOPPING ITSELF, which no hardware test can see: a recorder - that sets its flag and keeps recording anyway, or one that stops recording but - stays on the transport, reads exactly like a working one from the outside. + that sets its flag and keeps recording anyway, one that stops recording but + stays on the transport, or one that stops on the FIRST message of the burst's + closing group rather than the whole of it, reads exactly like a working one + from the outside - the last of those about fourteen runs in fifteen. Timing is real here rather than faked, so the code under test is the code that runs on the unit. @@ -70,6 +72,21 @@ def _file_push(): return pa.FileMessage(action=pa.MessageAction.UPDATE) +def _tail(): + """The four messages that close the burst, in the order the unit sends them. + + Built here rather than read off ``BURST_TAIL``, so a test that feeds the tail + and a recorder that waits for it cannot agree with each other by sharing one + wrong list. + """ + return [ + pa.RecallPresetMessage(action=pa.MessageAction.UPDATE), + pa.SetlistPositionMessage(action=pa.MessageAction.UPDATE, position=1), + pa.PresetDirtyMessage(action=pa.MessageAction.UPDATE, is_dirty=False), + pa.SceneMessage(action=pa.MessageAction.UPDATE, selected_scene=1), + ] + + def test_closing_stops_the_recording_and_takes_the_listener_off(recorder_class): transport = FakeTransport() burst = recorder_class() @@ -91,7 +108,7 @@ def test_closing_stops_the_recording_and_takes_the_listener_off(recorder_class): assert transport.listeners == [] -def test_record_until_stops_when_the_sentinel_arrives(recorder_class): +def test_record_until_stops_when_the_whole_tail_has_arrived(recorder_class): transport = FakeTransport() burst = recorder_class() burst.attach(transport) @@ -99,19 +116,20 @@ def test_record_until_stops_when_the_sentinel_arrives(recorder_class): def push_the_burst(): for _ in range(3): burst(_file_push()) - burst(pa.RecallPresetMessage(action=pa.MessageAction.UPDATE)) + for message in _tail(): + burst(message) feeder = threading.Timer(0.1, push_the_burst) feeder.start() started = time.monotonic() - burst.record_until("RecallPresetMessage", patience=5.0) + burst.record_until(recorder_class.BURST_TAIL, patience=5.0) took = time.monotonic() - started feeder.join() - assert took < 5.0, "it waited out its patience instead of noticing the sentinel" + assert took < 5.0, "it waited out its patience instead of noticing the tail" assert burst.settled_in is not None assert burst.closed - assert burst.names() == ["FileMessage"] * 3 + ["RecallPresetMessage"] + assert burst.names() == ["FileMessage"] * 3 + list(recorder_class.BURST_TAIL) def test_record_until_gives_up_rather_than_hanging_on_a_silent_unit(recorder_class): @@ -123,9 +141,9 @@ def test_record_until_gives_up_rather_than_hanging_on_a_silent_unit(recorder_cla burst.attach(transport) burst(_file_push()) - burst.record_until("RecallPresetMessage", patience=0.2) + burst.record_until(recorder_class.BURST_TAIL, patience=0.2) - assert burst.settled_in is None, "it reported settling on a sentinel it never saw" + assert burst.settled_in is None, "it reported settling on a tail it never saw" assert burst.closed assert transport.listeners == [] @@ -153,3 +171,88 @@ def push(): writer.join() assert len(burst.names()) == 800 + + +def test_record_until_waits_for_the_whole_tail_not_just_the_first_of_it( + recorder_class): + """The race that failed two hardware tests together, one run in fifteen. + + The burst's four closing messages arrive in the order ``RecallPreset``, + ``SetlistPosition``, ``PresetDirty``, ``Scene``, inside six milliseconds - + re-measured 2026-09-14 on d14e over three sessions, spreads of 3.6, 5.8 and + 6.0 ms (``docs/protocol.md``, "Connect burst, measured"). ``RecallPreset`` is + the FIRST of the four, not the last. A recorder that stops on it stops with + the poll's granularity of slack after it - so it can close in the gap before + the other three land, and the fixture takes its ``burst_warmed`` snapshot of + a cache that has not heard them yet. The two hardware tests that read that + snapshot then fail together while every test reading the live cache passes, + because the cache stays on the transport and gets the messages milliseconds + later. + + The gap staged here is far wider than the unit's, so this does not depend on + winning a race to say something. What it proves is the thing the old stop + condition got wrong: arriving AFTER the first of a group is not arriving + late. + """ + transport = FakeTransport() + burst = recorder_class() + burst.attach(transport) + + def push_the_tail(): + first, *rest = _tail() + burst(first) + # Three poll intervals, where the unit takes six milliseconds. + time.sleep(0.3) + for message in rest: + burst(message) + + feeder = threading.Thread(target=push_the_tail) + feeder.start() + burst.record_until(recorder_class.BURST_TAIL, patience=5.0) + feeder.join() + + assert burst.settled_in is not None, ( + f"the recorder gave up rather than settling; it is still waiting for " + f"{sorted(burst.missing())}") + assert burst.unfinished() is None + assert set(burst.names()) == set(recorder_class.BURST_TAIL), ( + f"the recorder stopped before the whole tail arrived: it holds " + f"{burst.names()}") + + +def test_an_unfinished_recording_says_what_never_arrived(recorder_class): + """A burst that times out must be readable as that, not as a bare absence. + + Without this the two possible failures are indistinguishable from the + recording alone: the unit sent no ``PresetDirty`` at all, which is a finding + about the unit, and the recorder stopped before it arrived, which is a bug + here. The hardware tests put this string in front of their own assertions + so a future failure says which it was. + """ + transport = FakeTransport() + burst = recorder_class() + burst.attach(transport) + burst(pa.RecallPresetMessage(action=pa.MessageAction.UPDATE)) + + burst.record_until(recorder_class.BURST_TAIL, patience=0.2) + + assert burst.settled_in is None + assert burst.missing() == {"SetlistPositionMessage", "PresetDirtyMessage", + "SceneMessage"} + unfinished = burst.unfinished() + assert "PresetDirtyMessage" in unfinished, unfinished + assert "0.2" in unfinished, unfinished + + +def test_record_until_refuses_a_single_name(recorder_class): + """One name is the bug this signature replaced, and a string is a collection. + + ``frozenset("RecallPresetMessage")`` is twelve letters, so passing the old + argument to the new signature would wait for message types called ``R``, + ``e``, ``c`` and never settle - a thirty-second timeout per run and a + recording of nothing, arriving as a puzzle rather than as a mistake. + """ + burst = recorder_class() + burst.attach(FakeTransport()) + with pytest.raises(TypeError, match="collection"): + burst.record_until("RecallPresetMessage", patience=0.1) From 297e19468125e78c069711942727ef63da01e082 Mon Sep 17 00:00:00 2001 From: Jonathan Stokes Date: Mon, 14 Sep 2026 16:09:40 -0500 Subject: [PATCH 2/4] fix: pre-PR self-review findings on the burst recorder BURST_TAIL and entries.ENTRIES were two hand-written lists about one moment on the wire, with nothing holding them together: an entry added with a new burst message would be snapshotted before that message arrived, which is this bug again with a different name on it. An offline test now makes every entry either fed by something in BURST_TAIL or named in NOT_WARMED_BY_THE_BURST with the reason, and there is no third way. The reverse direction is held too - a name in BURST_TAIL that feeds no entry is a wait paid on every run for nothing. missing() walked the whole recording under the lock, ten times a second, while the RX thread wanted that lock to append at ~1490 reports/s - the cost the _recorded() it replaced said it existed to avoid. The seen type names are now kept as a set in __call__, so the append stays O(1) and the poll is a set difference. record_until now refuses an empty collection as well as a single name: nothing to wait for is satisfied by the first poll, so the recorder would come off the transport before the burst began and then report it as finished. Both refusals close the recorder on the way out - they run with the listener already attached, and a raise that skipped close() would leave it recording the whole session with nothing left to stop it. The unfinished() guard came off the identity test. Identity arrives at 0.7 s from connect's own Version read, long before the tail, so a cut-off burst would have failed a test whose evidence was still good. Also: the whole-tail test compared against BURST_TAIL's declaration order, which record_until does not use - it builds a frozenset - so correcting that order would have failed a test for no behavioural reason; the never-recorded branch of unfinished() had no coverage; the broadcast-listener failure printed "settled in None s" beside the sentence that already explained it; and the preset-surface docstring still carried 10 s / 10 ms beside this change's 11.1 s / 3.6-6.0 ms. Co-Authored-By: Claude Opus 5 --- tests/hardware/conftest.py | 67 +++++++++++------ tests/hardware/test_broadcast_listener.py | 5 +- tests/hardware/test_model_state.py | 2 - tests/hardware/test_preset_surface.py | 5 +- tests/test_handshake_burst_recorder.py | 91 ++++++++++++++++++++++- 5 files changed, 139 insertions(+), 31 deletions(-) diff --git a/tests/hardware/conftest.py b/tests/hardware/conftest.py index 2a5b576..6372c54 100644 --- a/tests/hardware/conftest.py +++ b/tests/hardware/conftest.py @@ -233,6 +233,11 @@ def __init__(self): #: that only counts cannot tell a retried identity read from a changed #: handshake. Shapes only - the values are not this recorder's business. self._versions = [] + #: Type names seen, kept alongside ``_names`` so :meth:`missing` is a set + #: difference rather than a walk of the whole recording. The recording + #: reaches several hundred names and is polled while the RX thread is + #: delivering ~1490 reports/s and wants this same lock to append. + self._seen_types = set() self._detach = None self._sentinels = frozenset() self._patience = None @@ -249,7 +254,9 @@ def __call__(self, message): # The RX thread notifies from a snapshot, so a message can still # arrive after removal. It must not reopen the recording. return - self._names.append(type(message).__name__) + name = type(message).__name__ + self._names.append(name) + self._seen_types.add(name) if isinstance(message, pa.VersionMessage): self._versions.append( (message.action, frozenset(f.name for f, _ in message.ListFields()))) @@ -276,35 +283,49 @@ def record_until(self, sentinels, patience): Stops on ``patience`` seconds regardless, so a unit that never sends one cannot hang the run. ``settled_in`` says which of the two happened, and :meth:`unfinished` says it in words for a failure message. + + Both refusals below close the recorder on the way out, which is what the + ``finally`` is for: this runs with the listener already attached, so a + raise that skipped it would leave the recording on the transport for the + whole session with nothing left to stop it. """ - if isinstance(sentinels, str): - raise TypeError( - "record_until takes a collection of type names, not one name - " - "the burst is over when all of them have arrived, and a single " - "name is the bug this signature replaced. A str is a collection " - "of letters, so this would otherwise wait for message types " - "called 'R', 'e' and 'c' and never settle.") - self._sentinels = frozenset(sentinels) - self._patience = patience - started = time.monotonic() - deadline = started + patience - while time.monotonic() < deadline: - if not self.missing(): - self.settled_in = time.monotonic() - started - break - time.sleep(0.1) - self.close() + try: + if isinstance(sentinels, str): + raise TypeError( + "record_until takes a collection of type names, not one " + "name - the burst is over when all of them have arrived, " + "and a single name is the bug this signature replaced. A " + "str is a collection of letters, so this would otherwise " + "wait for message types called 'R', 'e' and 'c' and never " + "settle.") + self._sentinels = frozenset(sentinels) + if not self._sentinels: + raise ValueError( + "record_until needs at least one type name. An empty " + "collection is satisfied by the first poll, so the recorder " + "would come off the transport before the burst began and " + "then report it as finished - which is the misattribution " + "this whole method exists to prevent.") + self._patience = patience + started = time.monotonic() + deadline = started + patience + while time.monotonic() < deadline: + if not self.missing(): + self.settled_in = time.monotonic() - started + break + time.sleep(0.1) + finally: + self.close() def missing(self): """The sentinel types :meth:`record_until` has not seen yet. - Builds a set of the recording each time rather than scanning it once per - sentinel: the recording runs to several hundred names by the time the - burst closes, and this is polled ten times a second at the busiest - moment the link has. + Reads the set :meth:`__call__` maintains rather than walking the + recording: this is polled ten times a second at the busiest moment the + link has, under the lock the RX thread needs to append. """ with self._lock: - return self._sentinels - set(self._names) + return self._sentinels - self._seen_types def unfinished(self): """Why the recording is short of the whole burst, or ``None`` if it is not. diff --git a/tests/hardware/test_broadcast_listener.py b/tests/hardware/test_broadcast_listener.py index a87fe0f..ffede19 100644 --- a/tests/hardware/test_broadcast_listener.py +++ b/tests/hardware/test_broadcast_listener.py @@ -78,8 +78,9 @@ def test_a_listener_registered_before_connecting_sees_the_handshake_burst( """ names = handshake_burst.names() counted = {name: names.count(name) for name in sorted(set(names))} - report = (f"recorded {len(names)} message(s), settled in " - f"{handshake_burst.settled_in}s: {counted}") + settled = ("" if handshake_burst.settled_in is None + else f", settled in {handshake_burst.settled_in:.1f}s") + report = f"recorded {len(names)} message(s){settled}: {counted}" assert handshake_burst.closed, "the recorder was still running - see conftest" unfinished = handshake_burst.unfinished() diff --git a/tests/hardware/test_model_state.py b/tests/hardware/test_model_state.py index e4202d3..481546d 100644 --- a/tests/hardware/test_model_state.py +++ b/tests/hardware/test_model_state.py @@ -212,8 +212,6 @@ def test_the_burst_warms_identity_from_connects_own_version_read(burst_warmed, entry stays marked and the first read of ``device.firmware`` still goes to the unit (see ``identity`` in device/entries.py). """ - unfinished = handshake_burst.unfinished() - assert unfinished is None, unfinished shapes = handshake_burst.versions() full = [s for s in shapes if "device_serial_number" in s[1]] own_reads = [s for s in shapes diff --git a/tests/hardware/test_preset_surface.py b/tests/hardware/test_preset_surface.py index 40e9426..803d1e9 100644 --- a/tests/hardware/test_preset_surface.py +++ b/tests/hardware/test_preset_surface.py @@ -196,8 +196,9 @@ def test_an_inactive_scene_refuses_writes_and_reads_fine(device): def test_the_burst_delivered_every_entry_the_preset_surface_reads(burst_warmed, handshake_burst): - """Measured 2026-08-15: the burst carries RecallPreset, SetlistPosition, - PresetDirty and Scene at about 10 s, inside ten milliseconds. The unit never + """The burst carries RecallPreset, SetlistPosition, PresetDirty and Scene as + one group - about 11.1 s in, spread over 3.6 to 6.0 ms, re-measured + 2026-09-14 on 4.0.1 / d14e (``docs/protocol.md``). The unit never announces its own firmware; identity is in the cache anyway since ADR-0020, because ``connect()`` reads ``Version`` before the handshake and the state layer is listening by then (measured 2026-09-07 on 4.0.1 / d14e). Asserted diff --git a/tests/test_handshake_burst_recorder.py b/tests/test_handshake_burst_recorder.py index 173d02e..d574a5f 100644 --- a/tests/test_handshake_burst_recorder.py +++ b/tests/test_handshake_burst_recorder.py @@ -29,6 +29,7 @@ import pytest +from pyquadcortex.device import entries from pyquadcortex.protocol.proto import ProductionAutomation_pb2 as pa _HARDWARE_CONFTEST = ( @@ -129,7 +130,11 @@ def push_the_burst(): assert took < 5.0, "it waited out its patience instead of noticing the tail" assert burst.settled_in is not None assert burst.closed - assert burst.names() == ["FileMessage"] * 3 + list(recorder_class.BURST_TAIL) + # Against the order _tail() pushes, not the order BURST_TAIL declares: + # record_until turns that tuple into a frozenset, so its order is a record + # of what the unit does and nothing here may fail when it is corrected. + assert burst.names() == ["FileMessage"] * 3 + [ + type(m).__name__ for m in _tail()] def test_record_until_gives_up_rather_than_hanging_on_a_silent_unit(recorder_class): @@ -252,7 +257,89 @@ def test_record_until_refuses_a_single_name(recorder_class): ``e``, ``c`` and never settle - a thirty-second timeout per run and a recording of nothing, arriving as a puzzle rather than as a mistake. """ + transport = FakeTransport() burst = recorder_class() - burst.attach(FakeTransport()) + burst.attach(transport) with pytest.raises(TypeError, match="collection"): burst.record_until("RecallPresetMessage", patience=0.1) + assert transport.listeners == [], ( + "a refused call left the recorder on the transport, where it would " + "record the whole session with nothing left to stop it") + + +#: Cache entries the connect burst does NOT warm, and why. A new ``StateEntry`` +#: comes through here or through ``BURST_TAIL``, and there is no third way - +#: which is the point of the test below. +NOT_WARMED_BY_THE_BURST = { + "identity": "the unit never announces its identity. It reaches the cache " + "because connect() READs Version before the handshake and the " + "state layer is already listening (ADR-0020), which is not the " + "burst and does not arrive with it", +} + + +def test_burst_tail_names_every_entry_the_burst_is_expected_to_warm( + recorder_class): + """The drift that would re-open the race, caught offline. + + ``BURST_TAIL`` decides when the fixture stops recording and snapshots + ``burst_warmed``; ``entries.ENTRIES`` decides what that snapshot contains. + They are two hand-written lists about one moment on the wire, and nothing + held them together - so an entry added with a new burst message would be + snapshotted before its message had arrived, exactly the way ``dirty`` and + ``scene`` were. It would fail one run in however-many, with a message + blaming the unit. + + So: every entry is either fed by something in ``BURST_TAIL``, or named above + with the reason it is not. Deciding which is a judgement about the wire, and + this test is where a new entry has to state it. + """ + tail = set(recorder_class.BURST_TAIL) + for entry in entries.ENTRIES: + fed_by_the_burst = {t.__name__ for t in entry.feeds} & tail + excused = NOT_WARMED_BY_THE_BURST.get(entry.name) + assert bool(fed_by_the_burst) != bool(excused), ( + f"entry {entry.name!r} is fed by {sorted(fed_by_the_burst)} and " + f"excused with {excused!r} - it must be one or the other. If the " + f"burst warms it, the message that does so belongs in BURST_TAIL " + f"so the recorder waits for it; if it does not, say why in " + f"NOT_WARMED_BY_THE_BURST.") + + +def test_every_burst_tail_message_actually_feeds_an_entry(recorder_class): + """The other direction: a name in ``BURST_TAIL`` nothing reads is a wait + the fixture pays on every run for no reason, and it would hold the whole + suite for the full patience if the unit ever stopped sending it.""" + fed = {t.__name__ for entry in entries.ENTRIES for t in entry.feeds} + assert set(recorder_class.BURST_TAIL) <= fed, ( + f"BURST_TAIL waits for {sorted(set(recorder_class.BURST_TAIL) - fed)}, " + f"which feeds no cache entry") + + +def test_a_recorder_that_never_recorded_says_so(recorder_class): + """The branch that tells "never started" apart from "timed out". + + Unreachable from the fixture, which always calls ``record_until`` - so + without this the one line that would name a fixture that stopped calling it + is itself unverified. + """ + burst = recorder_class() + burst.attach(FakeTransport()) + assert burst.unfinished() == ( + "the burst was never recorded: record_until was not called") + + +def test_record_until_refuses_an_empty_collection(recorder_class): + """Nothing to wait for is satisfied immediately, which reads as success. + + The recorder would come off the transport before the burst began and report + a finished burst, and every guard added for this race would pass while the + snapshot held nothing. + """ + transport = FakeTransport() + burst = recorder_class() + burst.attach(transport) + with pytest.raises(ValueError, match="at least one type name"): + burst.record_until((), patience=5.0) + assert burst.settled_in is None, "it settled on an empty condition" + assert transport.listeners == [], "a refused call left the recorder attached" From e0adcc41eaa3060173ea0e4af8894fb012301cab Mon Sep 17 00:00:00 2001 From: Jonathan Stokes Date: Mon, 14 Sep 2026 16:33:36 -0500 Subject: [PATCH 3/4] fix: triage findings on the burst recorder The snapshot race was narrowed, not closed. Listeners run in registration order and the fixture registered the recorder first, so for the fourth tail message the recorder counted it and DeviceState.apply_push had not yet run - the poll could settle, close, and snapshot a cache one message behind. One __call__ wide instead of six milliseconds, and worse to debug, because unfinished() truthfully returned None and the test blamed the unit. The cache is registered first now, which makes "the recorder has all four" mean the cache has applied them. record_until tested its deadline before its condition, so a tail landing during the final sleep was recorded in full and then reported as a burst that never finished - naming nothing, because by then nothing was missing. It now reads the condition once per pass and uses that one read for both the settle and the give-up, so the set it names is the set that was outstanding when it gave up. unfinished() said the opposite of what it now means. Stopping early is impossible once the recorder waits for the whole group, which leaves the unit as the only thing that can make it non-None - and the sentence told the reader to rule that out. The drift guard asked whether an entry had ANY feed in BURST_TAIL, so an entry with one foot in the tail and one in a later burst message passed while re-opening the race. It is per feed type now, with OUTSIDE_THE_BURST naming the four feeds that arrive at some other time and why. Both lists are held against entries.ENTRIES so a stale excuse cannot linger. Also: record_until refuses message classes, which are neither a str nor empty and would have burned the whole patience and then named a class as something the unit never sent; a refused call no longer reports that it was never made; tail_positions() records how far from the end the group landed, which is the signal for the one thing the stop condition cannot prove - that these four are the LAST four; the give-up prints once in the run's own report, since --verifies or a single-file run deselects all three tests that guard on it; the settle path asserts detachment like every other exit; the concurrency test pushes mixed types so it touches _seen_types at all; the PresetDirty assertion was unreachable behind the guard above it and the surviving one says what is actually still open. Records: the measured 6.7% and 10.0% replace a derivation, with the method written down; the spreads keep the docs' per-session pairing; "fourteen runs in fifteen" was nine in ten; frozenset("RecallPresetMessage") is eleven distinct letters; the new paragraph says what its clock is measured from and that 9 to 11.2 s is session variance. tests/hardware/readme.md still described the old assertions. CLAUDE.md and STEERING.md carry the rule, since tests/ and docs/ are Owned Paths. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 2 +- docs/STEERING.md | 24 ++++ docs/protocol.md | 30 ++-- tests/hardware/conftest.py | 109 +++++++++++--- tests/hardware/readme.md | 13 +- tests/hardware/test_model_state.py | 30 ++-- tests/hardware/test_preset_surface.py | 4 +- tests/test_handshake_burst_recorder.py | 191 ++++++++++++++++++++++--- 8 files changed, 338 insertions(+), 65 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a7c61a2..b81a521 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,7 +7,7 @@ Read `docs/STEERING.md` before non-trivial work (new operations, transport or fr ## Conventions -- Dev setup: `uv venv && uv pip install -e ".[dev]"` (or plain venv + pip, see contributing.md). Run tests with `.venv/bin/python -m pytest`. The suite passes offline - no hardware, no `hid` import, no `DYLD_LIBRARY_PATH`. The hardware suite's `--hardware` gate is two hooks in `tests/hardware/conftest.py` and stays two: pytest skips `pytest_ignore_collect` for a path named on the command line, so `pytest_collection_modifyitems` refuses that one loudly. Never fold them into one; `tests/test_hardware_gate.py` proves both. A module in `tests/hardware/` also needs a basename no module under `tests/` owns - pytest maps `tests/hardware/test_scales.py` and `tests/test_scales.py` to one module name and refuses the second, which is why two files there end `_on_unit`. Rename; do not reach for `__init__.py` or `consider_namespace_packages`, because three offline modules do `from waiting import ...` and that works only while pytest puts `tests/` on `sys.path`. A hardware test names what it verifies AND actually asserts on with `@pytest.mark.verifies(*operations)`, checked against `QuadCortex.operations()` at collection; `--verifies NAME` narrows a run to the tests naming it, and refuses a name that is not an operation or that no collected test names. `--profile CLASSNAME` (with `--hardware`) connects as that profile class instead of the one the unit resolves to, which is how a unit the registry would refuse - an unmeasured firmware, or a Mini - gets measured by the suite that measures it. `scratch_preset` is the fixture for getting a disposable copy of the loaded preset to edit. An operation no hardware test names has to appear in `tests/test_hardware_markers.py`'s `UNMARKED_OPERATIONS`, with the reason - that dict is the list a new operation must come through if no hardware test names it. +- Dev setup: `uv venv && uv pip install -e ".[dev]"` (or plain venv + pip, see contributing.md). Run tests with `.venv/bin/python -m pytest`. The suite passes offline - no hardware, no `hid` import, no `DYLD_LIBRARY_PATH`. The hardware suite's `--hardware` gate is two hooks in `tests/hardware/conftest.py` and stays two: pytest skips `pytest_ignore_collect` for a path named on the command line, so `pytest_collection_modifyitems` refuses that one loudly. Never fold them into one; `tests/test_hardware_gate.py` proves both. A module in `tests/hardware/` also needs a basename no module under `tests/` owns - pytest maps `tests/hardware/test_scales.py` and `tests/test_scales.py` to one module name and refuses the second, which is why two files there end `_on_unit`. Rename; do not reach for `__init__.py` or `consider_namespace_packages`, because three offline modules do `from waiting import ...` and that works only while pytest puts `tests/` on `sys.path`. A hardware test names what it verifies AND actually asserts on with `@pytest.mark.verifies(*operations)`, checked against `QuadCortex.operations()` at collection; `--verifies NAME` narrows a run to the tests naming it, and refuses a name that is not an operation or that no collected test names. `--profile CLASSNAME` (with `--hardware`) connects as that profile class instead of the one the unit resolves to, which is how a unit the registry would refuse - an unmeasured firmware, or a Mini - gets measured by the suite that measures it. `scratch_preset` is the fixture for getting a disposable copy of the loaded preset to edit. The connect burst is waited for as a GROUP: `HandshakeBurst.BURST_TAIL` names the four messages that close it, and the fixture registers the model cache before the recorder so that "the recorder has all four" means the cache has applied them. Never wait on one message of that group - `RecallPreset` is its FIRST member and the other three trail it by 3.6-6.0 ms against a 100 ms poll, which cost two hardware tests one run in ten. A new `StateEntry` comes through `BURST_TAIL`, `OUTSIDE_THE_BURST` or `NOT_WARMED_BY_THE_BURST` in `tests/test_handshake_burst_recorder.py`, with its reason, and there is no fourth way. An operation no hardware test names has to appear in `tests/test_hardware_markers.py`'s `UNMARKED_OPERATIONS`, with the reason - that dict is the list a new operation must come through if no hardware test names it. - Two namespaces, one package (ADR-0006): `pyquadcortex` is the model of the unit, `pyquadcortex.protocol` is the message-level API. The model's code lives in `pyquadcortex/device/` - not `model/`, because in this codebase the identifier `model` means an amp or pedal block (`protocol/models.py`, `catalog.Model`, `ModelCatalog`, `set_block(model=...)`). The model imports the protocol layer; nothing under `pyquadcortex/protocol/` may import from `pyquadcortex/device/`. - Every conversion between a screen value and a wire value lives in the `pyquadcortex/device/translate/` PACKAGE and nowhere else in the package outside `pyquadcortex/protocol/` - the whole package, not just `device/`, because a rule scoped to a directory is satisfied by moving the code one directory up. It covers rows 1-4, slots 1-8, scene and footswitch letters, preset addresses and display units. Outside the boundary that means no `+1`/`-1` on a coordinate AND none of the other spellings of the same conversion (`ord`/`chr`, a letter table in any container, `divmod` on a position, a one-based `enumerate`, `ROWS.index(...)`), and no module reaching past the boundary for a protocol-layer name that carries a coordinate or a raw scale - the converters and also the readers that hand back wire indexes, such as `protocol.stomp_assignments`. `tests/test_translation.py` reads the source and proves both, and pins where each check stops seeing rather than implying it sees everything. The boundary is a package, so its exemption covers a DIRECTORY: `BOUNDARY_MODULES` in that file names the modules inside it, and adding `translate/anything.py` has to come through that list with a reason - otherwise the arithmetic scan skips the new file for the same reason it skips the real converters. The same file now carries a second list, `PROTOCOL_NON_CONVERSIONS`, for protocol-layer names the boundary uses that are NOT conversions (`field_present`, the port enums). It exists because the boundary reads whole presets now, so "everything it reaches for is a conversion" stopped being true; a name has to be on one list or the other, and which one is a judgement a reviewer makes from the reason written beside it. A model API takes `FootswitchLetter`, never a bare footswitch integer, because a footswitch index and a block's column are different numbers that usually agree. A new conversion goes in that module with its own test, however small it is, and its protocol-layer name joins that file's allowlist in the same commit. - The model represents what the unit shows, in the unit's own words, and never guesses. A control we understand but cannot yet drive is modelled and REFUSES the operation (ADR-0007); a control we do not understand is omitted, with the reason recorded in `docs/domain-model.md`'s appendix. Nothing ships with a "this might be stale or wrong" caveat. A refusal is `protocol.ControlNotDrivable(control, evidence, workaround)` - never a bare `ValueError` and never a message-only raise. All three fields are required, and `evidence` records what was actually tried, because a refusal nobody can audit is the guess the rule exists to prevent. Before writing one, ADR-0010 applies: the differential capture comes first, and "the unit broadcasts nothing" is not on its own a finding about the wire. Do not derive a refusal from a RULE about parameter types either - the first instance had three plausible rules tried against hardware and all three were false, so `LANE_OUTPUT_UNASSIGNABLE` is a measured list with the disproofs written beside it. diff --git a/docs/STEERING.md b/docs/STEERING.md index 57cfa85..f56a296 100644 --- a/docs/STEERING.md +++ b/docs/STEERING.md @@ -138,6 +138,30 @@ Single-device, single-connection USB HID at interactive rates (129-byte reports) ## Change Log +### 2026-09-14 - The connect burst is waited for as a group, not a head message + +**What changed:** the hardware suite's burst recorder waits for every message in +`HandshakeBurst.BURST_TAIL` - `RecallPreset`, `SetlistPosition`, `PresetDirty`, +`Scene` - rather than for `RecallPreset` alone. The connection fixture registers +the model cache BEFORE the recorder, so "the recorder has all four" implies "the +cache has applied all four" rather than merely nearly. `unfinished()` names what +never arrived, and prints once in the run's own report as well as in the three +tests that guard on it. A new cache entry now has to come through `BURST_TAIL`, +`OUTSIDE_THE_BURST` or `NOT_WARMED_BY_THE_BURST` in +`tests/test_handshake_burst_recorder.py`, each with its reason - the same shape +as `UNMARKED_OPERATIONS` and `BOUNDARY_MODULES`. + +**Why:** `RecallPreset` is the FIRST of those four on the wire and the other three +follow within 3.6 to 6.0 ms, against a 100 ms poll. Two hardware tests failed +together on 2026-09-11 and passed on a re-run of the same commit; measured at +6.7% and 10.0% of runs. A gate that fails intermittently for reasons unrelated to +the change teaches people to re-run until green, which is how a real regression +gets waved through. + +**Scope of impact:** +- **Updated:** tests/hardware/conftest.py, tests/hardware/test_model_state.py, tests/hardware/test_preset_surface.py, tests/hardware/test_broadcast_listener.py, tests/hardware/readme.md, tests/test_handshake_burst_recorder.py, docs/protocol.md "Connect burst, measured", CLAUDE.md, STEERING.md section 10 +- **Not updated (intentionally):** ADR.md - the stop condition is a bug fix in the suite, not an architectural choice; nothing under `pyquadcortex/` - the library never had this bug; `BURST_TAIL` stays off the profile class (ADR-0020) until a second profile has measured its own burst, and the timeout names the message it never saw, which is the evidence that would move it + ### 2026-09-11 - The Global EQ gain span is measured at its ends (ADR-0017) **What changed:** `units.SETTING_SPANS["GLOBAL_EQ_GAIN_DB"]` is still diff --git a/docs/protocol.md b/docs/protocol.md index 1b5d6b1..c5de100 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -2095,15 +2095,27 @@ the order `RecallPreset`, `SetlistPosition`, `PresetDirty`, `Scene` - and with n pushes, because nothing changed. The seed `RecallPreset` sets `action`, `preset` and `reason`. -Re-measured 2026-09-14 on 4.0.1 / d14e over three consecutive connections: the four landed -at 11.21 s, 11.08 s and 11.07 s, spread over 6.0 ms, 3.6 ms and 5.8 ms, in that same order -every time. **`RecallPreset` is the FIRST of the four, not the last**, which matters to -anything that treats it as the signal the burst is over: the other three are still a few -milliseconds out. The hardware suite's burst recorder did exactly that and lost -`PresetDirty` in roughly one run in fifteen - it polls at 100 ms, so its stop landed at a -uniformly random point inside the 100 ms after `RecallPreset`, and about 6 ms of that -window is before the group has finished arriving. It now waits for all four -(`BURST_TAIL` in `tests/hardware/conftest.py`). Wait for the group, not for its head. +Re-measured 2026-09-14 on 4.0.1 / d14e over three consecutive connections, timestamped from +before the handshake by a `connect(before_handshake=...)` listener, so these are seconds +after connect on the same clock as the 4.9 s / 5.1 s figures above. The four landed at +11.21 s, 11.08 s and 11.07 s, spread over 6.0 ms, 3.6 ms and 5.8 ms, in that same order +every time. The arrival time varies session to session - 9 s to 11.2 s observed - so it is +the ORDER and the SPREAD that are stable enough to build on, not the moment. + +**`RecallPreset` is the FIRST of the four, not the last.** That matters to anything +treating it as the signal that the burst is over: the other three are still a few +milliseconds out. The hardware suite's burst recorder did exactly that, and it polls at +100 ms, so its stop landed at a uniformly random point in the 100 ms after `RecallPreset` +and fell inside the group about one time in fifteen. + +Measured rather than derived, by driving the real recorder with the four messages at their +measured spacing and the poll's phase drawn uniformly from its 100 ms interval - which is +what a connection does to it - over 300 runs: it lost `PresetDirty` 20 times (6.7%) and +`Scene` 30 times (10.0%). `Scene` is lost more often because it arrives last, and losing +`Scene` alone is already enough to fail a test. After the change, 0 of 300 for both. The +recorder now waits for all four (`BURST_TAIL` in `tests/hardware/conftest.py`), and +`tests/test_handshake_burst_recorder.py` holds the mechanism offline. Wait for the group, +not for its head. **`read_preset()` RECALLS the slot it reads** - that was already documented - and the recall **resets the active scene to the preset's default, discards unsaved edits, and diff --git a/tests/hardware/conftest.py b/tests/hardware/conftest.py index 6372c54..8c69c78 100644 --- a/tests/hardware/conftest.py +++ b/tests/hardware/conftest.py @@ -220,8 +220,14 @@ class HandshakeBurst: #: 11.1 s inside 3.6, 5.8 and 6.0 ms respectively, always in this order #: (``docs/protocol.md``, "Connect burst, measured"). ``RecallPreset`` is #: the FIRST of them. Waiting for that one alone is what this used to do, - #: and it cost two hardware tests roughly one run in fifteen - see + #: and it cost two hardware tests about nine runs in ten - see #: ``tests/test_handshake_burst_recorder.py``. + #: + #: Measured on ONE profile. ADR-0020 puts what differs by firmware or model + #: on the profile class, and this has not earned that yet because there is + #: one measurement of it; a ``--profile`` run against a Mini or a 4.1 that + #: closes its burst differently will time out and name the message it never + #: saw, which is the evidence that would move it. BURST_TAIL = ("RecallPresetMessage", "SetlistPositionMessage", "PresetDirtyMessage", "SceneMessage") @@ -240,7 +246,9 @@ def __init__(self): self._seen_types = set() self._detach = None self._sentinels = frozenset() + self._missing_at_giveup = frozenset() self._patience = None + self._called = False self.closed = False self.settled_in = None # seconds the burst took, or None if it timed out @@ -289,15 +297,24 @@ def record_until(self, sentinels, patience): raise that skipped it would leave the recording on the transport for the whole session with nothing left to stop it. """ + self._called = True try: if isinstance(sentinels, str): raise TypeError( "record_until takes a collection of type names, not one " "name - the burst is over when all of them have arrived, " "and a single name is the bug this signature replaced. A " - "str is a collection of letters, so this would otherwise " - "wait for message types called 'R', 'e' and 'c' and never " - "settle.") + "str is a collection of its letters, so this would " + "otherwise wait for message types called 'R', 'e' and 'c' " + "and never settle.") + wrong = sorted(repr(s) for s in sentinels if not isinstance(s, str)) + if wrong: + raise TypeError( + f"record_until matches NAMES against what the recorder " + f"stores, which is type(message).__name__; it was handed " + f"{', '.join(wrong)}. A message class never equals its own " + f"name, so the wait would burn the whole patience and then " + f"name that class as something the unit never sent.") self._sentinels = frozenset(sentinels) if not self._sentinels: raise ValueError( @@ -309,10 +326,21 @@ def record_until(self, sentinels, patience): self._patience = patience started = time.monotonic() deadline = started + patience - while time.monotonic() < deadline: - if not self.missing(): + while True: + # ONE read of the condition per pass, used for both the settle + # and the give-up. Reading it again after the deadline would let + # the RX thread empty it in between, and :meth:`unfinished` + # would then name nothing at all - a sentence with a blank where + # the missing types belong, on a recording that is in fact + # whole. Checking it BEFORE the deadline is what credits a tail + # that lands during the last sleep. + outstanding = self.missing() + if not outstanding: self.settled_in = time.monotonic() - started break + if time.monotonic() >= deadline: + self._missing_at_giveup = outstanding + break time.sleep(0.1) finally: self.close() @@ -330,22 +358,50 @@ def missing(self): def unfinished(self): """Why the recording is short of the whole burst, or ``None`` if it is not. - A test asserting on the recording puts this in front of its own - assertions, because without it the two failures read identically from - the recording alone: the unit sent no ``PresetDirty`` at all, which is a - finding about the unit, and the recorder stopped before it arrived, - which is a bug in here. + The two tests that read ``burst_warmed`` for entries the burst delivers + put this in front of their own assertions. Without it, "the cache never + got ``PresetDirty``" and "the recorder stopped before ``PresetDirty`` + arrived" read identically from the snapshot, and they want opposite + responses. + + Which of the two this sentence describes changed with the stop + condition, and the wording follows it. The recorder now waits for every + message in :data:`BURST_TAIL`, so it can no longer stop early - which + leaves the unit as the only thing that can make this non-``None``. """ + if not self._called: + return "the burst was never recorded: record_until was not called" if self.settled_in is not None: return None if not self._sentinels: - return "the burst was never recorded: record_until was not called" + return ("the burst was never recorded: record_until was called and " + "refused its argument - see the error it raised") return ( - f"the connect burst did not finish within {self._patience}s - " - f"{', '.join(sorted(self.missing()))} never arrived, of the " - f"{len(self._sentinels)} messages that close it. Everything below " - f"is a recording of a burst that was cut off, so read it as that " - f"rather than as a finding about the unit or the cache.") + f"the connect burst did not finish within {self._patience}s: " + f"{', '.join(sorted(self._missing_at_giveup))} never arrived, of " + f"the {len(self._sentinels)} messages that close it. The recorder " + f"waits for all of them, so it did not stop early - what is absent " + f"from the recording is absent from the unit's burst. Read it as a " + f"finding about the unit or the link.") + + def tail_positions(self): + """How far from the END of the recording each sentinel first appeared. + + The stop condition proves the four ARRIVED. It cannot prove they are the + LAST four, which is what the fixture's snapshot on the next line + actually rests on - and a firmware that appended a fifth closing message + would re-open this bug with nothing to catch it. Recorded rather than + asserted, because the number that would be asserted is one this project + has measured on one profile: a sentinel sitting well back from the end + shows up in the run's report before it shows up as a flake. + """ + with self._lock: + total = len(self._names) + first = {} + for index, name in enumerate(self._names): + if name in self._sentinels and name not in first: + first[name] = total - index + return first def close(self): """Stop recording and come off the transport. Idempotent. @@ -432,8 +488,16 @@ def _connection(request): cache = DeviceState() def subscribe(transport): - burst.attach(transport) + # The CACHE first, and the order is load-bearing. Listeners run in + # registration order (``Transport._notify_listeners``), and the + # recorder's stop condition is read one line below as "the cache has + # these too". Registered the other way round the poll can see the + # fourth tail message recorded and take the ``warmed`` snapshot before + # ``DeviceState.apply_push`` has run for that same message - the stale + # snapshot this file was just fixed for, one ``__call__`` wide instead + # of six milliseconds. This way round the implication is real. cache.listen_on(transport) + burst.attach(transport) # `--profile CLASSNAME` connects as that class instead of the one the unit's # identity resolves to, which is how a unit the registry would refuse - an @@ -449,6 +513,10 @@ def subscribe(transport): cache.bind(client) # Read by pytest_terminal_summary, which has a config and no fixtures. request.config._profile = type(client) + # Read by pytest_terminal_summary: a cut-off burst is otherwise reported + # only by the three tests that guard on it, and --verifies or a + # single-file run deselects all three while the fixture still waits. + request.config._burst = burst burst.record_until(HandshakeBurst.BURST_TAIL, patience=30.0) # Taken here, before any test can read through the cache, so "the burst # warmed this" cannot later be confused with "some test read it". @@ -734,6 +802,11 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config): tr.line("the hardware session never connected, so no operation was measured; " "see the setup errors above") return + burst = getattr(config, "_burst", None) + unfinished = None if burst is None else burst.unfinished() + if unfinished is not None: + tr.section("connect burst: cut off") + tr.line(unfinished) claimed = _claimed(_VERIFIES, _RAN) lines = _report_lines(cls, _OUTCOMES, claimed) width = max(len(label) for label, _names, _note in lines) diff --git a/tests/hardware/readme.md b/tests/hardware/readme.md index 09bc311..20ce00b 100644 --- a/tests/hardware/readme.md +++ b/tests/hardware/readme.md @@ -105,20 +105,23 @@ the burst, whatever order the tests run in. The metronome's tempo stream never stops, so a recorder left running would hold the whole run's traffic and a test asserting on it would really be asserting on whatever other tests provoked first. -The wait costs about 8 seconds once per run and buys more than it costs. +The wait costs about 9 seconds once per run and buys more than it costs. `connect()` returns roughly 3 seconds before the unit starts streaming several hundred messages, so without it every latency measurement below would be taken on a link still busy answering the handshake. -The burst test's `assert handshake_burst.closed` and `settled_in is not None` are +The burst test's `assert handshake_burst.closed` and `unfinished() is None` are what hold that up. They are not belt-and-braces: they are the only things that fail if the fixture stops waiting for the burst, since every other assertion in that test is a floor and contamination satisfies a floor. Do not delete them as -redundant. +redundant. `unfinished()` also prints once in the run's own report, so a cut-off +burst is named even on a run that deselects all three tests that guard on it. What they cannot see is a recorder that sets its flag and keeps recording anyway, -or one that stops recording but stays attached to the transport. Both read like a -working recorder from the outside, so both are pinned offline in +one that stops recording but stays attached to the transport, or one that stops +on the FIRST message of the burst's closing group instead of the whole of it. +All three read like a working recorder from the outside - the last of them about +nine runs in ten - so all three are pinned offline in `tests/test_handshake_burst_recorder.py`. ## The model's cache rides the same connection diff --git a/tests/hardware/test_model_state.py b/tests/hardware/test_model_state.py index 481546d..233d7dc 100644 --- a/tests/hardware/test_model_state.py +++ b/tests/hardware/test_model_state.py @@ -154,24 +154,30 @@ def test_the_connect_burst_warms_the_cache(burst_warmed, handshake_burst, finished, so this says the BURST filled it rather than some earlier test. ``unfinished`` comes first because the snapshot is only evidence if the - recording it came from is whole. A burst that was cut off leaves the same - absence as a unit that never sent the message, and the two want opposite - responses - so the cut-off case says so in its own words rather than being - reported as a finding about the unit. + recording it came from is whole: a burst cut off by the fixture's patience + leaves the same absence a unit that stopped sending would, and the two want + opposite responses. It says which in its own words. """ unfinished = handshake_burst.unfinished() assert unfinished is None, unfinished - record_property("burst_settled_in_s", round(handshake_burst.settled_in, 3)) + # One decimal, because the poll that produced it runs at 100 ms - three + # would read as a measurement the number cannot support. + record_property("burst_settled_in_s", round(handshake_burst.settled_in, 1)) record_property("burst_warmed", {name: sorted(fields) for name, fields in burst_warmed.items()}) - counted = collections.Counter(handshake_burst.names()) - - assert counted.get("PresetDirtyMessage"), ( - f"the burst ran to completion and carried no PresetDirty, so there was " - f"nothing to warm the cache with - it recorded {dict(counted)}") + # How far from the end of the recording each closing message landed. Not + # asserted - see HandshakeBurst.tail_positions - but a firmware that stopped + # closing the burst with these four shows up here first. + record_property("burst_tail_positions", handshake_burst.tail_positions()) + + # That PresetDirty ARRIVED is already settled: it is in BURST_TAIL, so the + # guard above cannot pass without it. What is still open, and is this test's + # subject, is whether the model KEPT what the unit announced. assert "is_dirty" in burst_warmed["dirty"], ( - f"the unit announced its unsaved-changes state during the burst and the " - f"model did not keep it - the cache held {burst_warmed}") + f"the unit announced its unsaved-changes state during the burst - the " + f"recording holds {collections.Counter(handshake_burst.names())['PresetDirtyMessage']} " + f"PresetDirty message(s) - and the model did not keep it. The cache " + f"held {burst_warmed}") def test_nothing_the_burst_delivered_is_read_again_on_first_access( diff --git a/tests/hardware/test_preset_surface.py b/tests/hardware/test_preset_surface.py index 803d1e9..d86f1a0 100644 --- a/tests/hardware/test_preset_surface.py +++ b/tests/hardware/test_preset_surface.py @@ -206,8 +206,8 @@ def test_the_burst_delivered_every_entry_the_preset_surface_reads(burst_warmed, delivered nothing at all. The snapshot is read only once the recording it came from is known to be - whole: a burst cut off early leaves exactly the absence a unit that stopped - sending would, and only one of those two is a finding about the unit.""" + whole: a burst cut off by the fixture's patience leaves exactly the absence + a unit that stopped sending would, and the two want opposite responses.""" unfinished = handshake_burst.unfinished() assert unfinished is None, unfinished for name in ("preset", "scene", "dirty", "loaded"): diff --git a/tests/test_handshake_burst_recorder.py b/tests/test_handshake_burst_recorder.py index d574a5f..33d7184 100644 --- a/tests/test_handshake_burst_recorder.py +++ b/tests/test_handshake_burst_recorder.py @@ -17,7 +17,14 @@ that sets its flag and keeps recording anyway, one that stops recording but stays on the transport, or one that stops on the FIRST message of the burst's closing group rather than the whole of it, reads exactly like a working one - from the outside - the last of those about fourteen runs in fifteen. + from the outside - the last of those about NINE runs in TEN, because losing + ``Scene`` alone is enough and ``Scene`` is the one lost most often. + +The two hardware tests that read ``burst_warmed`` for entries the burst delivers +call ``unfinished()`` before their own assertions. The identity test does not, +deliberately: identity reaches the cache from ``connect()``'s own ``Version`` +READ at about 0.7 s, so a tail that never arrived would fail it for something it +does not depend on. Timing is real here rather than faked, so the code under test is the code that runs on the unit. @@ -130,6 +137,11 @@ def push_the_burst(): assert took < 5.0, "it waited out its patience instead of noticing the tail" assert burst.settled_in is not None assert burst.closed + # The path the fixture actually takes, and the only one that used to go + # unchecked here: a stop inlined as `self.closed = True; break` leaves the + # recorder on the RX thread's listener list for the rest of the session and + # every other assertion in this file stays green. + assert transport.listeners == [], "the settle path left the recorder attached" # Against the order _tail() pushes, not the order BURST_TAIL declares: # record_until turns that tuple into a frozenset, so its order is a record # of what the unit does and nothing here may fail when it is corrected. @@ -163,19 +175,26 @@ def test_recording_and_reading_at_the_same_time_loses_nothing(recorder_class): burst = recorder_class() burst.attach(transport) + # Mixed types on purpose: 800 identical messages leave _seen_types at one + # element and never call missing() from the other thread, so the test that + # advertises concurrency would not touch the state the poll reads. def push(): - for _ in range(200): - burst(pa.FileMessage(action=pa.MessageAction.UPDATE)) + for _ in range(50): + for message in [pa.FileMessage(action=pa.MessageAction.UPDATE)] + _tail(): + burst(message) writers = [threading.Thread(target=push) for _ in range(4)] for writer in writers: writer.start() + known = {"FileMessage"} | set(recorder_class.BURST_TAIL) while any(writer.is_alive() for writer in writers): - assert all(name == "FileMessage" for name in burst.names()) + assert all(name in known for name in burst.names()) + assert burst.missing() <= frozenset() # no sentinels set: nothing outstanding for writer in writers: writer.join() - assert len(burst.names()) == 800 + # 4 writers x 50 passes x (one File plus the four-message tail). + assert len(burst.names()) == 4 * 50 * 5 def test_record_until_waits_for_the_whole_tail_not_just_the_first_of_it( @@ -184,8 +203,8 @@ def test_record_until_waits_for_the_whole_tail_not_just_the_first_of_it( The burst's four closing messages arrive in the order ``RecallPreset``, ``SetlistPosition``, ``PresetDirty``, ``Scene``, inside six milliseconds - - re-measured 2026-09-14 on d14e over three sessions, spreads of 3.6, 5.8 and - 6.0 ms (``docs/protocol.md``, "Connect burst, measured"). ``RecallPreset`` is + re-measured 2026-09-14 on d14e over three sessions, spreads of 6.0, 3.6 and + 5.8 ms (``docs/protocol.md``, "Connect burst, measured"). ``RecallPreset`` is the FIRST of the four, not the last. A recorder that stops on it stops with the poll's granularity of slack after it - so it can close in the gap before the other three land, and the fixture takes its ``burst_warmed`` snapshot of @@ -252,10 +271,11 @@ def test_an_unfinished_recording_says_what_never_arrived(recorder_class): def test_record_until_refuses_a_single_name(recorder_class): """One name is the bug this signature replaced, and a string is a collection. - ``frozenset("RecallPresetMessage")`` is twelve letters, so passing the old - argument to the new signature would wait for message types called ``R``, - ``e``, ``c`` and never settle - a thirty-second timeout per run and a - recording of nothing, arriving as a puzzle rather than as a mistake. + ``frozenset("RecallPresetMessage")`` is its eleven distinct letters, so + passing the old argument to the new signature would wait for message types + called ``R``, ``e``, ``c`` and never settle - a thirty-second timeout per + run and a recording of nothing, arriving as a puzzle rather than as a + mistake. """ transport = FakeTransport() burst = recorder_class() @@ -270,6 +290,18 @@ def test_record_until_refuses_a_single_name(recorder_class): #: Cache entries the connect burst does NOT warm, and why. A new ``StateEntry`` #: comes through here or through ``BURST_TAIL``, and there is no third way - #: which is the point of the test below. +#: Feed types that reach an entry OUTSIDE the connect burst, with when they do. +#: The recorder must not wait for these: nothing sends them on connect, so a +#: wait would cost the full patience on every run. +OUTSIDE_THE_BURST = { + "VersionMessage": "connect() READs it before the handshake (ADR-0020), and " + "the unit never volunteers it", + "GridMessage": "one per block on a recall, and a burst of them on an edit - " + "never on connect, because nothing has changed yet", + "SceneLabelMessage": "only when somebody renames a scene", + "SceneColorMessage": "only when somebody recolours a scene", +} + NOT_WARMED_BY_THE_BURST = { "identity": "the unit never announces its identity. It reaches the cache " "because connect() READs Version before the handshake and the " @@ -296,14 +328,29 @@ def test_burst_tail_names_every_entry_the_burst_is_expected_to_warm( """ tail = set(recorder_class.BURST_TAIL) for entry in entries.ENTRIES: - fed_by_the_burst = {t.__name__ for t in entry.feeds} & tail + # PER FEED TYPE, not per entry. An entry with one foot in BURST_TAIL and + # one in a message that arrives after the group would pass an + # any-of-its-feeds check while re-opening the race exactly as before: + # the recorder settles on the tail, the snapshot is taken, and the other + # message is still in flight. + for message_type in sorted(entry.feeds, key=lambda t: t.__name__): + name = message_type.__name__ + if name in tail: + continue + excuse = OUTSIDE_THE_BURST.get(name) or NOT_WARMED_BY_THE_BURST.get( + entry.name) + assert excuse, ( + f"entry {entry.name!r} is fed by {name}, which is neither in " + f"BURST_TAIL nor excused. If the burst carries it, it belongs " + f"in BURST_TAIL so the recorder waits for it; if it arrives at " + f"some other time, say when in OUTSIDE_THE_BURST; if the burst " + f"does not warm this entry at all, say so in " + f"NOT_WARMED_BY_THE_BURST.") + warmed_by = {t.__name__ for t in entry.feeds} & tail excused = NOT_WARMED_BY_THE_BURST.get(entry.name) - assert bool(fed_by_the_burst) != bool(excused), ( - f"entry {entry.name!r} is fed by {sorted(fed_by_the_burst)} and " - f"excused with {excused!r} - it must be one or the other. If the " - f"burst warms it, the message that does so belongs in BURST_TAIL " - f"so the recorder waits for it; if it does not, say why in " - f"NOT_WARMED_BY_THE_BURST.") + assert bool(warmed_by) != bool(excused), ( + f"entry {entry.name!r} is warmed by {sorted(warmed_by)} and excused " + f"with {excused!r} - it must be one or the other.") def test_every_burst_tail_message_actually_feeds_an_entry(recorder_class): @@ -343,3 +390,111 @@ def test_record_until_refuses_an_empty_collection(recorder_class): burst.record_until((), patience=5.0) assert burst.settled_in is None, "it settled on an empty condition" assert transport.listeners == [], "a refused call left the recorder attached" + + +def test_the_burst_excuse_lists_name_nothing_that_has_gone_away(recorder_class): + """A stale excuse is an entry nobody is checking any more. + + Both lists above are opt-outs, so a key that no longer matches anything is + an exemption still being granted to something renamed or deleted - which is + how a list like this stops being the gate it was written as. + """ + assert set(NOT_WARMED_BY_THE_BURST) <= {e.name for e in entries.ENTRIES}, ( + f"NOT_WARMED_BY_THE_BURST excuses entries that no longer exist: " + f"{sorted(set(NOT_WARMED_BY_THE_BURST) - {e.name for e in entries.ENTRIES})}") + fed = {t.__name__ for entry in entries.ENTRIES for t in entry.feeds} + assert set(OUTSIDE_THE_BURST) <= fed, ( + f"OUTSIDE_THE_BURST names message types that feed nothing: " + f"{sorted(set(OUTSIDE_THE_BURST) - fed)}") + + +def test_a_tail_that_lands_during_the_last_sleep_still_settles(recorder_class): + """Patience expiring is a claim about the unit, so it must not be a race. + + The loop used to test its deadline before its condition, so a tail arriving + in the final 100 ms was recorded in full and then reported as a burst that + never finished - with a blank where the missing names go, because by then + nothing was missing. The whole point of the sentence is to be read. + """ + transport = FakeTransport() + burst = recorder_class() + burst.attach(transport) + + def push_the_tail(): + time.sleep(0.25) # after the last poll, before the deadline + for message in _tail(): + burst(message) + + feeder = threading.Thread(target=push_the_tail) + feeder.start() + burst.record_until(recorder_class.BURST_TAIL, patience=0.3) + feeder.join() + + assert burst.settled_in is not None, ( + f"a complete recording was reported as cut off: {burst.unfinished()}") + assert burst.unfinished() is None + assert set(burst.names()) == set(recorder_class.BURST_TAIL) + + +def test_an_unfinished_recording_never_names_nothing(recorder_class): + """The blank the check above prevents, asserted directly. + + A sentence that says "- never arrived" sends the reader looking for a + recorder bug the recorder's own state has already disproved. + """ + transport = FakeTransport() + burst = recorder_class() + burst.attach(transport) + first, *rest = _tail() + burst(first) + + burst.record_until(recorder_class.BURST_TAIL, patience=0.15) + for message in rest: # arrives after the give-up, as on a slow unit + burst(message) + + unfinished = burst.unfinished() + assert "never arrived" in unfinished + named = unfinished.split("s: ")[1].split(" never arrived")[0] + assert named.strip(), f"it named nothing missing: {unfinished!r}" + assert "SceneMessage" in named, unfinished + + +def test_record_until_refuses_message_classes(recorder_class): + """Names, not classes. A class is neither a str nor empty, so it passes both + other refusals, matches nothing the recorder stores, and comes back as a + message the unit supposedly never sent.""" + transport = FakeTransport() + burst = recorder_class() + burst.attach(transport) + with pytest.raises(TypeError, match="NAMES"): + burst.record_until({pa.PresetDirtyMessage}, patience=5.0) + assert transport.listeners == [] + + +def test_a_refused_call_does_not_claim_it_was_never_made(recorder_class): + """"record_until was not called" is a different fault from "it was refused", + and the caller who catches the refusal is the one reading this.""" + burst = recorder_class() + burst.attach(FakeTransport()) + with pytest.raises(ValueError): + burst.record_until((), patience=5.0) + assert "refused its argument" in burst.unfinished() + + +def test_tail_positions_says_where_the_closing_group_landed(recorder_class): + """The signal for the thing the stop condition cannot prove: that the four + are the LAST four. A fifth closing message would push them back from the + end, and this is what would show it.""" + transport = FakeTransport() + burst = recorder_class() + burst.attach(transport) + for _ in range(5): + burst(_file_push()) + for message in _tail(): + burst(message) + burst.record_until(recorder_class.BURST_TAIL, patience=1.0) + + # Counted from the end: Scene arrived last, so it sits at 1. + assert burst.tail_positions() == { + "RecallPresetMessage": 4, "SetlistPositionMessage": 3, + "PresetDirtyMessage": 2, "SceneMessage": 1} From 89aa01cefe1bab4548ee1193c69b578fede8073f Mon Sep 17 00:00:00 2001 From: Jonathan Stokes Date: Mon, 14 Sep 2026 16:47:44 -0500 Subject: [PATCH 4/4] fix: re-triage findings on the fix batch The fix batch reopened the blank sentence it was written to close, through a different door. `_called` is set first and the give-up flag last, so anything that leaves record_until in between - a KeyboardInterrupt during the thirty second wait is how - left a recorder that had neither settled nor timed out and fell into the give-up branch, which named nothing and told the reader the unit was at fault. That state has its own branch now, and pytest prints the string on an interrupted run, so it is read by the person who just pressed Ctrl-C. The give-up froze the set read before the deadline test, so the RX thread could land the last message in between and the report would name something the recording holds. It re-reads at the deadline, and that one read decides both branches: empty settles, non-empty is frozen and reported. The type scan iterated `sentinels` before the frozenset was built, so a generator argument arrived empty at the next check and was refused for being empty when it was not. The rate was quoted three ways and one of them was inverted: "it cost two hardware tests nine runs in ten" says they failed nine times in ten, where the measurement says they passed. Worse, the two simulations of it disagree - 6.7% and 10.0% here, 5.7% and 6.7% from an independent run, differing only in how the 3.6 to 6.0 ms spacing is modelled. So the digits are not a finding and the repo no longer prints them: what is observed (one real failure on 2026-09-11, green on re-run) is now written as observed, what is simulated is written as simulated, and every copy says "a few runs in a hundred". Also: the comment block describing NOT_WARMED_BY_THE_BURST had ended up heading OUTSIDE_THE_BURST, where it stated a rule this branch had already replaced; the XOR assertion's message lost the half that says which list to edit, which is the half somebody hits; a test docstring claimed to assert the blank sentence and asserts the frozen reason instead; the fixture docstring still listed the recorder before the state layer, one commit after that order was reversed and called load-bearing; and two places disagreed about whether the burst wait costs 8 or 9 seconds. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 2 +- docs/STEERING.md | 4 +- docs/protocol.md | 21 ++++--- tests/hardware/conftest.py | 46 +++++++++++--- tests/hardware/readme.md | 4 +- tests/test_handshake_burst_recorder.py | 87 ++++++++++++++++++++++---- 6 files changed, 128 insertions(+), 36 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index b81a521..f2a22cb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,7 +7,7 @@ Read `docs/STEERING.md` before non-trivial work (new operations, transport or fr ## Conventions -- Dev setup: `uv venv && uv pip install -e ".[dev]"` (or plain venv + pip, see contributing.md). Run tests with `.venv/bin/python -m pytest`. The suite passes offline - no hardware, no `hid` import, no `DYLD_LIBRARY_PATH`. The hardware suite's `--hardware` gate is two hooks in `tests/hardware/conftest.py` and stays two: pytest skips `pytest_ignore_collect` for a path named on the command line, so `pytest_collection_modifyitems` refuses that one loudly. Never fold them into one; `tests/test_hardware_gate.py` proves both. A module in `tests/hardware/` also needs a basename no module under `tests/` owns - pytest maps `tests/hardware/test_scales.py` and `tests/test_scales.py` to one module name and refuses the second, which is why two files there end `_on_unit`. Rename; do not reach for `__init__.py` or `consider_namespace_packages`, because three offline modules do `from waiting import ...` and that works only while pytest puts `tests/` on `sys.path`. A hardware test names what it verifies AND actually asserts on with `@pytest.mark.verifies(*operations)`, checked against `QuadCortex.operations()` at collection; `--verifies NAME` narrows a run to the tests naming it, and refuses a name that is not an operation or that no collected test names. `--profile CLASSNAME` (with `--hardware`) connects as that profile class instead of the one the unit resolves to, which is how a unit the registry would refuse - an unmeasured firmware, or a Mini - gets measured by the suite that measures it. `scratch_preset` is the fixture for getting a disposable copy of the loaded preset to edit. The connect burst is waited for as a GROUP: `HandshakeBurst.BURST_TAIL` names the four messages that close it, and the fixture registers the model cache before the recorder so that "the recorder has all four" means the cache has applied them. Never wait on one message of that group - `RecallPreset` is its FIRST member and the other three trail it by 3.6-6.0 ms against a 100 ms poll, which cost two hardware tests one run in ten. A new `StateEntry` comes through `BURST_TAIL`, `OUTSIDE_THE_BURST` or `NOT_WARMED_BY_THE_BURST` in `tests/test_handshake_burst_recorder.py`, with its reason, and there is no fourth way. An operation no hardware test names has to appear in `tests/test_hardware_markers.py`'s `UNMARKED_OPERATIONS`, with the reason - that dict is the list a new operation must come through if no hardware test names it. +- Dev setup: `uv venv && uv pip install -e ".[dev]"` (or plain venv + pip, see contributing.md). Run tests with `.venv/bin/python -m pytest`. The suite passes offline - no hardware, no `hid` import, no `DYLD_LIBRARY_PATH`. The hardware suite's `--hardware` gate is two hooks in `tests/hardware/conftest.py` and stays two: pytest skips `pytest_ignore_collect` for a path named on the command line, so `pytest_collection_modifyitems` refuses that one loudly. Never fold them into one; `tests/test_hardware_gate.py` proves both. A module in `tests/hardware/` also needs a basename no module under `tests/` owns - pytest maps `tests/hardware/test_scales.py` and `tests/test_scales.py` to one module name and refuses the second, which is why two files there end `_on_unit`. Rename; do not reach for `__init__.py` or `consider_namespace_packages`, because three offline modules do `from waiting import ...` and that works only while pytest puts `tests/` on `sys.path`. A hardware test names what it verifies AND actually asserts on with `@pytest.mark.verifies(*operations)`, checked against `QuadCortex.operations()` at collection; `--verifies NAME` narrows a run to the tests naming it, and refuses a name that is not an operation or that no collected test names. `--profile CLASSNAME` (with `--hardware`) connects as that profile class instead of the one the unit resolves to, which is how a unit the registry would refuse - an unmeasured firmware, or a Mini - gets measured by the suite that measures it. `scratch_preset` is the fixture for getting a disposable copy of the loaded preset to edit. The connect burst is waited for as a GROUP: `HandshakeBurst.BURST_TAIL` names the four messages that close it, and the fixture registers the model cache before the recorder so that "the recorder has all four" means the cache has applied them. Never wait on one message of that group - `RecallPreset` is its FIRST member and the other three trail it by 3.6-6.0 ms against a 100 ms poll, which cost two hardware tests a few runs in a hundred. A new `StateEntry` comes through `BURST_TAIL`, `OUTSIDE_THE_BURST` or `NOT_WARMED_BY_THE_BURST` in `tests/test_handshake_burst_recorder.py`, with its reason, and there is no fourth way. An operation no hardware test names has to appear in `tests/test_hardware_markers.py`'s `UNMARKED_OPERATIONS`, with the reason - that dict is the list a new operation must come through if no hardware test names it. - Two namespaces, one package (ADR-0006): `pyquadcortex` is the model of the unit, `pyquadcortex.protocol` is the message-level API. The model's code lives in `pyquadcortex/device/` - not `model/`, because in this codebase the identifier `model` means an amp or pedal block (`protocol/models.py`, `catalog.Model`, `ModelCatalog`, `set_block(model=...)`). The model imports the protocol layer; nothing under `pyquadcortex/protocol/` may import from `pyquadcortex/device/`. - Every conversion between a screen value and a wire value lives in the `pyquadcortex/device/translate/` PACKAGE and nowhere else in the package outside `pyquadcortex/protocol/` - the whole package, not just `device/`, because a rule scoped to a directory is satisfied by moving the code one directory up. It covers rows 1-4, slots 1-8, scene and footswitch letters, preset addresses and display units. Outside the boundary that means no `+1`/`-1` on a coordinate AND none of the other spellings of the same conversion (`ord`/`chr`, a letter table in any container, `divmod` on a position, a one-based `enumerate`, `ROWS.index(...)`), and no module reaching past the boundary for a protocol-layer name that carries a coordinate or a raw scale - the converters and also the readers that hand back wire indexes, such as `protocol.stomp_assignments`. `tests/test_translation.py` reads the source and proves both, and pins where each check stops seeing rather than implying it sees everything. The boundary is a package, so its exemption covers a DIRECTORY: `BOUNDARY_MODULES` in that file names the modules inside it, and adding `translate/anything.py` has to come through that list with a reason - otherwise the arithmetic scan skips the new file for the same reason it skips the real converters. The same file now carries a second list, `PROTOCOL_NON_CONVERSIONS`, for protocol-layer names the boundary uses that are NOT conversions (`field_present`, the port enums). It exists because the boundary reads whole presets now, so "everything it reaches for is a conversion" stopped being true; a name has to be on one list or the other, and which one is a judgement a reviewer makes from the reason written beside it. A model API takes `FootswitchLetter`, never a bare footswitch integer, because a footswitch index and a block's column are different numbers that usually agree. A new conversion goes in that module with its own test, however small it is, and its protocol-layer name joins that file's allowlist in the same commit. - The model represents what the unit shows, in the unit's own words, and never guesses. A control we understand but cannot yet drive is modelled and REFUSES the operation (ADR-0007); a control we do not understand is omitted, with the reason recorded in `docs/domain-model.md`'s appendix. Nothing ships with a "this might be stale or wrong" caveat. A refusal is `protocol.ControlNotDrivable(control, evidence, workaround)` - never a bare `ValueError` and never a message-only raise. All three fields are required, and `evidence` records what was actually tried, because a refusal nobody can audit is the guess the rule exists to prevent. Before writing one, ADR-0010 applies: the differential capture comes first, and "the unit broadcasts nothing" is not on its own a finding about the wire. Do not derive a refusal from a RULE about parameter types either - the first instance had three plausible rules tried against hardware and all three were false, so `LANE_OUTPUT_UNASSIGNABLE` is a measured list with the disproofs written beside it. diff --git a/docs/STEERING.md b/docs/STEERING.md index f56a296..fcedea2 100644 --- a/docs/STEERING.md +++ b/docs/STEERING.md @@ -153,8 +153,8 @@ as `UNMARKED_OPERATIONS` and `BOUNDARY_MODULES`. **Why:** `RecallPreset` is the FIRST of those four on the wire and the other three follow within 3.6 to 6.0 ms, against a 100 ms poll. Two hardware tests failed -together on 2026-09-11 and passed on a re-run of the same commit; measured at -6.7% and 10.0% of runs. A gate that fails intermittently for reasons unrelated to +together on 2026-09-11 and passed on a re-run of the same commit; simulating the +poll's phase puts it at a few runs in a hundred. A gate that fails intermittently for reasons unrelated to the change teaches people to re-run until green, which is how a real regression gets waved through. diff --git a/docs/protocol.md b/docs/protocol.md index c5de100..c102615 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -2106,16 +2106,19 @@ the ORDER and the SPREAD that are stable enough to build on, not the moment. treating it as the signal that the burst is over: the other three are still a few milliseconds out. The hardware suite's burst recorder did exactly that, and it polls at 100 ms, so its stop landed at a uniformly random point in the 100 ms after `RecallPreset` -and fell inside the group about one time in fifteen. - -Measured rather than derived, by driving the real recorder with the four messages at their -measured spacing and the poll's phase drawn uniformly from its 100 ms interval - which is -what a connection does to it - over 300 runs: it lost `PresetDirty` 20 times (6.7%) and -`Scene` 30 times (10.0%). `Scene` is lost more often because it arrives last, and losing -`Scene` alone is already enough to fail a test. After the change, 0 of 300 for both. The +and falls inside the group a few times in a hundred. `Scene` is lost most often, because it +arrives last - and losing `Scene` alone is already enough to fail a test. + +Two kinds of evidence, and they are worth keeping apart. **Observed:** it failed a real run +on 2026-09-11 and passed on a re-run of the same commit. **Simulated:** driving the real +recorder with the four messages at their measured spacing and the poll's phase drawn +uniformly from its 100 ms interval - which is what a connection does to it - loses one or +more of the three over 300 runs, and 0 of 300 after the change. Two independent runs of +that simulation put the rate between 6% and 10% per message, differing by how the 3.6-6.0 ms +spacing is modelled, so the order of magnitude is the finding and the digits are not. The recorder now waits for all four (`BURST_TAIL` in `tests/hardware/conftest.py`), and -`tests/test_handshake_burst_recorder.py` holds the mechanism offline. Wait for the group, -not for its head. +`tests/test_handshake_burst_recorder.py` holds the mechanism offline with a gap staged wide +enough that it needs no simulation. Wait for the group, not for its head. **`read_preset()` RECALLS the slot it reads** - that was already documented - and the recall **resets the active scene to the preset's default, discards unsaved edits, and diff --git a/tests/hardware/conftest.py b/tests/hardware/conftest.py index 8c69c78..48c7652 100644 --- a/tests/hardware/conftest.py +++ b/tests/hardware/conftest.py @@ -220,7 +220,8 @@ class HandshakeBurst: #: 11.1 s inside 3.6, 5.8 and 6.0 ms respectively, always in this order #: (``docs/protocol.md``, "Connect burst, measured"). ``RecallPreset`` is #: the FIRST of them. Waiting for that one alone is what this used to do, - #: and it cost two hardware tests about nine runs in ten - see + #: and it cost two hardware tests a few runs in a hundred - most often by + #: losing ``Scene``, which arrives last. See #: ``tests/test_handshake_burst_recorder.py``. #: #: Measured on ONE profile. ADR-0020 puts what differs by firmware or model @@ -249,6 +250,7 @@ def __init__(self): self._missing_at_giveup = frozenset() self._patience = None self._called = False + self._gave_up = False self.closed = False self.settled_in = None # seconds the burst took, or None if it timed out @@ -282,8 +284,8 @@ def record_until(self, sentinels, patience): settings at 5.1 s, then those four together at about 11 s, and they are a group six milliseconds wide rather than a sequence. This loop polls at 100 ms, so a stop condition naming only the first of the group stops at - a uniformly random point in the 100 ms after it - inside the group about - one time in fifteen. When that happened the recorder came off the + a uniformly random point in the 100 ms after it, and lands inside the + group a few times in a hundred. When that happened the recorder came off the transport and the fixture snapshotted ``burst_warmed`` before the other three reached the cache, and the two tests reading that snapshot failed together while every test reading the live cache passed. @@ -307,7 +309,12 @@ def record_until(self, sentinels, patience): "str is a collection of its letters, so this would " "otherwise wait for message types called 'R', 'e' and 'c' " "and never settle.") - wrong = sorted(repr(s) for s in sentinels if not isinstance(s, str)) + # Materialized BEFORE the scan below: `sentinels` may be a + # generator, and scanning it first would leave frozenset() an + # exhausted one - refused as empty, for a non-empty argument. + self._sentinels = frozenset(sentinels) + wrong = sorted(repr(s) for s in self._sentinels + if not isinstance(s, str)) if wrong: raise TypeError( f"record_until matches NAMES against what the recorder " @@ -315,7 +322,6 @@ def record_until(self, sentinels, patience): f"{', '.join(wrong)}. A message class never equals its own " f"name, so the wait would burn the whole patience and then " f"name that class as something the unit never sent.") - self._sentinels = frozenset(sentinels) if not self._sentinels: raise ValueError( "record_until needs at least one type name. An empty " @@ -339,7 +345,16 @@ def record_until(self, sentinels, patience): self.settled_in = time.monotonic() - started break if time.monotonic() >= deadline: - self._missing_at_giveup = outstanding + # One last look, and this read decides BOTH branches. The + # RX thread can have emptied it since the read above, and + # freezing that stale set would name a message the recording + # holds. Whatever this read says is what gets reported. + outstanding = self.missing() + if not outstanding: + self.settled_in = time.monotonic() - started + else: + self._missing_at_giveup = outstanding + self._gave_up = True break time.sleep(0.1) finally: @@ -376,6 +391,16 @@ def unfinished(self): if not self._sentinels: return ("the burst was never recorded: record_until was called and " "refused its argument - see the error it raised") + if not self._gave_up: + # Neither settled nor timed out, so the wait did not finish: a + # KeyboardInterrupt during the burst is the way this happens. Said + # plainly, because the branch below would otherwise name nothing at + # all and blame the unit for it. + return (f"the wait for the connect burst did not run to an end - " + f"interrupted, most likely. The recording holds what had " + f"arrived by then and is still short of " + f"{', '.join(sorted(self.missing())) or 'nothing'}; it says " + f"nothing about the unit either way.") return ( f"the connect burst did not finish within {self._patience}s: " f"{', '.join(sorted(self._missing_at_giveup))} never arrived, of " @@ -465,17 +490,18 @@ def _connection(request): Two things are attached before the handshake, and neither can be attached later on demand, because the burst happens during ``connect``: - * the burst recorder, for every run rather than only the tests that read it; - * the model's state layer, which is what ``pyquadcortex.connect()`` does at + * the model's state layer FIRST - see ``subscribe`` below, where the order + is load-bearing - which is what ``pyquadcortex.connect()`` does at exactly this point. It stays attached for the whole run, which costs the RX thread one small message copy per ``Version`` or ``PresetDirty`` push and nothing at all for anything else - orders of magnitude under the hundred-millisecond latencies ``test_write_echo.py`` measures. Its own - tests are in ``test_model_state.py``. + tests are in ``test_model_state.py``; + * the burst recorder, for every run rather than only the tests that read it. The fixture then waits for the burst to finish before handing the connection over, so the recording is exactly the burst whatever order the tests run in. - It costs about 8 s once per run and buys more than it costs: `connect()` + It costs about 9 s once per run and buys more than it costs: `connect()` returns roughly 3 s before the unit starts streaming several hundred messages, so without the wait every latency measurement in this suite would be taken on a link that is still busy answering the handshake. diff --git a/tests/hardware/readme.md b/tests/hardware/readme.md index 20ce00b..be73f26 100644 --- a/tests/hardware/readme.md +++ b/tests/hardware/readme.md @@ -120,8 +120,8 @@ burst is named even on a run that deselects all three tests that guard on it. What they cannot see is a recorder that sets its flag and keeps recording anyway, one that stops recording but stays attached to the transport, or one that stops on the FIRST message of the burst's closing group instead of the whole of it. -All three read like a working recorder from the outside - the last of them about -nine runs in ten - so all three are pinned offline in +All three read like a working recorder from the outside - the last of them on +all but a few runs in a hundred - so all three are pinned offline in `tests/test_handshake_burst_recorder.py`. ## The model's cache rides the same connection diff --git a/tests/test_handshake_burst_recorder.py b/tests/test_handshake_burst_recorder.py index 33d7184..aeccbdd 100644 --- a/tests/test_handshake_burst_recorder.py +++ b/tests/test_handshake_burst_recorder.py @@ -17,8 +17,9 @@ that sets its flag and keeps recording anyway, one that stops recording but stays on the transport, or one that stops on the FIRST message of the burst's closing group rather than the whole of it, reads exactly like a working one - from the outside - the last of those about NINE runs in TEN, because losing - ``Scene`` alone is enough and ``Scene`` is the one lost most often. + from the outside - the last of those on all but a few runs in a hundred, + because losing ``Scene`` alone is enough and ``Scene`` is the one lost most + often. The two hardware tests that read ``burst_warmed`` for entries the burst delivers call ``unfinished()`` before their own assertions. The identity test does not, @@ -189,7 +190,6 @@ def push(): known = {"FileMessage"} | set(recorder_class.BURST_TAIL) while any(writer.is_alive() for writer in writers): assert all(name in known for name in burst.names()) - assert burst.missing() <= frozenset() # no sentinels set: nothing outstanding for writer in writers: writer.join() @@ -199,7 +199,7 @@ def push(): def test_record_until_waits_for_the_whole_tail_not_just_the_first_of_it( recorder_class): - """The race that failed two hardware tests together, one run in fifteen. + """The race that failed two hardware tests together, a few runs in a hundred. The burst's four closing messages arrive in the order ``RecallPreset``, ``SetlistPosition``, ``PresetDirty``, ``Scene``, inside six milliseconds - @@ -287,9 +287,6 @@ def test_record_until_refuses_a_single_name(recorder_class): "record the whole session with nothing left to stop it") -#: Cache entries the connect burst does NOT warm, and why. A new ``StateEntry`` -#: comes through here or through ``BURST_TAIL``, and there is no third way - -#: which is the point of the test below. #: Feed types that reach an entry OUTSIDE the connect burst, with when they do. #: The recorder must not wait for these: nothing sends them on connect, so a #: wait would cost the full patience on every run. @@ -302,6 +299,9 @@ def test_record_until_refuses_a_single_name(recorder_class): "SceneColorMessage": "only when somebody recolours a scene", } +#: Cache entries the connect burst does not warm AT ALL, and why. An entry whose +#: feeds are all in :data:`OUTSIDE_THE_BURST` belongs here too - the two lists +#: answer different questions, and the test below wants both answers. NOT_WARMED_BY_THE_BURST = { "identity": "the unit never announces its identity. It reaches the cache " "because connect() READs Version before the handshake and the " @@ -350,7 +350,13 @@ def test_burst_tail_names_every_entry_the_burst_is_expected_to_warm( excused = NOT_WARMED_BY_THE_BURST.get(entry.name) assert bool(warmed_by) != bool(excused), ( f"entry {entry.name!r} is warmed by {sorted(warmed_by)} and excused " - f"with {excused!r} - it must be one or the other.") + f"with {excused!r} - it must be exactly one of the two. An entry " + f"fed only by messages in OUTSIDE_THE_BURST is not warmed by the " + f"burst, so it needs a line in NOT_WARMED_BY_THE_BURST as well: " + f"that list answers 'is this entry in the snapshot at all', and " + f"OUTSIDE_THE_BURST answers 'must the recorder wait for this " + f"message'. If the burst DOES warm it, put the message that does " + f"so in BURST_TAIL instead.") def test_every_burst_tail_message_actually_feeds_an_entry(recorder_class): @@ -436,11 +442,15 @@ def push_the_tail(): assert set(burst.names()) == set(recorder_class.BURST_TAIL) -def test_an_unfinished_recording_never_names_nothing(recorder_class): - """The blank the check above prevents, asserted directly. +def test_an_unfinished_recording_names_what_was_outstanding(recorder_class): + """A give-up reports the set that was outstanding when it gave up. - A sentence that says "- never arrived" sends the reader looking for a - recorder bug the recorder's own state has already disproved. + Not the same claim as the test above, and weaker than it looks on its own: + messages pushed after ``record_until`` returns are dropped, so a live + ``missing()`` would name these too. What it holds is that the give-up + FREEZES its reason rather than recomputing it later - delete the + ``_missing_at_giveup`` assignment and this fails. The blank sentence itself + is prevented by the loop's last-look, which the test above covers. """ transport = FakeTransport() burst = recorder_class() @@ -498,3 +508,56 @@ def test_tail_positions_says_where_the_closing_group_landed(recorder_class): assert burst.tail_positions() == { "RecallPresetMessage": 4, "SetlistPositionMessage": 3, "PresetDirtyMessage": 2, "SceneMessage": 1} + + +def test_a_wait_that_never_ran_to_an_end_says_so_rather_than_blaming_the_unit( + recorder_class): + """Ctrl-C during the thirty-second burst wait, in the shape it reaches here. + + ``record_until`` sets its "called" flag first and its "gave up" flag last, + so an exception in between leaves a recorder that has neither settled nor + timed out. Without a branch of its own that state falls into the give-up + sentence, which names whatever is outstanding - nothing, by then, in the + interrupt case - and tells the reader the unit is at fault. The run report + prints this string, so it is read by someone who has just pressed Ctrl-C. + """ + transport = FakeTransport() + burst = recorder_class() + burst.attach(transport) + + class Interrupt(Exception): + pass + + def interrupting_sleep(_seconds): + raise Interrupt + + real_sleep, time.sleep = time.sleep, interrupting_sleep + try: + with pytest.raises(Interrupt): + burst.record_until(recorder_class.BURST_TAIL, patience=30.0) + finally: + time.sleep = real_sleep + + assert burst.settled_in is None + assert burst.closed, "an interrupted wait left the recorder on the transport" + unfinished = burst.unfinished() + assert "interrupted" in unfinished, unfinished + assert "nothing about the unit" in unfinished, unfinished + + +def test_record_until_takes_a_generator(recorder_class): + """A one-pass argument must not be spent by the type check before the wait. + + Scanning `sentinels` and then building the frozenset from it drains a + generator, and the empty-collection refusal then fires on an argument that + was not empty - a refusal naming the opposite of what went wrong. + """ + transport = FakeTransport() + burst = recorder_class() + burst.attach(transport) + for message in _tail(): + burst(message) + + burst.record_until((name for name in recorder_class.BURST_TAIL), patience=1.0) + + assert burst.settled_in is not None, burst.unfinished()