From 25a7126f302480315b02bb044bede3169abc7eeb Mon Sep 17 00:00:00 2001 From: Antoni Ivanov <262547018+tony-xmelon@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:54:24 +0300 Subject: [PATCH 1/5] Harden live preset reads --- changelog.md | 9 ++++++ docs/protocol.md | 19 +++++++----- pyquadcortex/device/entries.py | 55 ++++++++++++++++++++------------- pyquadcortex/device/state.py | 14 ++++++--- pyquadcortex/protocol/client.py | 41 +++++++++++++++++------- tests/test_client.py | 39 +++++++++++++++++++++++ tests/test_state.py | 33 ++++++++++++++++++++ 7 files changed, 165 insertions(+), 45 deletions(-) diff --git a/changelog.md b/changelog.md index d8b2b27f..9f8b3a52 100644 --- a/changelog.md +++ b/changelog.md @@ -20,6 +20,15 @@ correction. ## Unreleased +### Live-preset reads tolerate the device's lazy two-message reply + +`read_current_preset()` and `read_current_preset_push()` now make a second +attempt when the first keyed reply times out. CorOS 4.1.0 sends an uncorrelated +current-preset update before the request-id-correlated answer; the state cache +models both as one normal read and no longer schedules an unnecessary reread. +Version messages that only answer the Cortex Control compatibility announcement +are likewise kept out of the device-identity cache. + ### The first hardware run of the profile seam, and what it corrected Run 2026-09-07 on Quad Cortex, CorOS 4.0.1 / d14e, straight after the seam diff --git a/docs/protocol.md b/docs/protocol.md index cbe3b806..ffe6b2d2 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -704,9 +704,14 @@ the preset (next section). ### 7.2 Read the current preset -**There is no host-initiated "read preset" request.** A `Grid` or `RecallPreset` -READ gets no reply. Instead, whenever a preset is recalled, by the host or on the -unit, the device **broadcasts**: +`RecallPreset{READ, request_id}` reads the live grid without recalling a stored +slot or discarding edits. On CorOS 4.1.0 one request produces two broadcasts: +first an uncorrelated current-state `UPDATE`, then an otherwise equivalent +`UPDATE` echoing the request id. A client must wait for the keyed second message; +the first is not evidence that some other preset was loaded. + +Whenever a preset is recalled, by the host or on the unit, the device also +**broadcasts**: ``` RecallPreset{action: UPDATE, preset: , reason: } @@ -718,10 +723,10 @@ is the only way to obtain the full current preset. Consequences for a client: -- reading a preset **is not side-effect free**: it recalls the slot, which loads - it onto the grid; -- the device services the push lazily; 10 to 25 seconds has been observed, so - timeouts must be generous (the library defaults to 40 seconds); +- reading the live preset is side-effect free; reading a stored preset still + recalls that slot and is a different operation; +- the first request after connecting can be dropped, so the library makes two + attempts of 15 seconds each before reporting a timeout; - the push must be correlated by `request_id` to avoid returning a stale or seed push (see [section 5](#5-requestresponse-correlation)). diff --git a/pyquadcortex/device/entries.py b/pyquadcortex/device/entries.py index afa037f0..866b493f 100644 --- a/pyquadcortex/device/entries.py +++ b/pyquadcortex/device/entries.py @@ -84,11 +84,15 @@ class FieldPlan: either has to decide for itself, and this flag is that decision written down. It does not widen :data:`SCAFFOLDING`, and it is set per entry and per message type rather than globally. + accepts: optional predicate for message types reused by independent + conversations. A rejected message says nothing about this entry; + it is neither applied nor counted as an arrival. """ kept: frozenset = frozenset() no_presence: frozenset = frozenset() invalidates: bool = False + accepts: typing.Callable[[Message], bool] | None = None def voids_the_copy(self) -> bool: """Whether a message of this type makes the entry untrusted on its own. @@ -99,6 +103,10 @@ def voids_the_copy(self) -> bool: """ return self.invalidates + def applies_to(self, message: Message) -> bool: + """Whether this same-type message belongs to the entry's conversation.""" + return self.accepts is None or self.accepts(message) + @dataclasses.dataclass(frozen=True, eq=False) class StateEntry: @@ -110,25 +118,21 @@ class StateEntry: the unit's whole answer for this entry. Runs on the CALLER's thread, never the RX thread. feeds: message class -> :class:`FieldPlan`. - - Every entry's :attr:`read` is one request and one ANSWER, which the read - path relies on to tell its own answer apart from a push that arrived while - it was waiting. One answer is not the same as one message: a ``Version`` - READ is answered by the unit's reply and then by a question of the unit's - own, and the read path survives that because a message that said nothing is - not counted. Said nothing means it applied no field this entry keeps AND - named none it does not - a plan with :attr:`FieldPlan.invalidates` set is - never in that case, because every message of its type makes the copy - untrusted whatever it carried. ``device/state.py``'s ``_apply_one`` decides - it. An entry whose read provokes a STREAM OF ANSWERS instead - a ``File`` - enumeration, a preset dump - has to say how many messages that is, and this - class does not carry that yet because nothing needs it. It lands with the - first such entry, along with the test that a number other than one works. + read_arrivals: messages that say something to this entry during one + normal read. Most reads have one. A live-preset read produces two + on CorOS 4.1.0, so the first must not be mistaken for an unrelated + push that raced the request. + + The read path compares meaningful arrivals with :attr:`read_arrivals`. + A symmetric protocol question carrying only scaffolding does not count. A + larger count means an external push landed during the read and its cache + mark must survive. """ name: str read: typing.Callable feeds: typing.Mapping + read_arrivals: int = 1 #: Entries whose copies stop being true when THIS entry's value moves. #: Applied only on a real change, which is what makes it different from #: listing the same message type on each of them: the model's own READ of @@ -269,9 +273,20 @@ def _carries_unknown_fields(message) -> bool: #: fields followed 0.5-0.8 ms later by a ``Version{READ}`` carrying ``action`` #: alone. The question is not news about the unit and the cache does not count #: it - see ``_apply_one`` in ``device/state.py``, where counting it cost a -#: second round trip whenever it landed before the reading thread woke. +#: second round trip whenever it landed before the reading thread woke. The +#: device's answer to the host's Cortex Control announcement is also a Version, +#: but it carries only handshake validity and likewise says nothing about the +#: unit's identity. +def _version_describes_device(message) -> bool: + """Exclude the two Version shapes that are only handshake traffic.""" + named = {field.name for field, _ in message.ListFields()} + host_handshake = SCAFFOLDING | {"cortex_control_version_valid"} + return bool(named - host_handshake) or _carries_unknown_fields(message) + + _VERSION_FOR_IDENTITY = FieldPlan( kept=frozenset({"app_fw_version", "device_serial_number"}), + accepts=_version_describes_device, ) @@ -466,6 +481,7 @@ def _read_preset(client) -> dict: pa.SceneLabelMessage: _SCENE_TEXT_CHANGED, pa.SceneColorMessage: _SCENE_TEXT_CHANGED, }, + read_arrivals=2, ) @@ -500,13 +516,8 @@ def _read_scene(client) -> dict: #: a plan, not a fact, and every push mentioning a field it did not keep would #: mark it for a read nobody had asked for. #: -#: The Directory's rows are the ones that need something this class does not -#: have. Every read here is one request and one answer, which is how the read path -#: tells its own answer apart from a push that arrived while it was waiting. A -#: setlist listing is a STREAM - one `File` READ makes the unit enumerate its -#: whole tree, several hundred messages over about fifteen seconds - so those -#: entries land with the change to `StateEntry` that lets a read say how many -#: messages it expects. +#: Directory rows are not cache entries: one ``File`` READ streams hundreds of +#: messages and completes by convergence rather than by a fixed arrival count. ENTRIES = (IDENTITY, DIRTY, PRESET, SCENE, LOADED) ENTRY_BY_NAME = {entry.name: entry for entry in ENTRIES} diff --git a/pyquadcortex/device/state.py b/pyquadcortex/device/state.py index 11cae181..39f1dced 100644 --- a/pyquadcortex/device/state.py +++ b/pyquadcortex/device/state.py @@ -213,6 +213,8 @@ def apply_push(self, message) -> None: from this thread outright (ADR-0009). """ for entry, plan in entries.FEEDS.get(type(message), ()): + if not plan.applies_to(message): + continue try: self._apply_one(entry, plan, message) except Exception: @@ -421,8 +423,8 @@ def value(self, entry_name: str, field: str): # asks one of its own, and a question is not news about the # unit's state. `_apply_one` is where that is decided, because # the read path cannot tell the two apart after the fact. An - # entry whose read provokes a stream of ANSWERS will still have - # to say how many messages that is; see `StateEntry`. + # entry whose read normally produces more than one meaningful + # message declares that measured count on `StateEntry`. extra = slot.witnessed - witnessed_before # And the marks that came from something OTHER than a # message to this entry, which the count above cannot see @@ -437,14 +439,16 @@ def value(self, entry_name: str, field: str): # snapshot and the request going out is kept although the unit # might have accounted for it. `witnessed_before` has had the # same gap since it was written, and both err by re-reading. The obvious - # `slot.needs_read or extra > 1` does not work: the read's own + # `slot.needs_read or extra > entry.read_arrivals` does not + # work: the read's own # reply re-arms the mark through the listener from the fields # this entry does not keep, so the entry would never cache # anything again. marked_since = slot.marks - marks_before was_marked = slot.needs_read - slot.needs_read = extra > 1 or marked_since > 0 - if extra > 1: + slot.needs_read = (extra > entry.read_arrivals + or marked_since > 0) + if extra > entry.read_arrivals: log.info("push.forced_reread %s - %d message(s) arrived " "while it was being read", entry_name, extra) elif slot.needs_read: diff --git a/pyquadcortex/protocol/client.py b/pyquadcortex/protocol/client.py index 4e9e102d..8d5b7387 100644 --- a/pyquadcortex/protocol/client.py +++ b/pyquadcortex/protocol/client.py @@ -1426,7 +1426,7 @@ def preset_dirty(self, timeout: float = 5.0) -> bool: pa.PresetDirtyMessage(action=pa.MessageAction.READ), timeout=timeout) return bool(reply.is_dirty) - def read_current_preset(self, timeout: float = 15.0): + def read_current_preset(self, timeout: float = 15.0, attempts: int = 2): """The LIVE grid - the current editing state, unsaved changes included. ``RecallPreset{READ}`` answers with the preset as it exists on the device @@ -1449,10 +1449,15 @@ def read_current_preset(self, timeout: float = 15.0): every time** - even when it recalls the preset already loaded. Interleaving it with scene-targeted writes silently retargets them; use this method for inspection during editing. + + The first request after a connection can be dropped. ``attempts`` + defaults to two; ``timeout`` applies to each attempt. """ - return self.read_current_preset_push(timeout=timeout).preset + return self.read_current_preset_push( + timeout=timeout, attempts=attempts).preset - def read_current_preset_push(self, timeout: float = 15.0): + def read_current_preset_push(self, timeout: float = 15.0, + attempts: int = 2): """The whole ``RecallPreset`` reply, not just the preset inside it. Same request and same match as :meth:`read_current_preset` - this is @@ -1463,15 +1468,29 @@ def read_current_preset_push(self, timeout: float = 15.0): It exists because the reply carries ``reason`` beside the preset, and a caller tracking state needs both from one answer. Confirmed on hardware 2026-08-15: the connect burst's seed push sets ``action``, ``preset`` - and ``reason``, and so does the push a recall produces. + and ``reason``, and so does the push a recall produces. Re-measured on + CorOS 4.1.0 on 2026-09-11: a READ emits an uncorrelated copy before the + request-id-correlated answer, and its first request can be dropped. """ - request_id = self._t.next_request_id() - message = pa.RecallPresetMessage(action=pa.MessageAction.READ, - request_id=request_id) - return self._t.await_broadcast( - pa.RecallPresetMessage, lambda: self._t.send(message), timeout=timeout, - match=lambda m: (m.HasField("request_id") - and m.request_id == request_id)) + last = None + tries = max(1, attempts) + for _ in range(tries): + request_id = self._t.next_request_id() + message = pa.RecallPresetMessage(action=pa.MessageAction.READ, + request_id=request_id) + try: + return self._t.await_broadcast( + pa.RecallPresetMessage, + lambda outgoing=message: self._t.send(outgoing), + timeout=timeout, + match=lambda m, wanted=request_id: ( + m.HasField("request_id") and m.request_id == wanted)) + except TimeoutError as exc: + last = exc + raise TimeoutError( + f"the device did not answer a live-preset read in {tries} " + f"attempt(s) of {timeout}s each; the first request after connecting " + f"can be dropped") from last def loaded_position(self, timeout: float = 10.0): """Which preset slot is on the grid right now. diff --git a/tests/test_client.py b/tests/test_client.py index daa43778..c6262127 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1301,6 +1301,45 @@ def test_read_current_preset_push_hands_back_the_whole_reply(): assert asked.HasField("request_id") +def test_read_current_preset_retries_a_dropped_first_request(): + push = pa.RecallPresetMessage(action=pa.MessageAction.UPDATE, request_id=2) + push.preset.name = "live after retry" + + class DropsFirst(StateTransport): + def __init__(self, reply): + super().__init__(reply) + self.calls = 0 + + def await_broadcast(self, expected_class, trigger, timeout=40.0, + match=None): + trigger() + self.calls += 1 + if self.calls == 1: + raise TimeoutError("dropped") + self.matches.append(match) + return self.push + + transport = DropsFirst(push) + got = client.QuadCortex(transport).read_current_preset() + assert got.name == "live after retry" + assert transport.calls == 2 + assert [m.request_id for m in transport.sent] == [1, 2] + assert transport.matches[-1](push) is True + + +def test_read_current_preset_timeout_reports_all_attempts(): + class NeverAnswers(FakeTransport): + def await_broadcast(self, expected_class, trigger, timeout=40.0, + match=None): + trigger() + raise TimeoutError("nothing") + + qc = client.QuadCortex(NeverAnswers()) + with pytest.raises(TimeoutError, match=r"2 attempt\(s\) of 0.01s each"): + qc.read_current_preset(timeout=0.01) + assert [m.request_id for m in qc._t.sent] == [1, 2] + + def test_loaded_position_reads_the_slot_without_recalling_it(): """`SetlistPosition{READ}`. The same message type as a recall, and the action is the whole difference between asking and loading - so the wire diff --git a/tests/test_state.py b/tests/test_state.py index bd6a201d..f0f8fb5c 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -440,6 +440,7 @@ def test_a_grid_push_that_lands_during_a_preset_read_is_not_lost_either(link): def preset_but_a_grid_first(triggering=None): transport.push(grid_push()) + transport.push(recall_push()) return recall_push(triggering) transport.broadcasts["RecallPresetMessage"] = preset_but_a_grid_first @@ -494,6 +495,20 @@ def answers_and_then_asks_back(): assert cache.needs_read("identity") is False +def test_the_cortex_control_answer_is_not_device_identity(link): + """The Version response to our host announce carries no device fact.""" + transport, cache = link + assert cache.value("identity", "app_fw_version") == "d14e" + + transport.push(pa.VersionMessage( + action=pa.MessageAction.UPDATE, request_id=0, + cortex_control_version_valid=True)) + + assert cache.value("identity", "app_fw_version") == "d14e" + assert cache.needs_read("identity") is False + assert transport.reads["VersionMessage"] == 1 + + def test_a_recall_landing_during_a_read_of_the_dirty_flag_is_not_lost(link): """The same window, reached by a path the arrival count cannot see. @@ -683,6 +698,8 @@ def test_a_marking_push_during_any_entrys_read_survives_it(entry, link): def marks_first(canned): def pushing(*args): transport.push(marking) + if entry is entries.PRESET: + transport.push(recall_push()) return canned(*args) if callable(canned) else canned return pushing @@ -1366,6 +1383,22 @@ def test_the_preset_entry_can_read_back_every_field_it_keeps(link): assert cache.value("preset", "preset").name == "Structural Fixture" +def test_two_normal_preset_pushes_still_settle_one_read(link): + """A live-preset READ emits an uncorrelated push, then its keyed answer.""" + transport, cache = link + + def uncorrelated_then_keyed(triggering): + transport.push(recall_push()) + return recall_push(triggering) + + transport.broadcasts["RecallPresetMessage"] = uncorrelated_then_keyed + cache.mark_for_reread("preset", "exercise the measured two-push reply") + + assert cache.value("preset", "preset").name == "Structural Fixture" + assert cache.needs_read("preset") is False + assert transport.reads["RecallPresetMessage"] == 1 + + def test_a_recall_push_does_not_invalidate_the_preset_it_delivers(): plan = entries.PRESET.feeds[pa.RecallPresetMessage] assert not plan.voids_the_copy() From 66752e5968feb6e5372259f5166b8375eb991ab1 Mon Sep 17 00:00:00 2001 From: Antoni Ivanov <262547018+tony-xmelon@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:00:30 +0300 Subject: [PATCH 2/5] Scope preset reply shape to CorOS 4.1 --- pyquadcortex/device/entries.py | 15 ++++----------- pyquadcortex/device/state.py | 12 ++++++------ pyquadcortex/protocol/client.py | 5 +++++ pyquadcortex/protocol/profiles.py | 1 + tests/test_profiles.py | 1 + tests/test_state.py | 9 ++++++--- 6 files changed, 23 insertions(+), 20 deletions(-) diff --git a/pyquadcortex/device/entries.py b/pyquadcortex/device/entries.py index 866b493f..1768e615 100644 --- a/pyquadcortex/device/entries.py +++ b/pyquadcortex/device/entries.py @@ -118,21 +118,15 @@ class StateEntry: the unit's whole answer for this entry. Runs on the CALLER's thread, never the RX thread. feeds: message class -> :class:`FieldPlan`. - read_arrivals: messages that say something to this entry during one - normal read. Most reads have one. A live-preset read produces two - on CorOS 4.1.0, so the first must not be mistaken for an unrelated - push that raced the request. - - The read path compares meaningful arrivals with :attr:`read_arrivals`. - A symmetric protocol question carrying only scaffolding does not count. A - larger count means an external push landed during the read and its cache - mark must survive. + The read path normally expects one meaningful arrival. A profile whose + normal read has another shape declares its count on + ``QuadCortex.READ_ARRIVALS``; keeping that firmware fact on the profile is + ADR-0020's boundary. """ name: str read: typing.Callable feeds: typing.Mapping - read_arrivals: int = 1 #: Entries whose copies stop being true when THIS entry's value moves. #: Applied only on a real change, which is what makes it different from #: listing the same message type on each of them: the model's own READ of @@ -481,7 +475,6 @@ def _read_preset(client) -> dict: pa.SceneLabelMessage: _SCENE_TEXT_CHANGED, pa.SceneColorMessage: _SCENE_TEXT_CHANGED, }, - read_arrivals=2, ) diff --git a/pyquadcortex/device/state.py b/pyquadcortex/device/state.py index 39f1dced..71502bb5 100644 --- a/pyquadcortex/device/state.py +++ b/pyquadcortex/device/state.py @@ -423,9 +423,10 @@ def value(self, entry_name: str, field: str): # asks one of its own, and a question is not news about the # unit's state. `_apply_one` is where that is decided, because # the read path cannot tell the two apart after the fact. An - # entry whose read normally produces more than one meaningful - # message declares that measured count on `StateEntry`. + # profile whose read normally produces more than one meaningful + # message declares that measured count on the client class. extra = slot.witnessed - witnessed_before + expected = client.READ_ARRIVALS.get(entry_name, 1) # And the marks that came from something OTHER than a # message to this entry, which the count above cannot see # however carefully it counts: a recall's `resets`, the @@ -439,16 +440,15 @@ def value(self, entry_name: str, field: str): # snapshot and the request going out is kept although the unit # might have accounted for it. `witnessed_before` has had the # same gap since it was written, and both err by re-reading. The obvious - # `slot.needs_read or extra > entry.read_arrivals` does not + # `slot.needs_read or extra > expected` does not # work: the read's own # reply re-arms the mark through the listener from the fields # this entry does not keep, so the entry would never cache # anything again. marked_since = slot.marks - marks_before was_marked = slot.needs_read - slot.needs_read = (extra > entry.read_arrivals - or marked_since > 0) - if extra > entry.read_arrivals: + slot.needs_read = extra > expected or marked_since > 0 + if extra > expected: log.info("push.forced_reread %s - %d message(s) arrived " "while it was being read", entry_name, extra) elif slot.needs_read: diff --git a/pyquadcortex/protocol/client.py b/pyquadcortex/protocol/client.py index 8d5b7387..998a558a 100644 --- a/pyquadcortex/protocol/client.py +++ b/pyquadcortex/protocol/client.py @@ -313,6 +313,11 @@ class QuadCortex: #: Annotated `Any`: this holds either the `EVERYTHING` sentinel or a #: `frozenset[str]`, and both only need to answer `in`. VERIFIED: typing.Any = EVERYTHING + #: Meaningful inbound messages produced by one normal state-cache read, + #: when a profile differs from the default of one. This is profile data: + #: CorOS 4.1.0's live-preset read has a measured unkeyed update before its + #: keyed answer, while no such claim is made for the 4.0.1 baseline. + READ_ARRIVALS: typing.Mapping[str, int] = {} #: The constants snapshot read from this firmware. A subclass rebinds these. #: Dynamic on purpose: `qc.models` follows the connection, at the price of #: mypy seeing `Any` through it - import a snapshot module directly for diff --git a/pyquadcortex/protocol/profiles.py b/pyquadcortex/protocol/profiles.py index 18cd41e0..00448b33 100644 --- a/pyquadcortex/protocol/profiles.py +++ b/pyquadcortex/protocol/profiles.py @@ -44,6 +44,7 @@ class QuadCortex41(QuadCortex): MEASURED_ON = ("4.1.0",) EVIDENCE = Evidence.CONTRIBUTED VERIFIED = frozenset() + READ_ARRIVALS = {"preset": 2} models = NoSnapshot("coros_4_1_0") params = NoSnapshot("coros_4_1_0") options = NoSnapshot("coros_4_1_0") diff --git a/tests/test_profiles.py b/tests/test_profiles.py index 4fd3c8f0..52450723 100644 --- a/tests/test_profiles.py +++ b/tests/test_profiles.py @@ -254,6 +254,7 @@ def test_the_4_1_stub_connects_but_verifies_nothing_and_has_no_snapshot(): assert cls.MEASURED_ON == ("4.1.0",) assert cls.EVIDENCE is support.Evidence.CONTRIBUTED assert cls.VERIFIED == frozenset() + assert cls.READ_ARRIVALS == {"preset": 2} assert cls.CC_VERSION == "4.0.1", "inherited: the contributor's runs announced 4.0.1" assert isinstance(cls.models, support.NoSnapshot) with pytest.raises(AttributeError, match="coros_4_1_0"): diff --git a/tests/test_state.py b/tests/test_state.py index f0f8fb5c..d07b543f 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -440,7 +440,6 @@ def test_a_grid_push_that_lands_during_a_preset_read_is_not_lost_either(link): def preset_but_a_grid_first(triggering=None): transport.push(grid_push()) - transport.push(recall_push()) return recall_push(triggering) transport.broadcasts["RecallPresetMessage"] = preset_but_a_grid_first @@ -698,8 +697,6 @@ def test_a_marking_push_during_any_entrys_read_survives_it(entry, link): def marks_first(canned): def pushing(*args): transport.push(marking) - if entry is entries.PRESET: - transport.push(recall_push()) return canned(*args) if callable(canned) else canned return pushing @@ -1387,6 +1384,12 @@ def test_two_normal_preset_pushes_still_settle_one_read(link): """A live-preset READ emits an uncorrelated push, then its keyed answer.""" transport, cache = link + class CorOS41(protocol_client.QuadCortex): + MEASURED_ON = () + READ_ARRIVALS = {"preset": 2} + + cache.bind(CorOS41(transport)) + def uncorrelated_then_keyed(triggering): transport.push(recall_push()) return recall_push(triggering) From b73d38c1bd650656fc3963885d41675ab9b19ec8 Mon Sep 17 00:00:00 2001 From: Antoni Ivanov <262547018+tony-xmelon@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:07:11 +0300 Subject: [PATCH 3/5] Keep model cache compatible with client doubles --- pyquadcortex/device/state.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyquadcortex/device/state.py b/pyquadcortex/device/state.py index 71502bb5..fe834dfd 100644 --- a/pyquadcortex/device/state.py +++ b/pyquadcortex/device/state.py @@ -426,7 +426,8 @@ def value(self, entry_name: str, field: str): # profile whose read normally produces more than one meaningful # message declares that measured count on the client class. extra = slot.witnessed - witnessed_before - expected = client.READ_ARRIVALS.get(entry_name, 1) + expected = getattr(client, "READ_ARRIVALS", {}).get( + entry_name, 1) # And the marks that came from something OTHER than a # message to this entry, which the count above cannot see # however carefully it counts: a recall's `resets`, the From 9585bee7fd80d38a547f1a6cdaaa6866524ef0e9 Mon Sep 17 00:00:00 2001 From: Antoni Ivanov <262547018+tony-xmelon@users.noreply.github.com> Date: Fri, 11 Sep 2026 21:45:10 +0300 Subject: [PATCH 4/5] Address live preset resilience review --- CLAUDE.md | 2 +- docs/ADR.md | 6 ++++-- docs/protocol.md | 17 ++++++++------- pyquadcortex/device/entries.py | 12 +++++------ pyquadcortex/device/state.py | 35 +++++++++++++++++++++---------- pyquadcortex/protocol/client.py | 22 +++++++++---------- pyquadcortex/protocol/profiles.py | 1 - tests/test_client.py | 22 ++++++++++++++++++- tests/test_profiles.py | 1 - tests/test_state.py | 29 ++++++++++++++++++------- 10 files changed, 99 insertions(+), 48 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 3d6e8662..dff02732 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,7 +14,7 @@ Read `docs/STEERING.md` before non-trivial work (new operations, transport or fr - A model property that reads a device field checks the field is PRESENT (`protocol.field_present`) before reporting it. Most of this schema sits in synthetic `oneof`s, so protobuf returns `""` or `0` for a field the unit never sent, and reporting that as the answer is the guess the rule above forbids. `device/state.py` does this structurally - an absent field is simply not in the entry's copy - so a property reads through it rather than checking by hand. The exception is a field the schema gives no presence at all, where absent and default are the same bytes: those are declared in the entry's `FieldPlan.no_presence` WITH the recorded evidence for what the default means, and `tests/test_state.py` holds the declaration against the schema. Never assume; declare. - The cache holds a COPY of any submessage it keeps, never the container the RX thread decoded - that one is shared with every other listener and read afterwards from other threads. `entries._held` does it; `tests/test_state.py` proves it by mutating the source afterwards. - Never cache an incomplete reply AS IF IT WERE COMPLETE. Per field is the rule: the cache keeps what the unit actually sent and re-reads for what it did not, so a retry recovers the missing half without re-asking for the half already answered. What must never happen is a field the unit never sent being handed back as a value. -- Model state lives in `device/state.py`, and what is tracked is a `StateEntry` in `device/entries.py` - not an attribute a property fills in itself (ADR-0011). Pushes MERGE (an absent field means "not mentioned", never "reset to default"); a read REPLACES, because it is the unit's whole answer. A read consumes ONE mark and only the marks it could have answered, so it counts the messages that arrived for the entry AND the marks set by everything else: anything that stops trusting an entry for a reason other than a message for that entry - a recall's `resets`, the write watchdog - calls `mark_for_reread` and never sets `needs_read` itself, or a read in flight throws the mark away. A message that sets any field the entry does not keep - a schema field or a field number the bindings have never heard of - marks that entry for one re-read; there is no "harmless field" category, and adding one is a guess with a table around it. A message type no entry tracks is ignored outright, which is what makes the metronome's tempo stream free. +- Model state lives in `device/state.py`, and what is tracked is a `StateEntry` in `device/entries.py` - not an attribute a property fills in itself (ADR-0011). Pushes MERGE (an absent field means "not mentioned", never "reset to default"); a read REPLACES, because it is the unit's whole answer. A read consumes a mark only when every meaningful message in its window restates that answer exactly; any different push preserves the mark. It also counts marks set by everything else: anything that stops trusting an entry for a reason other than a message for that entry - a recall's `resets`, the write watchdog - calls `mark_for_reread` and never sets `needs_read` itself, or a read in flight throws the mark away. A message that sets any field the entry does not keep - a schema field or a field number the bindings have never heard of - marks that entry for one re-read; there is no "harmless field" category. `FieldPlan.accepts` is only for two independent conversations sharing one protobuf type, and an accepted message still gets the full unknown-field check. A message type no entry tracks is ignored outright, which is what makes the metronome's tempo stream free. - A new entry decides for itself what `action` means on the types that feed it. The shared `SCAFFOLDING` skip covers the plain entries because `action` gives them no meaning, and that is NOT true of `Grid`, where `action: DELETE` is what removes a block and an `UPDATE` with the same payload does nothing. Never widen `SCAFFOLDING` to make a new entry quiet. `Grid` made its decision and it is `FieldPlan(invalidates=True)`: `action` does not matter, because every `Grid` push means the grid moved, and the entry re-reads rather than merging (ADR-0012). `invalidates` is also the answer where the per-field check is BLIND - `SceneLabel` gives `index` and `label` no presence, so renaming a scene to a blank label sets nothing at all in `ListFields()`. A push carrying every field an entry keeps clears the mark, because that is what a read returns; judge it on what the message carried, not on what its plan could carry. - Anything the model caches is valid only while its connection is. A closed `Device` refuses reads rather than answering from cache, because a model that reports the unit's state through an object with no unit behind it is the failure the whole layer exists to avoid. `Device.close()` closes the state layer first, so a `Device` built by `from_client` stops listening on a connection it never owned. - `import hid` appears exactly once, lazily, inside `session.open_device()`. Never import `hid` at module scope. A new module that needs it imports it inside the function that opens the device; `tests/test_import_cleanliness.py` walks the whole package and proves it. diff --git a/docs/ADR.md b/docs/ADR.md index 3a7feae5..7fee5546 100644 --- a/docs/ADR.md +++ b/docs/ADR.md @@ -166,10 +166,10 @@ Records are append-only once `Decided` and built upon: a shipped decision is nev ## ADR-0011: A push merges, a read replaces, and anything the cache cannot place forces one read - **Status:** Decided (2026-08-14) -- **Decision:** The model's cache treats an inbound message and a read answer differently, on purpose. A **push merges**: the fields the model keeps are applied, and everything else it holds is left alone. A **read replaces**: the answer is the unit's whole account of that entry, so a field it does not carry is dropped rather than kept from before. The per-field check that decides whether our copy is still trustworthy is **conservative and has no exemptions**: a message that sets any field the entry does not keep - including a field number the recovered schema has never heard of - marks that entry, and the next read goes to the unit. The only fields skipped are `action` and `request_id`, which belong to the transport and describe no state. A mark is consumed by **exactly one** read, and what makes that safe is a count rather than a hope: a read clears the mark only if nothing that SAID ANYTHING arrived for that entry beyond the read's own answer (amended 2026-08-27, see the consequences). +- **Decision:** The model's cache treats an inbound message and a read answer differently, on purpose. A **push merges**: the fields the model keeps are applied, and everything else it holds is left alone. A **read replaces**: the answer is the unit's whole account of that entry, so a field it does not carry is dropped rather than kept from before. The per-field check that decides whether our copy is still trustworthy is **conservative and has no exemptions**: a message that sets any field the entry does not keep - including a field number the recovered schema has never heard of - marks that entry, and the next read goes to the unit. The only fields skipped are `action` and `request_id`, which belong to the transport and describe no state. A mark is consumed by one read only when every meaningful message arriving in its window restates that answer exactly; a different concurrent push leaves the mark set (amended 2026-09-11 after the CorOS 4.1 live-preset double reply was measured). - **Context:** `docs/domain-model.md` section 9 gives the rules and the reason for them - "applying the half of a message we understand and silently dropping the rest is the one failure mode that leaves the cache confidently wrong" - and issue #11 restates the check as per FIELD, not per message type. What section 9 does not settle is what happens when the read that resolves a mark carries the same unkept fields that caused it. Answered naively, the entry re-arms its own mark from its own answer and never caches anything again, so every access becomes a round trip and the cache is decorative. The other half of the problem is the opposite failure: clearing the mark unconditionally throws away any push that landed while the read was in flight, and that push is the only record of a change the replacement has just overwritten. - **Options:** - - **(a) Push merges, read replaces, one read consumes the mark, guarded by a count of what arrived - chosen.** Every message that tells an entry something is counted as the listener handles it. The read path notes the count before it asks and, when the answer comes back, clears the mark only if the count moved by no more than the answer itself. Because the transport notifies listeners before waking the thread that asked (ADR-0009), the answer is already counted by then, and a push that arrives in the microseconds afterwards is counted under the same lock - so there is no window, rather than a small one. + - **(a) Push merges, read replaces, one read consumes the mark, guarded by what arrived - chosen.** Every message that tells an entry something is counted and its applied fields are recorded while the read is in flight. The read clears its mark when only its answer arrived, or when additional arrivals restated that answer exactly. A different arrival preserves the mark. Because the transport notifies listeners before waking the thread that asked (ADR-0009), the answer and any intervening push are already recorded under the same lock. - **(b) Declare, field by field, which changes cannot affect what we hold.** A `Version` reply carries a bootloader version and a MAC address; neither can make the firmware string wrong, so an entry could say so and skip the read. It is also a judgement call per field with nothing to check it against, made by whoever adds the field, and wrong quietly. Section 9 exists because the model does not guess, and this is a guess with a table around it. - **(c) Tell a read answer from a push by correlating it.** The transport already refuses to promise this: READ replies carry no `request_id` echo, which is recorded in `Transport.request`. Correlating by type and arrival order would be the same count as (a) with more machinery and a worse failure mode. - **(d) Suppress the listener for an entry while it is being read.** Simple, and it silently drops any genuine push that lands in the read window - which is the exact change the cache most needs to hear about, because a replacement is about to overwrite it. @@ -184,6 +184,8 @@ Records are append-only once `Decided` and built upon: a shipped decision is nev - `action` and `request_id` are skipped on every entry, and only one of them is really the transport's. `request_id` always is. `action` is not: on `Grid` it is load-bearing state, because an `UPDATE` carrying `hash: 0` is transmitted and ignored while the same payload with `action: DELETE` removes the block. It is skipped today because the two tracked message types give it no meaning, and a `Grid` entry (issue #12) therefore cannot inherit the skip - two pushes with identical payloads and opposite meanings would apply identically and mark nothing. That entry gives `action` its own decision rather than widening the shared set. - A field the wire gives no presence and the entry does not KEEP is undetectable rather than merely unkept: proto3 writes such a field only when it differs from its default, so a message leaving one at its default carries no bytes for it and no implementation of this check could see it. `tests/test_state.py` asserts no feeding type has one, which turns a limit of the wire into a question about our own code, checkable and checked. - The arrival count is necessary and NOT sufficient: the read path counts marks as well. The count answers "did anything arrive for this entry beyond my own reply", which says nothing about a mark set by a path that sends no message for that entry at all - and there are two, both reproduced offline on 2026-08-27. A recall marks `dirty` through `LOADED.resets`, and the unit sends no `PresetDirty` after a recall, so nothing was ever coming to correct it; the write watchdog marks from its own thread when no echo arrives, and the answer to a read already in flight may have been composed before the write reached the unit. A read discarded both, the `dirty` window being the 2-11 ms `preset_dirty()` read. So every path that makes an entry untrusted for a reason other than a message for that entry goes through `mark_for_reread`, which counts, and the read snapshots that counter alongside the arrival count and clears only the marks it can have answered. What does NOT work is `needs_read or extra > 1`, and a test catches it: the read's own reply re-arms the mark through the listener from the fields the entry does not keep, which is the entry-caches-nothing failure this record's Context already names. + - **Amended 2026-09-11:** CorOS 4.1.0 emits an unkeyed live-preset `UPDATE` immediately before the keyed answer. A fixed per-profile arrival allowance reopened the race this ADR closes: if that duplicate was absent and a genuine recall arrived after the answer, the recall could be mistaken for the allowed second message and overwritten. The cache instead discounts extra arrivals only when their applied fields equal the answer. A genuine recall carries a different preset and preserves the mark; an exact restatement carries no new state and is safe to discount. This is content comparison, not a declaration that any field is harmless. + - A `FieldPlan.accepts` predicate is reserved for one protobuf message type carrying independent conversations, not for suppressing noisy fields. The first case is `Version`: device identity replies and host-handshake validity replies share the type, and only the former belongs to the identity entry. A rejected conversation is neither applied nor counted, but an accepted message still goes through the complete unknown/unkept-field check. Using `accepts` to quiet a field within one conversation would recreate rejected option (b). ## ADR-0012: A grid push is noted and re-read, not merged, and the model publishes what it noticed diff --git a/docs/protocol.md b/docs/protocol.md index ffe6b2d2..d05d5b07 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -705,10 +705,12 @@ the preset (next section). ### 7.2 Read the current preset `RecallPreset{READ, request_id}` reads the live grid without recalling a stored -slot or discarding edits. On CorOS 4.1.0 one request produces two broadcasts: -first an uncorrelated current-state `UPDATE`, then an otherwise equivalent -`UPDATE` echoing the request id. A client must wait for the keyed second message; -the first is not evidence that some other preset was loaded. +slot or discarding edits. Measured on CorOS 4.0.1 on 2026-08-14, the device may +service this push lazily: 10 to 25 seconds was observed. Measured on CorOS 4.1.0 +on 2026-09-11, one request produces two broadcasts: first an uncorrelated +current-state `UPDATE`, then an otherwise equivalent `UPDATE` echoing the +request id. A client must wait for the keyed second message; the first is not +evidence that some other preset was loaded. Whenever a preset is recalled, by the host or on the unit, the device also **broadcasts**: @@ -725,8 +727,9 @@ Consequences for a client: - reading the live preset is side-effect free; reading a stored preset still recalls that slot and is a different operation; -- the first request after connecting can be dropped, so the library makes two - attempts of 15 seconds each before reporting a timeout; +- on CorOS 4.1.0, measured 2026-09-11, the first request after connecting can + be dropped. The library deliberately applies a bounded two-attempt retry to + every profile, dividing the caller's total timeout between the attempts; - the push must be correlated by `request_id` to avoid returning a stale or seed push (see [section 5](#5-requestresponse-correlation)). @@ -2797,7 +2800,7 @@ screen; **captured only** = seen on the wire, with no independent read-back. | connect handshake | `ResetCommsBuffers` + `Version` UPDATE + `ModelRepo` READ + `Connection` + subscribe READs | read-back | the connect gate; state pushes flow only after it | | version read | `Version{action: READ}` | read-back | serial and firmware returned. TWO messages come back: the `UPDATE` with fifteen fields, then the device's own `Version{READ}` 0.5-0.8 ms later. `version()` accepts only a `Version` carrying `device_serial_number` or `app_fw_version`, so the second one is never returned as the answer; five back-to-back calls after that change all returned the full reply (2026-09-03) - see "A `Version` READ is answered twice" | | `recall_preset` / `read_preset` | `SetlistPosition{UPDATE, folder_key, position, is_factory, request_id}` then a `RecallPreset` push | read-back | the push echoes the recall's `request_id` | -| `read_current_preset` / `read_current_preset_push` | `RecallPreset{READ, request_id}` | read-back | the live grid, no side effects. The push variant hands back the whole reply, which carries `reason` beside the preset | +| `read_current_preset` / `read_current_preset_push` | `RecallPreset{READ, request_id}` | read-back | the live grid, no side effects. Two bounded attempts share the caller's timeout; CorOS 4.1.0 was measured emitting an unkeyed duplicate before the keyed reply. The push variant hands back the whole reply, which carries `reason` beside the preset | | `loaded_position` | `SetlistPosition{READ, request_id}` | read-back | which slot is loaded; 3 ms measured. A READ names no slot - an UPDATE that did would recall it | | `list_presets` | `File{action: READ}` then `File{folder{files[] = ProductData}}` | read-back | factory listing gzipped; 256 slots; listings lag a few seconds after a `File` mutation | | `switch_scene` | `Scene{UPDATE, selected_scene}` | on-unit | zero-based | diff --git a/pyquadcortex/device/entries.py b/pyquadcortex/device/entries.py index 1768e615..d6320f08 100644 --- a/pyquadcortex/device/entries.py +++ b/pyquadcortex/device/entries.py @@ -118,10 +118,9 @@ class StateEntry: the unit's whole answer for this entry. Runs on the CALLER's thread, never the RX thread. feeds: message class -> :class:`FieldPlan`. - The read path normally expects one meaningful arrival. A profile whose - normal read has another shape declares its count on - ``QuadCortex.READ_ARRIVALS``; keeping that firmware fact on the profile is - ADR-0020's boundary. + The read path normally expects one meaningful arrival. Exact duplicate + restatements of that answer may be discounted; a different concurrent push + always leaves the entry untrusted (ADR-0011). """ name: str @@ -509,8 +508,9 @@ def _read_scene(client) -> dict: #: a plan, not a fact, and every push mentioning a field it did not keep would #: mark it for a read nobody had asked for. #: -#: Directory rows are not cache entries: one ``File`` READ streams hundreds of -#: messages and completes by convergence rather than by a fixed arrival count. +#: A future entry whose read provokes a stream of answers must declare how that +#: stream completes. No directory entry exists yet; issue #12 still owns that +#: design decision. ENTRIES = (IDENTITY, DIRTY, PRESET, SCENE, LOADED) ENTRY_BY_NAME = {entry.name: entry for entry in ENTRIES} diff --git a/pyquadcortex/device/state.py b/pyquadcortex/device/state.py index fe834dfd..a351ff1c 100644 --- a/pyquadcortex/device/state.py +++ b/pyquadcortex/device/state.py @@ -54,8 +54,8 @@ class _Slot: """One entry's copy of the unit's state, and how much we trust it.""" - __slots__ = ("fields", "needs_read", "witnessed", "marks", "_arrivals", - "_marks") + __slots__ = ("fields", "needs_read", "witnessed", "marks", "observed", + "_arrivals", "_marks") def __init__(self): #: field name -> value, holding only what the unit has actually said. @@ -89,6 +89,11 @@ def __init__(self): #: rule. Do not simplify this back. self._arrivals = itertools.count(1) self.witnessed = 0 + #: Fields applied by meaningful messages since the current read began. + #: The read path uses these to distinguish CorOS 4.1's exact duplicate + #: live-preset reply from a genuine recall that happens in the same + #: window. Cleared under the slot lock immediately before each read. + self.observed = [] #: How many times a path OTHER than the listener has said this copy #: cannot be trusted: `mark_for_reread` and everything that calls it - #: a recall's `resets`, the write watchdog giving up, a caller of its @@ -240,6 +245,7 @@ def _apply_one(self, entry, plan, message) -> None: # unit's own `Version{READ}` is exactly that message - see # :class:`_Slot`. slot.arrived() + slot.observed.append((dict(applied), why)) was_empty = not slot.fields # Worked out BEFORE the update, and against the values we held, # because a caller tracking changes wants the ones that are @@ -395,6 +401,7 @@ def value(self, entry_name: str, field: str): return slot.fields[field] witnessed_before = slot.witnessed marks_before = slot.marks + slot.observed = [] client = self._client if client is None: raise RuntimeError( @@ -422,12 +429,17 @@ def value(self, entry_name: str, field: str): # than one message: the unit answers a `Version` READ and then # asks one of its own, and a question is not news about the # unit's state. `_apply_one` is where that is decided, because - # the read path cannot tell the two apart after the fact. An - # profile whose read normally produces more than one meaningful - # message declares that measured count on the client class. + # the read path cannot tell the two apart after the fact. extra = slot.witnessed - witnessed_before - expected = getattr(client, "READ_ARRIVALS", {}).get( - entry_name, 1) + observations = tuple(slot.observed) + # CorOS 4.1.0 was measured returning an unkeyed RecallPreset + # immediately before the keyed answer. It is safe to discount + # that second arrival only when every arrival restates the + # fields in the answer exactly. A real recall in this window + # carries a different preset and therefore keeps the mark. + restated_answer = all(applied == answer and why is None + for applied, why in observations) + conflicting_push = extra > 1 and not restated_answer # And the marks that came from something OTHER than a # message to this entry, which the count above cannot see # however carefully it counts: a recall's `resets`, the @@ -441,17 +453,18 @@ def value(self, entry_name: str, field: str): # snapshot and the request going out is kept although the unit # might have accounted for it. `witnessed_before` has had the # same gap since it was written, and both err by re-reading. The obvious - # `slot.needs_read or extra > expected` does not + # `slot.needs_read or conflicting_push` does not # work: the read's own # reply re-arms the mark through the listener from the fields # this entry does not keep, so the entry would never cache # anything again. marked_since = slot.marks - marks_before was_marked = slot.needs_read - slot.needs_read = extra > expected or marked_since > 0 - if extra > expected: + slot.needs_read = conflicting_push or marked_since > 0 + if conflicting_push: log.info("push.forced_reread %s - %d message(s) arrived " - "while it was being read", entry_name, extra) + "while it was being read and did not all restate " + "its answer", entry_name, extra) elif slot.needs_read: # The mirror of the line below, at info rather than its # debug: a surviving mark costs a real round trip, where a diff --git a/pyquadcortex/protocol/client.py b/pyquadcortex/protocol/client.py index 998a558a..e490d26c 100644 --- a/pyquadcortex/protocol/client.py +++ b/pyquadcortex/protocol/client.py @@ -313,11 +313,6 @@ class QuadCortex: #: Annotated `Any`: this holds either the `EVERYTHING` sentinel or a #: `frozenset[str]`, and both only need to answer `in`. VERIFIED: typing.Any = EVERYTHING - #: Meaningful inbound messages produced by one normal state-cache read, - #: when a profile differs from the default of one. This is profile data: - #: CorOS 4.1.0's live-preset read has a measured unkeyed update before its - #: keyed answer, while no such claim is made for the 4.0.1 baseline. - READ_ARRIVALS: typing.Mapping[str, int] = {} #: The constants snapshot read from this firmware. A subclass rebinds these. #: Dynamic on purpose: `qc.models` follows the connection, at the price of #: mypy seeing `Any` through it - import a snapshot module directly for @@ -1455,8 +1450,10 @@ def read_current_preset(self, timeout: float = 15.0, attempts: int = 2): scene-targeted writes silently retargets them; use this method for inspection during editing. - The first request after a connection can be dropped. ``attempts`` - defaults to two; ``timeout`` applies to each attempt. + On CorOS 4.1.0, measured 2026-09-11, the first request after a + connection can be dropped. ``attempts`` deliberately defaults to two + on every profile as a bounded resilience measure; ``timeout`` is the + total budget and is divided between attempts. """ return self.read_current_preset_push( timeout=timeout, attempts=attempts).preset @@ -1479,7 +1476,8 @@ def read_current_preset_push(self, timeout: float = 15.0, """ last = None tries = max(1, attempts) - for _ in range(tries): + attempt_timeout = timeout / tries + for attempt in range(1, tries + 1): request_id = self._t.next_request_id() message = pa.RecallPresetMessage(action=pa.MessageAction.READ, request_id=request_id) @@ -1487,15 +1485,17 @@ def read_current_preset_push(self, timeout: float = 15.0, return self._t.await_broadcast( pa.RecallPresetMessage, lambda outgoing=message: self._t.send(outgoing), - timeout=timeout, + timeout=attempt_timeout, match=lambda m, wanted=request_id: ( m.HasField("request_id") and m.request_id == wanted)) except TimeoutError as exc: last = exc + log.warning("live-preset read attempt %d/%d timed out after " + "%ss", attempt, tries, attempt_timeout) raise TimeoutError( f"the device did not answer a live-preset read in {tries} " - f"attempt(s) of {timeout}s each; the first request after connecting " - f"can be dropped") from last + f"attempt(s) within {timeout}s total; CorOS 4.1.0 was measured " + f"dropping the first request after connecting") from last def loaded_position(self, timeout: float = 10.0): """Which preset slot is on the grid right now. diff --git a/pyquadcortex/protocol/profiles.py b/pyquadcortex/protocol/profiles.py index 00448b33..18cd41e0 100644 --- a/pyquadcortex/protocol/profiles.py +++ b/pyquadcortex/protocol/profiles.py @@ -44,7 +44,6 @@ class QuadCortex41(QuadCortex): MEASURED_ON = ("4.1.0",) EVIDENCE = Evidence.CONTRIBUTED VERIFIED = frozenset() - READ_ARRIVALS = {"preset": 2} models = NoSnapshot("coros_4_1_0") params = NoSnapshot("coros_4_1_0") options = NoSnapshot("coros_4_1_0") diff --git a/tests/test_client.py b/tests/test_client.py index c6262127..ec6d2b6e 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1325,6 +1325,24 @@ def await_broadcast(self, expected_class, trigger, timeout=40.0, assert transport.calls == 2 assert [m.request_id for m in transport.sent] == [1, 2] assert transport.matches[-1](push) is True + assert transport.matches[-1]( + pa.RecallPresetMessage(request_id=1)) is False + + +def test_read_current_preset_one_attempt_sends_once_and_raises(): + cause = TimeoutError("one miss") + + class NeverAnswers(FakeTransport): + def await_broadcast(self, expected_class, trigger, timeout=40.0, + match=None): + trigger() + raise cause + + qc = client.QuadCortex(NeverAnswers()) + with pytest.raises(TimeoutError, match=r"1 attempt\(s\)") as raised: + qc.read_current_preset(timeout=0.25, attempts=1) + assert raised.value.__cause__ is cause + assert len(qc._t.sent) == 1 def test_read_current_preset_timeout_reports_all_attempts(): @@ -1335,9 +1353,11 @@ def await_broadcast(self, expected_class, trigger, timeout=40.0, raise TimeoutError("nothing") qc = client.QuadCortex(NeverAnswers()) - with pytest.raises(TimeoutError, match=r"2 attempt\(s\) of 0.01s each"): + with pytest.raises(TimeoutError, match=r"2 attempt\(s\) within 0.01s total") \ + as raised: qc.read_current_preset(timeout=0.01) assert [m.request_id for m in qc._t.sent] == [1, 2] + assert isinstance(raised.value.__cause__, TimeoutError) def test_loaded_position_reads_the_slot_without_recalling_it(): diff --git a/tests/test_profiles.py b/tests/test_profiles.py index 52450723..4fd3c8f0 100644 --- a/tests/test_profiles.py +++ b/tests/test_profiles.py @@ -254,7 +254,6 @@ def test_the_4_1_stub_connects_but_verifies_nothing_and_has_no_snapshot(): assert cls.MEASURED_ON == ("4.1.0",) assert cls.EVIDENCE is support.Evidence.CONTRIBUTED assert cls.VERIFIED == frozenset() - assert cls.READ_ARRIVALS == {"preset": 2} assert cls.CC_VERSION == "4.0.1", "inherited: the contributor's runs announced 4.0.1" assert isinstance(cls.models, support.NoSnapshot) with pytest.raises(AttributeError, match="coros_4_1_0"): diff --git a/tests/test_state.py b/tests/test_state.py index d07b543f..d250699b 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -1381,15 +1381,9 @@ def test_the_preset_entry_can_read_back_every_field_it_keeps(link): def test_two_normal_preset_pushes_still_settle_one_read(link): - """A live-preset READ emits an uncorrelated push, then its keyed answer.""" + """An exact uncorrelated restatement cannot make the cache stale.""" transport, cache = link - class CorOS41(protocol_client.QuadCortex): - MEASURED_ON = () - READ_ARRIVALS = {"preset": 2} - - cache.bind(CorOS41(transport)) - def uncorrelated_then_keyed(triggering): transport.push(recall_push()) return recall_push(triggering) @@ -1402,6 +1396,27 @@ def uncorrelated_then_keyed(triggering): assert transport.reads["RecallPresetMessage"] == 1 +def test_a_different_recall_after_the_read_answer_keeps_the_mark(link): + """A real recall must not be mistaken for CorOS 4.1's duplicate answer.""" + transport, cache = link + + answer = recall_push() + + class RecallDuringRead: + def read_current_preset_push(self): + transport.push(answer) + recalled = recall_push() + recalled.preset.name = "newer preset" + transport.push(recalled) + return answer + + cache.bind(RecallDuringRead()) + cache.mark_for_reread("preset", "exercise a concurrent genuine recall") + + assert cache.value("preset", "preset").name == "Structural Fixture" + assert cache.needs_read("preset") is True + + def test_a_recall_push_does_not_invalidate_the_preset_it_delivers(): plan = entries.PRESET.feeds[pa.RecallPresetMessage] assert not plan.voids_the_copy() From 098f3b1bbc3b675d5c9b7d1bcb06b0ff832fed57 Mon Sep 17 00:00:00 2001 From: Antoni Ivanov <262547018+tony-xmelon@users.noreply.github.com> Date: Sat, 26 Sep 2026 18:58:19 +0300 Subject: [PATCH 5/5] Restore recall cache review documentation --- docs/ADR.md | 2 +- pyquadcortex/device/entries.py | 32 +++++++++++++++++++++----------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/docs/ADR.md b/docs/ADR.md index 3fc34619..ae557596 100644 --- a/docs/ADR.md +++ b/docs/ADR.md @@ -168,7 +168,7 @@ Records are append-only once `Decided` and built upon: a shipped decision is nev ## ADR-0011: A push merges, a read replaces, and anything the cache cannot place forces one read - **Status:** Decided (2026-08-14) -- **Decision:** The model's cache treats an inbound message and a read answer differently, on purpose. A **push merges**: the fields the model keeps are applied, and everything else it holds is left alone. A **read replaces**: the answer is the unit's whole account of that entry, so a field it does not carry is dropped rather than kept from before. The per-field check that decides whether our copy is still trustworthy is **conservative and has no exemptions**: a message that sets any field the entry does not keep - including a field number the recovered schema has never heard of - marks that entry, and the next read goes to the unit. The only fields skipped are `action` and `request_id`, which belong to the transport and describe no state. A mark is consumed only when every meaningful message in its read window restates the answer exactly; a different concurrent push leaves the mark set (amended 2026-09-11 after the CorOS 4.1 live-preset double reply was measured). +- **Decision:** The model's cache treats an inbound message and a read answer differently, on purpose. A **push merges**: the fields the model keeps are applied, and everything else it holds is left alone. A **read replaces**: the answer is the unit's whole account of that entry, so a field it does not carry is dropped rather than kept from before. The per-field check that decides whether our copy is still trustworthy is **conservative and has no exemptions**: a message that sets any field the entry does not keep - including a field number the recovered schema has never heard of - marks that entry, and the next read goes to the unit. The only fields skipped are `action` and `request_id`, which belong to the transport and describe no state. A mark is consumed by **exactly one** read, and what makes that safe is a count rather than a hope: a read clears the mark only if nothing that said anything arrived for that entry beyond the read's own answer (amended 2026-08-27, see the consequences). - **Context:** `docs/domain-model.md` section 9 gives the rules and the reason for them - "applying the half of a message we understand and silently dropping the rest is the one failure mode that leaves the cache confidently wrong" - and issue #11 restates the check as per field, not per message type. What section 9 does not settle is what happens when the read that resolves a mark carries the same unkept fields that caused it. Answered naively, the entry re-arms its own mark from its own answer and never caches anything again, so every access becomes a round trip and the cache is decorative. The other half of the problem is the opposite failure: clearing the mark unconditionally throws away any push that landed while the read was in flight, and that push is the only record of a change the replacement has just overwritten. - **Options:** - **(a) Push merges, read replaces, one read consumes the mark, guarded by a count of what arrived - chosen.** Every message that tells an entry something is counted as the listener handles it. The read path notes the count before it asks and, when the answer comes back, clears the mark only if the count moved by no more than the answer itself. Because the transport notifies listeners before waking the thread that asked (ADR-0009), the answer is already counted by then, and a push that arrives in the microseconds afterwards is counted under the same lock - so there is no window, rather than a small one. diff --git a/pyquadcortex/device/entries.py b/pyquadcortex/device/entries.py index d6320f08..73a621a7 100644 --- a/pyquadcortex/device/entries.py +++ b/pyquadcortex/device/entries.py @@ -118,9 +118,19 @@ class StateEntry: the unit's whole answer for this entry. Runs on the CALLER's thread, never the RX thread. feeds: message class -> :class:`FieldPlan`. - The read path normally expects one meaningful arrival. Exact duplicate - restatements of that answer may be discounted; a different concurrent push - always leaves the entry untrusted (ADR-0011). + Every entry's :attr:`read` is one request and one ANSWER, which the read + path relies on to tell its own answer apart from a push that arrived while + it was waiting. One answer is not the same as one message: a ``Version`` + READ is answered by the unit's reply and then by a question of the unit's + own, and the read path survives that because a message that said nothing is + not counted. Said nothing means it applied no field this entry keeps AND + named none it does not - a plan with :attr:`FieldPlan.invalidates` set is + never in that case, because every message of its type makes the copy + untrusted whatever it carried. ``device/state.py``'s ``_apply_one`` decides + it. An entry whose read provokes a STREAM OF ANSWERS instead - a ``File`` + enumeration, a preset dump - has to say how many messages that is, and this + class does not carry that yet because nothing needs it. It lands with the + first such entry, along with the test that a number other than one works. """ name: str @@ -239,6 +249,13 @@ def _carries_unknown_fields(message) -> bool: # -- the entries ------------------------------------------------------------- +def _version_describes_device(message) -> bool: + """Exclude the two Version shapes that are only handshake traffic.""" + named = {field.name for field, _ in message.ListFields()} + host_handshake = SCAFFOLDING | {"cortex_control_version_valid"} + return bool(named - host_handshake) or _carries_unknown_fields(message) + + #: The unit's identity. Firmware and serial cannot change while a connection is #: up: the only thing that changes either is a firmware update, and the firmware #: `Updater` surface is permanently out of scope for this library (repo-root @@ -270,13 +287,6 @@ def _carries_unknown_fields(message) -> bool: #: device's answer to the host's Cortex Control announcement is also a Version, #: but it carries only handshake validity and likewise says nothing about the #: unit's identity. -def _version_describes_device(message) -> bool: - """Exclude the two Version shapes that are only handshake traffic.""" - named = {field.name for field, _ in message.ListFields()} - host_handshake = SCAFFOLDING | {"cortex_control_version_valid"} - return bool(named - host_handshake) or _carries_unknown_fields(message) - - _VERSION_FOR_IDENTITY = FieldPlan( kept=frozenset({"app_fw_version", "device_serial_number"}), accepts=_version_describes_device, @@ -509,7 +519,7 @@ def _read_scene(client) -> dict: #: mark it for a read nobody had asked for. #: #: A future entry whose read provokes a stream of answers must declare how that -#: stream completes. No directory entry exists yet; issue #12 still owns that +#: stream completes. No directory entry exists yet; issue #25 still owns that #: design decision. ENTRIES = (IDENTITY, DIRTY, PRESET, SCENE, LOADED)