diff --git a/.gitattributes b/.gitattributes index eca11902..b19d56a2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ tests/fixtures/surface/pre_flip_init.py.txt text eol=lf +tests/fixtures/catalog/*.bin binary diff --git a/CLAUDE.md b/CLAUDE.md index 413e5366..2c5be738 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,6 +65,11 @@ small documentation fixes. footswitch index and a block's column are different numbers that usually agree. - `import hid` appears once, lazily, inside `session.open_device()`. `tests/test_import_cleanliness.py` proves it. +- Never gitignore, delete or edit `tests/fixtures/catalog/*.bin`. + Each is the captured input its snapshot is generated from, and deleting one + puts regeneration back behind a unit nobody else has. + `tests/test_catalog_payload.py` holds each payload to the snapshot it + generates; a failure there means regenerating, never editing. (ADR-0022) - Never gitignore or delete `pyquadcortex/protocol/proto/*_pb2.py` or `*_pb2.pyi`. Regenerate only with `scripts/compile_protos.sh`, and bump the `protobuf` pin and the `grpcio-tools` floor in the same commit. Details in diff --git a/docs/ADR.md b/docs/ADR.md index ee610752..e6b05a80 100644 --- a/docs/ADR.md +++ b/docs/ADR.md @@ -6,7 +6,7 @@ Architectural decisions for this repository are captured in this file, appended ## Format -Each ADR has: ID, Title, Status, Decision, Context, Options, Open Questions, Rationale, Consequences. A `Supersedes: ADR-NNNN` line appears only when the record replaces a prior decision. +Each ADR has: ID, Title, Status, Decision, Context, Options, Open Questions, Rationale, Consequences. A `Supersedes: ADR-NNNN` line appears only when the record replaces a prior decision. A `Settles with: ADR-NNNN` line appears when a record leaves an earlier decision standing but overturns the reasoning of one of its consequences, which `Supersedes` would overstate. **Status** is one of `Proposed`, `Under Discussion`, `Decided`, `Superseded`, `Rejected`, optionally annotated: e.g. `Decided (confirmed 2026-08-03)` or `Superseded by ADR-NNNN (2026-08-03)`. @@ -396,3 +396,36 @@ Records are append-only once `Decided` and built upon: a shipped decision is nev - The settings start to matter the day any contributor is granted write access. Granting that access is a separate, deliberate act and is not part of this record. - This is repository governance rather than library architecture, and it lives here rather than in `contributing.md` or `docs/STEERING.md` because the durable part is the reasoning - in particular why `require_last_push_approval` was rejected, which is the decision a future maintainer would otherwise re-litigate from scratch. `contributing.md` gains its note about sync-before-review only once contributors can actually merge. - Reversal is one API call and costs nothing, which is why this record is short on hedging: the setting is cheap to try and cheap to undo. + +## ADR-0022: The payload a catalog snapshot is generated from is committed beside it + +- **Status:** Decided (2026-09-22) +- **Decision:** The `ModelRepo` reply each snapshot in `pyquadcortex/protocol/catalogs/` was generated from is committed to this repository, under `tests/fixtures/catalog/model_repo_.bin`. `tests/test_catalog_payload.py` regenerates from every payload listed in its `PAYLOADS` table and compares byte for byte, so a snapshot and its recorded input cannot drift apart unnoticed. A snapshot whose payload is not listed is not covered. A profile that saves a payload adds a row to `PAYLOADS` there. The payload is a captured input and is never edited; a snapshot is still produced only by its generator. +- **Context:** The three generators in `scripts/` have always accepted `--payload`, but the only 4.0.1 payload lived on the maintainer's disk. Regenerating the 4.0.1 snapshot therefore needed the one unit running that firmware, which is the single input a contributor cannot borrow, share or reconstruct. #45 stalled on exactly this: its parser change makes `options.py` stale, and its author runs CorOS 4.1.0, where generating into the 4.0.1 snapshot would stamp the wrong firmware's data into a file named for 4.0.1. The maintainer could regenerate on request, but that makes every future catalog change wait on one person owning one device. + Measured while writing this record, against the committed payload: 533 models, 414 of them factory across 22 categories, which matches the numbers the generators already recorded. Using main's own generators, the payload reproduces `models.py`, `params.py` and `options.py` byte for byte, `options.py` together with the readings in `tests/fixtures/catalog/option_readings.json`, which its generator reads besides the payload. The package `__init__.py` is not payload-derived: `scripts/_snapshots.py` builds it from the directory listing, and `tests/test_generators.py` already pins it. + Two properties of the payload decided the shape of this record. It describes every model the firmware knows, including plugin content no licence has been bought for: 90 models carry a `sku` and a `plugin_id`, always together and never one alone, and the unit that produced this payload owns none of them. And each read rotates a per-model `blob` token, so two reads of the same catalog never match byte for byte. There are 351 of these attributes, 13 of them empty strings, which is why `docs/domain-model.md` counts 338 models carrying one and a reader counting attributes finds 351. +- **Options:** + - **(a) Commit the payload beside the snapshot, held by a test - chosen.** Makes regeneration reproducible for anyone, puts the input under review like any other file, and lets CI catch a snapshot that no longer matches its own input. + - **(b) Leave it in the private lab repository.** Where it was. Rejected: the people who most need it are outside contributors, who cannot read that repository. It also splits a generated file from its input across two repositories, so nothing can check the pair. + - **(c) Attach it to the issue or pull request that needs it.** Rejected. It solves one request and leaves the next one asking again, and a comment attachment is not versioned, not reviewable and easily lost. + - **(d) The maintainer regenerates on request.** Rejected as the status quo that caused the stall. It makes a device the bottleneck for work that needs no device. + - **(e) Commit the extracted `ModelRepo.xml` instead of the payload.** Rejected: the payload is what the unit actually sent, and `parse_model_repo` is the thing under test. Committing a derived form would test the parser against our own extraction rather than against the wire. +- **Open Questions:** + - Whether a snapshot really does not vary with what a unit owns. The one unit that has produced a payload owns no plugin content, so the inference runs one way: the catalog lists content the unit does not have. Whether a unit that has bought an Archetype publishes anything different - a changed attribute, an extra model, a different `sku` - is unmeasured, and a payload from such a unit would settle it in one read. + - Whether a second profile should be required to commit its payload rather than merely invited to. Today `PAYLOADS` covers what it lists and nothing forces a new snapshot to join, so a profile can land regenerable only on its own unit, which is the state this record exists to end. Left open because no second profile has landed yet, and a rule written for a case nobody has met tends to be the wrong rule. + - Whether `tests/fixtures/catalog/scales.json` should still be maintained by hand. ADR-0015 created it so the catalog need not be committed, and the catalog is now committed. It could be derived from the payload, or checked against it. Not decided here: ADR-0015's hardware test already proves it has not drifted, and changing it is a separate piece of work with its own hardware consequence. +- **Settles with: ADR-0015.** Its last consequence says `tests/fixtures/catalog/scales.json` distils "the parameters under test" out of a real catalog "because committing `ModelRepo.xml` would republish 556 KB of the vendor's product catalog". That is this file: gunzip the payload and its one tar member is `ModelRepo.xml`, 556,732 bytes. ADR-0015's decision stands and its status is unchanged; this record overturns that one bullet's reasoning, and says so rather than leaving two records a reader cannot both act on. + **This does republish content the repository did not carry, and the decision accepts that rather than denying it.** `models.py` and `params.py` emit factory content only, and say so. Measured against main: 77 of the 119 non-factory model names appear nowhere else in the repository, and 97 parameter names carried only by non-factory models are likewise absent. 70 of those 77 carry a `sku` and a `plugin_id`, which is the licence-gated content ADR-0015 was weighing: the `Gojira`, `Cory Wong`, `Nolly`, `Plini`, `SLO-100`, `Fortin`, `Parallax` and `Nameless` families, and two hidden PCOM cab defaults. The remaining 7 are internal or diagnostic - the four `ZenLink` models, a self-test, a PCOM tape delay and the `2-Track Looper`. Even for factory content the payload publishes a form this repository did not hold: it carries the drawn name `110 US PRN Brown FatJ 10s (M)`, where the snapshot emits constants, and 137 of the 397 distinct factory names appear nowhere on main as the payload spells them. + What changed the weight of the concern is size and purpose, not content. ADR-0015 was weighing 556 KB of XML; what is committed is the 46,695-byte reply as the unit sent it, which stays compressed in git and is never expanded on disk. It is a functional input, read by three generators and held by a test, rather than a copy of a product catalog for its own sake. Set against a snapshot nobody but the maintainer can regenerate, which is the state this record exists to end, that trade is worth making. It is a trade, and this bullet is here so the next reader weighs it rather than inherits it. + `scales.json`'s stated reason for existing is weaker as a result, which is noted as an open question above. +- **Rationale:** The generated catalog files are already committed rather than built at install time (ADR-0001 makes the same trade for the protobuf bindings). Committing the output while leaving the input on one person's disk is the half of that argument which does not hold: a generated file whose source nobody else has is only nominally regenerable. This closes it. + The test is the part that matters. A saved input nothing reads rots silently, and the offline suite would pass identically if the payload were truncated or from the wrong firmware. Holding the payload to the snapshot it generates turns the file from documentation into a check. + This does not weaken ADR-0020 or replace the hardware check. `tests/hardware/test_generated_constants.py` asks whether the snapshot still matches the **unit**, which only a unit can answer. This asks whether the snapshot still matches its own **recorded input**, which is what makes regeneration reproducible off the unit. Both are needed and they fail for different reasons. +- **Consequences:** + - Regenerating a snapshot no longer needs a unit. A contributor on other firmware can rebuild the 4.0.1 snapshot correctly. + - The snapshot and its input are reviewed together, and CI catches them parting company. + - A payload and its provenance record ship in the sdist, because `pyproject.toml` includes `/tests`. Neither is in the wheel. The payload is about 46 KB; its exact size is recorded beside it in the provenance record, which `tests/test_catalog_payload.py` pins against the file, and it is already gzipped so it does not compress further. The two together add about 50 KB. No exact figure is recorded here, deliberately: the sdist's `include` covers the package, the tests, the scripts, the examples, the proto schema and three files at the root, so the number moves on almost any change to the repository, while `docs/` is not in the sdist at all and a change to this record never moves it. A byte count written here went stale twice, each time in the commit that was correcting the previous one, which is the argument for recording here only the size a test holds. The 556,732 above is the other number this record carries, quoted from ADR-0015's own reasoning and derivable from the committed payload. + - A new profile that wants the same guarantee saves its payload and adds a row to `PAYLOADS`. Nothing forces it to, so a profile with no committed payload stays regenerable only on its own unit. + - A payload carries no statement of its own provenance: the XML root has no attributes and the tar member has no metadata. `model_repo_.provenance.json` records the identifying fields of the `Version` reply beside it, minus the serial number, MAC address and custom name, which are the owner's and not the firmware's. + - Payload bytes are not reproducible between reads and must never be compared by hash; compare parsed catalogs. The one recorded comparison is the maintainer's, across reads on 2026-07-26 and 2026-09-22, identical once the `blob` attributes are blanked. The earlier payload is not committed, so that comparison cannot be repeated from this repository. + - The claim that a snapshot does not vary with what a unit owns rests on one unit, which owns no plugin content. A payload from a unit that does own some would settle it; none has been read. diff --git a/docs/STEERING.md b/docs/STEERING.md index f48c53a3..ec34ba46 100644 --- a/docs/STEERING.md +++ b/docs/STEERING.md @@ -44,10 +44,11 @@ generated bindings in `pyquadcortex/protocol/proto/`), the recovered schema in which users install themselves (see the README). - **PyPI.** Published as `pyquadcortex` with the `qcctl` console script. Release process in [`releasing.md`](releasing.md). -- **The lab repository.** Planning material, raw USB captures, the catalog dump, - and the narrative history behind the documents here live in the private - `quad-cortex` repository. This repository carries the library and its - engineering documents. +- **The lab repository.** Planning material, raw USB captures and the narrative + history behind the documents here live in the private `quad-cortex` + repository. This repository carries the library, its engineering documents, + and the `ModelRepo` payloads its catalog snapshots are generated from + (`tests/fixtures/catalog/`, ADR-0022). ## 3. Architecture Overview @@ -168,6 +169,7 @@ Decisions are recorded in [`ADR.md`](ADR.md): | ADR-0019 | The frame trailer's flags are read and reported; an encrypted payload is labelled, not decrypted | | ADR-0020 | Connect resolves a device profile, and nothing else branches on firmware or model | | ADR-0021 | An approval stops counting once the pull request's code changes | +| ADR-0022 | The payload a catalog snapshot is generated from is committed beside it | ## 8. Open Questions @@ -231,6 +233,18 @@ Entries are short by design ([`writing.md`](writing.md)). The full narrative behind each one is in the lab repository, `doc/pyquadcortex/history/steering-change-log.md`, and in the pull requests. +### 2026-09-22 - A snapshot's input is committed beside it (ADR-0022) + +- **What changed:** the `ModelRepo` payload the 4.0.1 snapshot is generated from + is committed, with a provenance record and a test regenerating each snapshot + from its own payload. Section 2 corrected. +- **Why:** regenerating needed the one unit on that firmware, blocking #45. +- **Scope:** `tests/fixtures/catalog/`, `tests/test_catalog_payload.py`, + `.gitattributes`, `CLAUDE.md`, `ADR.md`, `architecture.md`, `protocol.md`, + sections 2 and 7 here, and docstrings in `extract_scale_fixture.py` and three + hardware tests that named the catalog's absence. Left alone: `domain-model.md`. + No library code, no generated file. + ### 2026-09-21 - The hardware suite puts the edited flag back - **What changed:** a session teardown clears the preset's edited flag with a diff --git a/docs/architecture.md b/docs/architecture.md index d04e6d1c..7fd10c9e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -520,6 +520,18 @@ On the unit it covers: differently is written into `protocol.md` next to the existing entry, dated and named, and overridden on the new class. +Step 1 is the one you repeat. A generator change means regenerating, and without +a saved reply that means borrowing the unit again. Each generator also takes +`--payload`, a saved `ModelRepo` reply, so a snapshot can be rebuilt afterwards +with nothing attached: + + python scripts/generate_options.py --snapshot coros_4_0_1 \ + --payload tests/fixtures/catalog/model_repo_coros_4_0_1.bin + +Save the new profile's payload beside that one with its provenance record, and +add a row to `PAYLOADS` in `tests/test_catalog_payload.py`, which holds each +committed payload to the snapshot it generates (ADR-0022). + When measuring a new CorOS release, check these in order: 1. **The schema.** Re-recover the `.proto` files from the matching Cortex diff --git a/docs/protocol.md b/docs/protocol.md index 1b92e1f5..4d23749b 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -1962,7 +1962,9 @@ Named forms live in `pyquadcortex/protocol/enums.py` (`Input`, `Output`, `ModelRepo` is read during the connect burst as a readiness gate, and its payload is the unit's whole block catalog: gzip(tar(`ModelRepo.xml`)), about 46 KB -compressed and 557 KB expanded. Cortex Control reads it as the third message +compressed and 557 KB expanded. One such reply is committed, read from the 4.0.1 +unit on 2026-09-22: `tests/fixtures/catalog/model_repo_coros_4_0_1.bin`, with its +firmware recorded beside it (ADR-0022). Cortex Control reads it as the third message type of every session, after `ResetCommsBuffers` and `Version`, and the reply lands about 1.2 s after the session's first message in all three lab captures. diff --git a/scripts/extract_scale_fixture.py b/scripts/extract_scale_fixture.py index 5b06e8c2..cb69d0f6 100644 --- a/scripts/extract_scale_fixture.py +++ b/scripts/extract_scale_fixture.py @@ -4,11 +4,14 @@ python scripts/extract_scale_fixture.py --payload model_repo_payload.bin The offline suite has to prove that the device's own catalog reproduces every -reading taken off the unit's screen. That needs real numbers, and the obvious -way to get them - committing ``ModelRepo.xml`` - would republish 556 KB of the -vendor's product catalog: every model name, every impulse-response name, every -knob. ``tests/test_catalog.py`` has said "ships no vendor data" since it was -written, and this keeps that true. +reading taken off the unit's screen. That needs real numbers. Committing the +catalog whole was once the thing this script existed to avoid; since ADR-0022 +the 4.0.1 payload IS committed, at +``tests/fixtures/catalog/model_repo_coros_4_0_1.bin``, so that is no longer the +reason. What remains is that the fixture is the data under test, read by tests +that assert on a few dozen parameters and nothing else. Whether it should now +be derived from the committed payload rather than distilled by hand is +ADR-0022's third open question. So the fixture holds only the parameters the tests actually assert on: for each, its bounds, its taper and its floor. That is the data under test and nothing diff --git a/scripts/generate_models.py b/scripts/generate_models.py index b7e330a1..e463dd52 100644 --- a/scripts/generate_models.py +++ b/scripts/generate_models.py @@ -18,7 +18,7 @@ # from a saved payload, for reproducible regeneration python scripts/generate_models.py --snapshot coros_4_1_0 \ - --payload model_repo_payload.bin + --payload tests/fixtures/catalog/model_repo_coros_4_0_1.bin Only identifiers and integers are emitted - no descriptions and no "Based on" attribution text. diff --git a/scripts/generate_options.py b/scripts/generate_options.py index 5bb3de25..623e83b9 100644 --- a/scripts/generate_options.py +++ b/scripts/generate_options.py @@ -14,7 +14,7 @@ python scripts/generate_options.py --snapshot coros_4_1_0 python scripts/generate_options.py --snapshot coros_4_1_0 \ - --payload model_repo_payload.bin + --payload tests/fixtures/catalog/model_repo_coros_4_0_1.bin Four decisions this generator makes: diff --git a/scripts/generate_params.py b/scripts/generate_params.py index d988bdf3..690592be 100644 --- a/scripts/generate_params.py +++ b/scripts/generate_params.py @@ -15,7 +15,7 @@ # from a saved payload, for reproducible regeneration python scripts/generate_params.py --snapshot coros_4_1_0 \ - --payload model_repo_payload.bin + --payload tests/fixtures/catalog/model_repo_coros_4_0_1.bin Three things this generator knows that the catalog does not: diff --git a/tests/fixtures/catalog/model_repo_coros_4_0_1.bin b/tests/fixtures/catalog/model_repo_coros_4_0_1.bin new file mode 100644 index 00000000..b60c9c3e Binary files /dev/null and b/tests/fixtures/catalog/model_repo_coros_4_0_1.bin differ diff --git a/tests/fixtures/catalog/model_repo_coros_4_0_1.provenance.json b/tests/fixtures/catalog/model_repo_coros_4_0_1.provenance.json new file mode 100644 index 00000000..de312094 --- /dev/null +++ b/tests/fixtures/catalog/model_repo_coros_4_0_1.provenance.json @@ -0,0 +1,34 @@ +{ + "_comment": [ + "Provenance for model_repo_coros_4_0_1.bin: the identifying fields of the", + "Version reply from the unit that produced it. The payload itself states", + "nothing about its own origin, so without this file nothing can tell which", + "firmware generated the snapshot beside it.", + "The serial number, MAC address and custom name are left out on purpose:", + "they identify one owner's unit and say nothing about the firmware, and", + "tests/test_catalog_payload.py holds them out.", + "comms_version is in the reply and is left out for a different reason: it is", + "a protocol version rather than an identity, and being a bytes field it has", + "no faithful JSON spelling.", + "cortex_control_version and cortex_control_version_valid are absent because", + "the unit's reply does not carry them, not because they were dropped. The", + "first is what the host announces TO the unit; the second comes back in a", + "later message of its own. See docs/protocol.md, the connect handshake." + ], + "captured_utc": "2026-09-22T18:30:21Z", + "payload_bytes": 46695, + "payload_sha256": "fdc30861953be15ce4e776f3e66d101fb55d7bb4ec6aa85f77e35e856777ac14", + "payload_sha256_note": "Identifies THIS capture only. Each read rotates the per-model blob tokens, so a later read of the same catalog has a different hash. Compare parsed catalogs, never this value.", + "version_reply": { + "zenos_git_hash": "4.0.1", + "device_type": "QC", + "app_fw_version": "d14e", + "bootloader_fw_version": "d14d", + "zencoder_fw_app": "d111", + "zencoder_fw_bootloader": "d10d", + "zenwireless_fw_version": "cf9daede4300aaae664fc527cede12ae", + "uboot_version": "U-Boot 2015.01 ADI-1.3.0 (Sep 30 2021 - 01:01:44)", + "linux_kernel_version": "Linux buildroot 4.0.0-ADI-1.3.0 #1 PREEMPT Fri Feb 20 12:30:04 UTC 2026 armv7l (none)", + "is_ess": true + } +} diff --git a/tests/hardware/test_generated_constants.py b/tests/hardware/test_generated_constants.py index b49975bd..fa272834 100644 --- a/tests/hardware/test_generated_constants.py +++ b/tests/hardware/test_generated_constants.py @@ -2,8 +2,11 @@ `models.py` and `params.py` are generated from a device's ModelRepo and then COMMITTED, so they are a snapshot. A firmware or content update can renumber a -parameter or add a model, and nothing offline can notice - the generated file is -its own yardstick. This regenerates from the connected unit and compares. +parameter or add a model, and nothing offline can notice: since ADR-0022 +`tests/test_catalog_payload.py` holds the snapshot against the payload it was +generated from, but that payload is a capture of one moment, so the two move +together and agree while the unit has moved on. This regenerates from the +connected unit and compares. Which snapshot it compares against comes from the CONNECTED PROFILE (ADR-0020) rather than from a fixed path, so a unit on some other firmware is held against diff --git a/tests/hardware/test_option_audit_on_unit.py b/tests/hardware/test_option_audit_on_unit.py index c21e84ea..e94c5431 100644 --- a/tests/hardware/test_option_audit_on_unit.py +++ b/tests/hardware/test_option_audit_on_unit.py @@ -2,8 +2,8 @@ `tests/fixtures/catalog/option_readings.json` holds what a human read off the screen, and `options.OPTION_AUDIT` stamps each list from it. Both are pinned to -a SNAPSHOT of the catalog, and like the generated constants nothing offline can -notice that snapshot going stale - a firmware that renames an option would leave +a SNAPSHOT of the catalog, and like the generated constants no offline check can +notice the UNIT moving away from it - a firmware that renames an option would leave every offline test green while the library published a word the unit no longer uses, now with "audited" beside it, which is worse than never having checked. @@ -157,8 +157,10 @@ def test_the_hidden_attribute_is_still_shaped_the_way_it_was_counted(live_xml): block's `MOMENTARY` switch, which is the whole evidence that the catalog names the MODEL a parameter is hidden on, and a second independent sign that ATMA is the Mini. Both live in a docstring and in `docs/domain-model.md`, - and the parser turns the attribute into a bool, so nothing offline can see - either number go stale. + and the parser turns the attribute into a bool. Since ADR-0022 both counts + can be taken offline from the committed payload, and nothing takes them; + either way a count against that capture cannot see this unit's own catalog + move. A failure here is a finding about the device. If `atma` has spread to more parameters, `Parameter.hidden` answering only for a Quad Cortex matters more diff --git a/tests/hardware/test_option_structure_on_unit.py b/tests/hardware/test_option_structure_on_unit.py index 6d091056..40f3bfd7 100644 --- a/tests/hardware/test_option_structure_on_unit.py +++ b/tests/hardware/test_option_structure_on_unit.py @@ -188,12 +188,15 @@ def test_the_displayPos_counts_the_docs_quote_still_hold(live_catalog): this change each found a number wrong somewhere, which is the argument for putting these under a test rather than in prose alone. - Nothing offline can do it: the whole `ModelRepo.xml` is not committed, and - `tests/fixtures/catalog/scales.json` carries raw attributes for only a few - dozen parameters. So it happens here, against the catalog the unit is - actually running. A failure is a finding about the device, and it means the - numbers in `CLAUDE.md`, `docs/STEERING.md`, `docs/domain-model.md` and - `changelog.md` need re-deriving before anything else is trusted. + Offline can only do it against a snapshot. Since ADR-0022 the 4.0.1 payload + IS committed, at `tests/fixtures/catalog/model_repo_coros_4_0_1.bin`, so a + count can be taken without a unit - but it is a count against that capture, + not against the catalog this unit is running, and a unit whose content + has moved is exactly what these numbers need checking against. So it happens + here, against the catalog the unit is actually running. A failure is a + finding about the device, and it means the numbers in `CLAUDE.md`, + `docs/STEERING.md`, `docs/domain-model.md` and `changelog.md` need + re-deriving before anything else is trusted. """ placeable = [m for m in live_catalog if not (m.hidden or m.internal or m.category_hidden)] diff --git a/tests/hardware/test_scales_on_unit.py b/tests/hardware/test_scales_on_unit.py index d556cd4c..0d56c367 100644 --- a/tests/hardware/test_scales_on_unit.py +++ b/tests/hardware/test_scales_on_unit.py @@ -2,12 +2,16 @@ `tests/fixtures/catalog/scales.json` holds the bounds, taper and floor of every parameter the offline scale tests assert on. It is distilled from a device -rather than committed whole, so like the generated constants it is a snapshot -and nothing offline can notice it going stale. +rather than committed whole, so like the generated constants it is a snapshot, +and no offline check can notice the UNIT moving away from it. Since ADR-0022 it +could at least be checked against the committed payload, which is that record's +third open question; either way only a unit can say the catalog still reads +this way. A failure here means the fixture is out of date. Regenerate and READ the diff: - python scripts/extract_scale_fixture.py --payload + python scripts/extract_scale_fixture.py \ + --payload tests/fixtures/catalog/model_repo_coros_4_0_1.bin A changed bound or taper is a real protocol change and belongs in `docs/protocol.md` alongside the readings it invalidates. diff --git a/tests/test_catalog_payload.py b/tests/test_catalog_payload.py new file mode 100644 index 00000000..d83e4c46 --- /dev/null +++ b/tests/test_catalog_payload.py @@ -0,0 +1,199 @@ +"""The committed `ModelRepo` payload still generates the committed snapshot. + +`tests/fixtures/catalog/model_repo_coros_4_0_1.bin` is the reply the 4.0.1 unit +sent on 2026-09-22. It exists so a snapshot can be regenerated with no unit +attached (ADR-0022), which is the only way a contributor without that firmware +can touch the generated catalog at all. + +A saved input nothing reads is a file that rots silently. The offline suite +would pass identically if this payload were truncated, from the wrong firmware, +or replaced with noise, and `tests/hardware/test_generated_constants.py` cannot +help: it reads the unit, not this file. So the check that matters is the one +below - regenerate from the payload and compare with what is committed. + +This is NOT the hardware check. That one asks whether the snapshot still matches +the UNIT, and only a unit can answer it. This one asks whether the snapshot +still matches its own recorded input, which is what makes regeneration +reproducible for someone who has no unit. +""" +import functools +import hashlib +import json +import pathlib + +import pytest + +from pyquadcortex.protocol import catalog +from test_generators import _load + +REPO = pathlib.Path(__file__).resolve().parents[1] + +#: `test_generators._load` is the one loader for these scripts, so this file +#: borrows it rather than keeping a second copy. It does NOT cache: it re-execs +#: on every call, and every generator inserts the repo root and `scripts/` at +#: `sys.path[0]` as it runs, two entries each, which nothing removes. +#: +#: The wrapper saves exactly one exec today. This module makes five calls over +#: four distinct names: the three generators once each, and `_snapshots` twice, +#: for `GENERATORS` below and again in the provenance test. It earns more as +#: `PAYLOADS` grows, because every row asks the same names again. +_script = functools.cache(_load) + +#: Each committed payload, the snapshot package it generates, and that +#: catalog's recorded shape as `(models, factory models, factory categories)`. +#: A new profile that saves its payload adds a row here, and is then held the +#: same way. The shape lives in the row rather than in the test so that each +#: payload carries its own. It is filled in from the payload being added, so it +#: cannot fail then; it earns its place later, as the only thing that notices a +#: model the snapshot generates no constants for. See +#: `test_the_payload_still_has_the_shape_its_row_records`. +PAYLOADS = [ + ("coros_4_0_1", + "tests/fixtures/catalog/model_repo_coros_4_0_1.bin", + (533, 414, 22)), +] + +#: Taken from `_snapshots.MODULES`, which with `EXPORTS` beside it is what the +#: snapshot package's own `__init__` is built from, and what +#: `tests/test_generators.py` pins. A fourth generator still has to be added +#: there, but only there, instead of here as well. +GENERATORS = _script("_snapshots").MODULES + + +@pytest.fixture(scope="module", params=PAYLOADS, ids=[row[0] for row in PAYLOADS]) +def payload(request): + snapshot, relative, shape = request.param + path = REPO / relative + assert path.exists(), f"{relative} is missing; it is the snapshot's input" + return snapshot, relative, catalog.parse_model_repo(path.read_bytes()), shape + + +@pytest.mark.parametrize("name", GENERATORS) +def test_the_payload_still_generates_the_committed_snapshot(payload, name): + """Regenerate from the saved payload and compare, byte for byte. + + A failure means the payload and the snapshot have parted company. Either + someone edited a generated file by hand, which nothing may do, or a + generator changed its output and the snapshot was not regenerated with it, + or - for `options.py` alone - a row was added to + `tests/fixtures/catalog/option_readings.json`, which `generate_options` + reads besides the payload. The answer is the same in all three cases. + The fix is to run the generator, never to edit the file: + + python scripts/generate_.py --snapshot \\ + --payload tests/fixtures/catalog/model_repo_.bin + """ + snapshot, relative, parsed, _ = payload + generated = _script(f"generate_{name}").render(parsed, snapshot=snapshot) + committed_path = (REPO / "pyquadcortex" / "protocol" / "catalogs" + / snapshot / f"{name}.py") + committed = committed_path.read_text(encoding="utf-8") + if generated == committed: + return + + gen, com = generated.splitlines(), committed.splitlines() + first = next((i for i, (a, b) in enumerate(zip(gen, com)) if a != b), + min(len(gen), len(com))) + if gen == com: # identical lines, so it is the tail + pytest.fail( + f"{snapshot}/{name}.py has the same lines as the payload generates " + f"but not the same text: {len(committed)} characters committed " + f"against {len(generated)} generated. Suspect the trailing newline.") + pytest.fail( + f"{snapshot}/{name}.py no longer matches what the committed payload " + f"generates. First difference at line {first + 1}:\n" + f" committed: {com[first] if first < len(com) else ''}\n" + f" generated: {gen[first] if first < len(gen) else ''}\n" + f"Regenerate with `python scripts/generate_{name}.py --snapshot " + f"{snapshot} --payload {relative}` and read the diff. " + f"Never edit a generated file by hand.") + + +def test_every_committed_payload_has_a_row(): + """`PAYLOADS` covers every payload in the fixture directory. + + `CLAUDE.md` protects `tests/fixtures/catalog/*.bin` and says this file holds + each one to the snapshot it generates. That is only true while the two + agree: a payload committed with no row here is checked by nothing, while the + rule says it is covered. This is the assertion that keeps the sentence true. + """ + directory = REPO / "tests" / "fixtures" / "catalog" + committed = {p.relative_to(REPO).as_posix() for p in directory.glob("*.bin")} + listed = {relative for _, relative, _ in PAYLOADS} + assert committed == listed, ( + f"payloads with no PAYLOADS row: {sorted(committed - listed)}; " + f"rows with no payload: {sorted(listed - committed)}. Add the row, or " + f"the file is protected by CLAUDE.md and checked by nothing.") + + +def test_the_payload_still_has_the_shape_its_row_records(payload): + """The shape its row records, which is the one thing the comparison misses. + + Nearly everything is caught above: a truncated payload raises while the + fixture parses it, and another unit's catalog fails the byte-for-byte + comparison. One thing is not. The snapshot holds FACTORY content only, by + design and by its own docstrings, so a model this repository generates no + constants for can appear in the catalog without moving a single generated + byte. + + Measured, not supposed: adding one model that carries a `sku` and a + `plugin_id`, whose only parameter is a float and which therefore joins no + option list, leaves `models.py`, `params.py` and `options.py` byte for byte + identical. The model count is the only thing that moves. + + That case is ADR-0022's first open question - whether a unit that has + bought an Archetype publishes anything the maintainer's unit does not - so + this assertion is the guard for it rather than a nicer error message. It is + also a nicer error message: "533 models became 534" names the problem where + "first difference at line 812" would not. + """ + _, _, parsed, (models, factory_models, factory_categories) = payload + factory = [m for m in parsed if m.is_factory] + assert (len(parsed), len(factory)) == (models, factory_models) + assert len({m.category for m in factory}) == factory_categories + + +def test_every_payload_records_which_unit_produced_it(payload): + """A payload cannot say what firmware it came from, so a sibling file does. + + The XML root carries no attributes and the tar member no metadata, so + nothing in the payload states its own origin. Without the record beside it, + "this is the 4.0.1 catalog" rests on the committer's word and no later + reader can check which unit generated the snapshot. + + The record holds the firmware fields of the `Version` reply and leaves out + the serial number, MAC address and custom name. Those identify an owner's + unit and say nothing about the firmware. + """ + snapshot, relative, _, _ = payload + payload_path = REPO / relative + record = payload_path.with_name(payload_path.stem + ".provenance.json") + assert record.exists(), ( + f"{record.relative_to(REPO)} is missing. A payload with no provenance " + f"record cannot be checked against the firmware it claims to be from.") + raw = record.read_text(encoding="utf-8") + data = json.loads(raw) + reply = data["version_reply"] + assert reply["zenos_git_hash"] == _script("_snapshots").snapshot_version(snapshot) + assert reply["device_type"] and reply["app_fw_version"] + + # The record also states the size and digest of the file beside it, and a + # record describing some earlier capture is worse than none. Re-reading the + # catalog from the same unit produces a payload that parses identically, so + # every comparison above still passes while these two go stale in silence. + # This is not comparing two payloads by hash, which ADR-0022 forbids; it is + # checking that a record describes the file it sits next to. + blob = payload_path.read_bytes() + assert data["payload_bytes"] == len(blob), ( + f"{record.name} records {data['payload_bytes']} bytes and " + f"{payload_path.name} is {len(blob)}. Update the record with the payload.") + assert data["payload_sha256"] == hashlib.sha256(blob).hexdigest(), ( + f"{record.name}'s digest is not {payload_path.name}'s. The payload was " + f"replaced without its record; regenerate both together.") + + # Scanned over the whole document, not just `version_reply`: a field moved + # to the top level or pasted into a comment would pass a keys-only check. + for owners in ("device_serial_number", "mac_address", "custom_name"): + assert owners not in raw, ( + f"{owners} identifies a unit's owner, not its firmware; " + f"it does not belong in a committed record")