From 25ab906266fb4f65b152c58f049045bf824551d9 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Thu, 13 Aug 2026 09:24:12 +0100 Subject: [PATCH 1/4] Add gitattributes file Text files normalize to LF in the repository and check out with the platform convention, and graphics assets are marked binary so they are never normalized. The template is extended with the file types this repository uses, including python-version, toml and css. Python sources carry the python diff driver, so diff hunk headers name the enclosing function or method rather than the enclosing class. --- .gitattributes | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d9626d7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,48 @@ +# Auto detect text files and normalize line endings +* text=auto + +# Scripts +*.cmd text eol=crlf +*.sh text eol=lf +*.ps1 text + +# Config +*.gitignore text +*.gitattributes text +*.gitmodules text eol=lf +*.git-blame-ignore-revs text +*.python-version text +*.editorconfig text +*.toml text +*.sln text +*.proj text +*.props text +*.targets text +*.csproj text +*.wixproj text +*.config text +*.json text +*.xml text +*.yml text + +# Code +*.css text +*.py text diff=python +*.manifest text +*.vsixmanifest text +*.vstemplate text +*.resx text +*.cs text +*.bonsai text +*.wxs text + +# Documents +LICENSE text +*.md text diff=markdown +*.rtf diff=astextplain + +# Graphics +*.png binary +*.ico binary +*.gif binary +*.svg text From f0b6a3fb5e119fc569a8868f826b247f18b8386c Mon Sep 17 00:00:00 2001 From: glopesdev Date: Fri, 14 Aug 2026 10:17:47 +0100 Subject: [PATCH 2/4] Update documentation for the device module API Examples and READMEs now pass a device module where they previously passed a Device subclass, and DatasetReader is documented as taking a module rather than a Device. Both stopped being accurate when the device constructor started taking the module as an argument. get_info omits the device argument entirely for schema-free access, since it reads only core registers. Example imports follow the module convention, so each register shows the module it came from at the call site, while protocol primitives stay imported by name where there is no origin to disambiguate. The SERIAL_PORT warning becomes a single fragment included by the four pages that use it. Prose across all markdown drops em-dashes, second-person address and parenthetical asides, and flows one line per paragraph. The contributing section documents pyright, which is what CI runs, rather than ty check, which is not installed. mkdocs drops four extensions that produced no change in the rendered output. --- README.md | 83 ++++++++----------- .../create_device_module.md | 63 +++++--------- .../create_device_module.py | 51 +++++------- docs/examples/get_info/get_info.md | 5 +- docs/examples/get_info/get_info.py | 16 ++-- docs/examples/index.md | 4 +- .../read_and_write_from_registers.md | 5 +- .../read_and_write_from_registers.py | 33 +++----- .../read_data_to_dataframe.md | 9 +- .../read_data_to_dataframe.py | 20 ++--- docs/examples/read_dataset/read_dataset.md | 18 +--- docs/examples/read_dataset/read_dataset.py | 40 ++++----- .../subscribing_to_events.md | 11 ++- .../subscribing_to_events.py | 33 +++----- docs/includes/serial-port.md | 2 + mkdocs.yml | 7 +- src/packages/harp-benchmarks/README.md | 42 ++++------ src/packages/harp-data/README.md | 76 +++++++---------- src/packages/harp-device/README.md | 66 +++++---------- .../src/harp/device/core/_register_map.py | 4 +- src/packages/harp-protocol/README.md | 4 +- src/packages/harp-serial/README.md | 21 ++--- 22 files changed, 242 insertions(+), 371 deletions(-) create mode 100644 docs/includes/serial-port.md diff --git a/README.md b/README.md index ab7070e..f5e292c 100644 --- a/README.md +++ b/README.md @@ -6,22 +6,17 @@ This project includes four main packages: - - **harp-protocol**: Provides the core protocol definitions and utilities for the Harp protocol. - See [Protocol API Documentation](https://harp-tech.org/pyharp/api/protocol) for details. + - **harp-protocol**: Provides the core protocol definitions and utilities for the Harp protocol. See [Protocol API Documentation](https://harp-tech.org/pyharp/api/protocol) for details. - - **harp-serial**: Implements serial communication functionalities for generic Harp devices. - See [Serial API Documentation](https://harp-tech.org/pyharp/api/serial) for more information. + - **harp-serial**: Implements serial communication functionalities for generic Harp devices. See [Serial API Documentation](https://harp-tech.org/pyharp/api/serial) for more information. - - **harp-device**: Implements the transport-agnostic `Device` interface, the common register map, and the shared registers and enums. - See [Device API Documentation](https://harp-tech.org/pyharp/api/device) for details. + - **harp-device**: Implements the transport-agnostic `Device` interface, the common register map, and the shared registers and enums. See [Device API Documentation](https://harp-tech.org/pyharp/api/device) for details. - - **harp-data**: Parses register binary dumps into pandas DataFrames. - See [Data API Documentation](https://harp-tech.org/pyharp/api/data) for more information. + - **harp-data**: Parses register binary dumps into pandas DataFrames. See [Data API Documentation](https://harp-tech.org/pyharp/api/data) for more information. ## Installation -All packages are published to PyPI. The `harp` package is a metadata package with no code of -its own — it just depends on the four packages above, so it's the easiest way to get everything: +All packages are published to PyPI. The `harp` package is a metadata package with no code of its own. It depends on the four packages above, so it is the easiest way to get everything: ```sh pip install harp @@ -31,14 +26,13 @@ pip install harp uv add harp ``` -If you only need part of the stack (e.g. you're parsing offline data dumps and don't need serial -I/O), install just the packages you need — each one only pulls in what it actually depends on: +To install only part of the stack, for example when parsing offline data dumps with no need for serial I/O, install the individual packages. Each one only pulls in what it actually depends on: | Package | Provides | Depends on | | --- | --- | --- | -| `harp-protocol` | Core protocol types: registers, messages, payload parsing | — | +| `harp-protocol` | Core protocol types: registers, messages, payload parsing | none | | `harp-device` | Transport-agnostic `Device` class, common register map | `harp-protocol` | -| `harp-serial` | Serial (COM/tty) transport for `Device` | `harp-protocol`, `harp-device` | +| `harp-serial` | Serial COM or tty transport for `Device` | `harp-protocol`, `harp-device` | | `harp-data` | Parse register binary dumps into pandas DataFrames | `harp-protocol` | ```sh @@ -48,81 +42,74 @@ pip install harp-serial pip install harp-data ``` -`harp-benchmarks` (under `src/packages/`) is internal-only and is never published to PyPI. +`harp-benchmarks`, under `src/packages/`, is internal-only and is never published to PyPI. ## Quickstart -There are two ways you'll typically use `harp`: talking to a **live device** over a -serial connection, or reading **data recorded to disk**. +There are two typical ways to use `harp`: talking to a **live device** over a serial connection, or reading **data recorded to disk**. **Talk to a live device.** Open a connection and read/write registers by class: ```python -from harp.device.core import OperationControl, OperationControlPayload, OperationMode, WhoAmI -from harp.device.client import Device -from harp.serial import open_serial_device +from harp import serial +from harp.device import behavior, core # Use "COMx" on Windows, "/dev/ttyUSBx" on Linux. -with open_serial_device(Device, port="/dev/ttyUSB0") as device: - print("WhoAmI:", device.read(WhoAmI).parsed) - device.write(OperationControl, OperationControlPayload(operation_mode=OperationMode.ACTIVE)) +with serial.open_serial_device(behavior, port="COM3") as device: + print(device.read(core.WhoAmI).parsed) # a common register + print(device.read(behavior.AnalogData).parsed) # a device register + device.write( + core.OperationControl, + core.OperationControlPayload(operation_mode=core.OperationMode.ACTIVE), + ) ``` -**Read a recorded session.** Point a `DatasetReader` at a dataset folder and read -registers into pandas DataFrames — no hardware required: +**Read a recorded session.** Point a `DatasetReader` at a dataset folder and read registers into pandas DataFrames, with no hardware required: ```python -from harp.data import create_dataset_reader +from harp import data # Finds device.yml in the folder, builds the device, returns a ready-to-use reader. -reader = create_dataset_reader("session.harp") -df = reader.read(44) # one register, by address (or pass its class) +reader = data.create_dataset_reader("session.harp") +df = reader.read(44) # one register, by address, or pass its class everything = reader.read_all() # {register_name: DataFrame} ``` -Both paths are driven by a device schema. If you have only a `device.yml` and no -pre-generated package, `create_device_module` compiles it into a module of register classes -at runtime — no code-generation step — which is exactly what `create_dataset_reader` -does under the hood: +Both paths are based on a device schema. Given only a `device.yml` and no pre-generated package, `create_device_module` compiles it into a module of register classes at runtime, with no code-generation step. This is exactly what `create_dataset_reader` does internally: ```python from pathlib import Path -from harp.device.schema import create_device_module -behavior = create_device_module(Path("device.yml").read_bytes()) -AnalogData = behavior.AnalogData # registers are reached by name... -assert behavior.REGISTER_MAP[44] is AnalogData # ...or by address +from harp.device import schema + +behavior = schema.create_device_module(Path("device.yml").read_bytes()) +AnalogData = behavior.AnalogData # registers are reached by name +assert behavior.REGISTER_MAP[44] is AnalogData # or by address ``` -See the [Examples](https://harp-tech.org/pyharp/examples/) for the full walkthroughs, -including subscribing to device events and working with custom interface-type converters. +See the [Examples](https://harp-tech.org/pyharp/examples/) for the full walkthroughs, including subscribing to device events and working with custom interface-type converters. ## Contributing -harp is a [uv workspace](https://docs.astral.sh/uv/concepts/workspaces/): every package under -`src/packages/` is its own distribution, plus the root `harp` metadata package. Contributions are -welcome — please open an issue or PR. +harp is a [uv workspace](https://docs.astral.sh/uv/concepts/workspaces/): every package under `src/packages/` is its own distribution, plus the root `harp` metadata package. Bug reports and contributions are welcome, so please open an issue or pull request. -Clone the repo and install everything (all workspace packages, editable, plus test/lint tooling) -with the `dev` dependency group: +Clone the repository and install everything with the `dev` dependency group: all workspace packages, editable, plus test and lint tooling. ```sh uv sync --group dev ``` -Before opening a PR, run the same checks CI runs: +Before opening a pull request, run the same checks CI runs: ```sh uv run ruff format --check # formatting uv run ruff check # lint -uv run ty check # type checking +uv run pyright # type checking uv run codespell # spelling uv run pytest --cov harp # tests ``` -Adding a new package? Drop it under `src/packages//` with its own `pyproject.toml`, add it -to `[tool.uv.sources]` in the root `pyproject.toml`, and (if it should ship as part of `harp`) add -it to the root package's `dependencies` too. +To add a new package, place it under `src/packages//` with its own `pyproject.toml` and add it to `[tool.uv.sources]` in the root `pyproject.toml`. If it should ship as part of `harp`, add it to the dependencies of the root package as well. ## Building the documentation diff --git a/docs/examples/create_device_module/create_device_module.md b/docs/examples/create_device_module/create_device_module.md index 6a4721b..8c3b8ca 100644 --- a/docs/examples/create_device_module/create_device_module.md +++ b/docs/examples/create_device_module/create_device_module.md @@ -1,51 +1,28 @@ # Generating Registers from a Schema -This example demonstrates how to turn a Harp `device.yml` into a module of register -classes at runtime with `create_device_module`, without a code-generation step. This is the -quickest way to get started when you have only a device's schema and no -pre-generated package for it. - -A generated device package is a module: register classes at module level, with a -`REGISTER_MAP` beside them keyed by address. `create_device_module` builds that same shape -from a schema, so registers are reached the same way either way — by name -(`behavior.AnalogData`) or by address (`behavior.REGISTER_MAP[44]`). From there they -work exactly like a pre-generated package's: drive them over a transport with -[`Device`](../../api/device.md) to talk to hardware, or use them to decode recorded -data. +This example demonstrates how to turn a Harp `device.yml` into a module of register classes at runtime with `create_device_module`, without a code-generation step. This is the quickest way to get started given only the schema of a device and no pre-generated package for it. + +A generated device package is a module: register classes at module level, with a `REGISTER_MAP` beside them keyed by address. `create_device_module` builds that same structure from a schema, so registers are reached the same way, either by name as `behavior.AnalogData` or by address as `behavior.REGISTER_MAP[44]`. From there they work exactly like the registers of a pre-generated package. Pass the module to [`Device`](../../api/device.md) to talk to hardware, which validates the device identity on open, or use the registers with [`parse_to_dataframe`](../../api/data.md) to decode recorded data. ## When to use runtime generation -`create_device_module` trades statically generated device packages for schema-driven -convenience. It's worth understanding what that buys you and what it costs. - -**You gain:** - -- **No build step.** A `device.yml` — even one you just pulled off a device — - becomes a working module in a single call. There's nothing to generate, install, - or keep in sync with the schema. -- **Coverage for any device.** You don't need a published package for the device; - unreleased, custom, or one-off schemas work immediately. -- **The schema stays the single source of truth.** Registers, fields, and enums come - straight from the `device.yml`, under the same naming convention a generated - package uses — so code written against either lines up name for name. - -**You give up:** - -- **Static typing and autocomplete.** The names exist only once the module is built, - so an editor can't offer them and a type checker can't verify them. A generated - package is a real module on disk, so both work. The module also isn't in - `sys.modules`, so you bind it yourself rather than `import`-ing it. -- **Turn-key custom types.** A custom `interfaceType` must be injected yourself via - `converters=` (see below), whereas a generated package ships its own converters. - -For shipped, widely-used devices a pre-generated package from the -[Harp C# generator](https://github.com/harp-tech/generators) remains the -authoritative choice — better editor support and static typing. Reach for -`create_device_module` when you want to go from a schema to working code with no -generation step. - -!!! warning - Don't forget to change the `SERIAL_PORT` to the one that corresponds to your device! The `SERIAL_PORT` must be denoted as `/dev/ttyUSBx` in Linux and `COMx` in Windows, where `x` is the number of the serial port. +`create_device_module` trades statically generated device packages for schema-driven convenience. Both sides of that trade-off are worth understanding. + +**Benefits:** + +- **No build step.** A `device.yml`, even one just pulled off a device, becomes a working module in a single call. There is nothing to generate, install, or keep in sync with the schema. +- **Coverage for any device.** No published package is needed. Unreleased, custom, or one-off schemas work immediately. +- **Names match the generated package.** Registers, fields, and enums come straight from the `device.yml`, under the same naming convention a generated package uses, so code written against either lines up name for name. + +**Limitations:** + +- **Static typing and autocomplete.** The names exist only once the module is built, so an editor cannot offer them and a type checker cannot verify them. A generated package is a real module on disk, so both work. The module is also not in `sys.modules`, so it has to be bound rather than imported. +- **Reproducibility.** A generated package is a versioned dependency, so it can be pinned in a lock file and every install resolves the same register definitions. A runtime module is built from the `device.yml`, so the same analysis code can see different field names when it changes. +- **Turn-key custom types.** A custom `interfaceType` must be injected via `converters=`, shown below, whereas a generated package ships its own converters. + +For widely-used devices a pre-generated package remains the authoritative choice, with better editor support, static typing, and a pinnable version. Reach for `create_device_module` to go from a schema to working code with no code-generation step. + +{% include-markdown "includes/serial-port.md" %} ```python diff --git a/docs/examples/create_device_module/create_device_module.py b/docs/examples/create_device_module/create_device_module.py index 9746aa2..fa593dd 100644 --- a/docs/examples/create_device_module/create_device_module.py +++ b/docs/examples/create_device_module/create_device_module.py @@ -1,48 +1,41 @@ from pathlib import Path -from harp.data import parse_to_dataframe -from harp.device.client import Device -from harp.device.schema import create_device_module -from harp.serial import open_serial_device +from harp import data +from harp import serial +from harp.device import schema -SERIAL_PORT = "/dev/ttyUSB0" # or "COMx" in Windows ("x" is the number of the serial port) +SERIAL_PORT = "/dev/ttyUSB0" # or "COMx" in Windows, where "x" is the serial port number # `create_device_module` compiles a Harp `device.yml` into a module of register classes at -# runtime — no code-generation step. This is the quickest way to work with a device -# when you don't have a pre-generated package for it: point it at the schema and you -# get the same shape a generated package has, registers at module level beside a +# runtime, with no code-generation step. This is the quickest way to work with a device +# that has no pre-generated package: point it at the schema and it produces the same +# structure a generated package has, registers at module level beside a # `REGISTER_MAP`. -behavior = create_device_module(Path("device.yml").read_bytes()) +behavior = schema.create_device_module(Path("device.yml").read_bytes()) print("WhoAmI:", behavior.WHO_AM_I) # device identity, taken from the schema -AnalogData = behavior.AnalogData # registers are reached by name... -assert behavior.REGISTER_MAP[44] is AnalogData # ...or by address +AnalogData = behavior.AnalogData # registers are reached by name +assert behavior.REGISTER_MAP[44] is AnalogData # or by address -# Registers are ordinary register classes, so they drive `read`/`write` on any -# `Device` over a transport. The schema carries no Python code, so there is no -# generated device class here: use `Device` itself. -with open_serial_device(Device, port=SERIAL_PORT) as device: +# Registers are ordinary register classes, so they work with `read` and `write` on +# any `Device` over a transport. Passing the module itself validates the device +# identity on open, against its `WHO_AM_I`, which a value of `0` skips. +with serial.open_serial_device(behavior, port=SERIAL_PORT) as device: print("AnalogData:", device.read(AnalogData).parsed) -# ...or use the same register classes to decode a recorded binary dump into a -# pandas DataFrame (see the "Reading Data into a DataFrame" example for more): -df = parse_to_dataframe(AnalogData, "Behavior_44.bin") +# The same register classes also decode a recorded binary dump into a pandas +# DataFrame. See the "Reading Data into a DataFrame" example for more. +df = data.parse_to_dataframe(AnalogData, "Behavior_44.bin") print(df.head()) -# To have the identity checked on connect, pass the module itself. The check is -# driven by its `WHO_AM_I`, and `0` skips it: -# -# with open_serial_device(behavior, port=SERIAL_PORT) as device: -# print("AnalogData:", device.read(AnalogData).parsed) - # --- Custom interface types -------------------------------------------------- # A register with a custom `interfaceType` needs a converter so its field decodes # to the right Python type. Pass it via `converters=`, keyed by "Converter": # -# behavior = create_device_module(yml_text, converters={"DataConverter": DataConverter()}) +# behavior = schema.create_device_module(yml_text, converters={"DataConverter": DataConverter()}) # -# An unresolved custom type raises `UnknownConverterError`; pass `strict=False` to -# decode it natively instead. `exclude_private=True` (the default) drops registers -# marked `private` in the schema. If you only want the parsed schema model rather -# than a module, `parse_device_schema(yml_text)` returns that directly. +# An unresolved custom type raises `UnknownConverterError`. Pass `strict=False` to +# decode it natively instead. `exclude_private=True`, the default, drops registers +# marked `private` in the schema. For the parsed schema model rather than a module, +# `parse_device_schema(yml_text)` returns that directly. diff --git a/docs/examples/get_info/get_info.md b/docs/examples/get_info/get_info.md index 0ebfe38..3a4ae60 100644 --- a/docs/examples/get_info/get_info.md +++ b/docs/examples/get_info/get_info.md @@ -1,9 +1,8 @@ # Getting Device Info -This example demonstrates how to connect to a Harp device, read its info and dump the device's registers. +This example demonstrates how to connect to a Harp device, read its information and dump the device registers. -!!! warning - Don't forget to change the `SERIAL_PORT` to the one that corresponds to your device! The `SERIAL_PORT` must be denoted as `/dev/ttyUSBx` in Linux and `COMx` in Windows, where `x` is the number of the serial port. +{% include-markdown "includes/serial-port.md" %} ```python diff --git a/docs/examples/get_info/get_info.py b/docs/examples/get_info/get_info.py index 6446c38..2deb17e 100755 --- a/docs/examples/get_info/get_info.py +++ b/docs/examples/get_info/get_info.py @@ -1,15 +1,15 @@ -from harp.device.core import REGISTER_MAP, WhoAmI -from harp.device.client import Device -from harp.serial import open_serial_device +from harp import serial +from harp.device import core -SERIAL_PORT = "/dev/ttyUSB0" # or "COMx" in Windows ("x" is the number of the serial port) +SERIAL_PORT = "/dev/ttyUSB0" # or "COMx" in Windows, where "x" is the serial port number -# Open a serial connection to the device (closed automatically on exit). -with open_serial_device(Device, port=SERIAL_PORT) as device: +# Omitting the device argument gives schema-free access, which skips the identity +# check, so this works against any device. The connection closes on exit. +with serial.open_serial_device(port=SERIAL_PORT) as device: # Identify the device. - print("WhoAmI:", device.read(WhoAmI).parsed) + print("WhoAmI:", device.read(core.WhoAmI).parsed) # Dump every core register. - for address, register in sorted(REGISTER_MAP.items()): + for address, register in sorted(core.REGISTER_MAP.items()): reply = device.read(register) print(f"{register.__name__:24s} (addr {address:2d}) = {reply.parsed}") diff --git a/docs/examples/index.md b/docs/examples/index.md index 0feb522..cf3ca62 100644 --- a/docs/examples/index.md +++ b/docs/examples/index.md @@ -1,6 +1,6 @@ # Examples -This section contains some examples to help you get started with `harp`. +This section contains examples for getting started with `harp`. Working from a device schema: @@ -15,4 +15,4 @@ Talking to a device: Reading recorded data: - [Reading a Whole Dataset Folder](./read_dataset/read_dataset.md) - load an entire recorded session folder into pandas DataFrames with `DatasetReader`. -- [Reading Data into a DataFrame](./read_data_to_dataframe/read_data_to_dataframe.md) - decode a single register's binary file into a pandas DataFrame. +- [Reading Data into a DataFrame](./read_data_to_dataframe/read_data_to_dataframe.md) - decode the binary file of a single register into a pandas DataFrame. diff --git a/docs/examples/read_and_write_from_registers/read_and_write_from_registers.md b/docs/examples/read_and_write_from_registers/read_and_write_from_registers.md index d67f118..dd03ef0 100644 --- a/docs/examples/read_and_write_from_registers/read_and_write_from_registers.md +++ b/docs/examples/read_and_write_from_registers/read_and_write_from_registers.md @@ -1,9 +1,8 @@ # Read and Write from Registers -This example demonstrates how to read and write from registers, using the core registers exposed by `harp.device`. Device-specific registers (e.g. a [Harp Behavior](https://harp-tech.org/api/Harp.Behavior.html)'s digital I/O) are used the same way — pass that device's register classes to `read`/`write`. +This example demonstrates how to read and write from registers, using the core registers exposed by `harp.device.core`. Device-specific registers, for example the digital I/O of a Harp Behavior device, are used the same way. Pass the register classes of that device to `read` and `write`. -!!! warning - Don't forget to change the `SERIAL_PORT` to the one that corresponds to your device! The `SERIAL_PORT` must be denoted as `/dev/ttyUSBx` in Linux and `COMx` in Windows, where `x` is the number of the serial port. +{% include-markdown "includes/serial-port.md" %} ```python diff --git a/docs/examples/read_and_write_from_registers/read_and_write_from_registers.py b/docs/examples/read_and_write_from_registers/read_and_write_from_registers.py index 43bcf29..517fb3d 100755 --- a/docs/examples/read_and_write_from_registers/read_and_write_from_registers.py +++ b/docs/examples/read_and_write_from_registers/read_and_write_from_registers.py @@ -1,35 +1,28 @@ -from harp.device.core import ( - EnableFlag, - OperationControl, - OperationControlPayload, - OperationMode, - WhoAmI, -) -from harp.device.client import Device -from harp.serial import open_serial_device +from harp import serial +from harp.device import client, core -SERIAL_PORT = "/dev/ttyUSB0" # or "COMx" in Windows ("x" is the number of the serial port) +SERIAL_PORT = "/dev/ttyUSB0" # or "COMx" in Windows, where "x" is the serial port number -with open_serial_device(Device, port=SERIAL_PORT) as device: +with serial.open_serial_device(client.Device, port=SERIAL_PORT) as device: # Read a scalar register. - print("WhoAmI:", device.read(WhoAmI).parsed) + print("WhoAmI:", device.read(core.WhoAmI).parsed) # Read a structured register and inspect a field. - control = device.read(OperationControl).parsed + control = device.read(core.OperationControl).parsed print("operation_mode before:", control.operation_mode) # Write the register, then read it back to confirm the change. A struct payload # is built whole, so every field is given a value. device.write( - OperationControl, - OperationControlPayload( - operation_mode=OperationMode.ACTIVE, + core.OperationControl, + core.OperationControlPayload( + operation_mode=core.OperationMode.ACTIVE, dump_registers=False, mute_replies=False, - visual_indicators=EnableFlag.ENABLED, - operation_led=EnableFlag.ENABLED, - heartbeat=EnableFlag.DISABLED, + visual_indicators=core.EnableFlag.ENABLED, + operation_led=core.EnableFlag.ENABLED, + heartbeat=core.EnableFlag.DISABLED, ), ) - control = device.read(OperationControl).parsed + control = device.read(core.OperationControl).parsed print("operation_mode after:", control.operation_mode) diff --git a/docs/examples/read_data_to_dataframe/read_data_to_dataframe.md b/docs/examples/read_data_to_dataframe/read_data_to_dataframe.md index d0637b4..5f47f0b 100644 --- a/docs/examples/read_data_to_dataframe/read_data_to_dataframe.md +++ b/docs/examples/read_data_to_dataframe/read_data_to_dataframe.md @@ -1,14 +1,9 @@ # Reading Data into a DataFrame -This example demonstrates how to load a **single** Harp register's binary data -file into a pandas DataFrame using `harp.data`. The register definition tells -`parse_to_dataframe` how to decode each frame, so you get named columns (and -decoded enums) for free. +This example demonstrates how to load the binary data file of a **single** Harp register into a pandas DataFrame using `harp.data`. The register definition tells `parse_to_dataframe` how to decode each frame, so the result carries named columns and decoded enums. !!! tip - Have a whole recorded session folder rather than one loose file? Use - [`DatasetReader`](../read_dataset/read_dataset.md), which reads every register - in a dataset folder driven by the device schema. + For a whole recorded session folder rather than one loose file, use [`DatasetReader`](../read_dataset/read_dataset.md), which reads every register in a dataset folder based on the device schema. ```python diff --git a/docs/examples/read_data_to_dataframe/read_data_to_dataframe.py b/docs/examples/read_data_to_dataframe/read_data_to_dataframe.py index a08731b..d236fa7 100644 --- a/docs/examples/read_data_to_dataframe/read_data_to_dataframe.py +++ b/docs/examples/read_data_to_dataframe/read_data_to_dataframe.py @@ -1,20 +1,20 @@ -from harp.data import parse_to_dataframe -from harp.device.core import OperationControl +from harp import data +from harp.device import core -# Parse a single register's binary dump into a pandas DataFrame — one row per +# Parse the binary dump of a single register into a pandas DataFrame, one row per # frame, one column per field. The register class tells `parse_to_dataframe` how -# to decode each frame, so you get named columns (and decoded enums) for free. -df = parse_to_dataframe(OperationControl, "OperationControl.bin") +# to decode each frame, so the result carries named columns and decoded enums. +df = data.parse_to_dataframe(core.OperationControl, "OperationControl.bin") print(df.head()) -# When the frames are timestamped (the default), the Harp time becomes the -# DataFrame index, named "Time" — float seconds from device start. +# When the frames are timestamped, which is the default, the Harp time becomes the +# DataFrame index, named "Time", holding float seconds from device start. print(df.index.name, df.index[:3].to_list()) # `parse_to_dataframe` also accepts raw bytes or an open binary file object: with open("OperationControl.bin", "rb") as f: - df = parse_to_dataframe(OperationControl, f) + df = data.parse_to_dataframe(core.OperationControl, f) -# To read a whole recorded session folder at once (many registers, driven by the -# device schema) use `harp.data.DatasetReader` — see the "Reading a Whole Dataset +# To read a whole recorded session folder at once, covering many registers based on +# the device schema, use `harp.data.DatasetReader`. See the "Reading a Whole Dataset # Folder" example. diff --git a/docs/examples/read_dataset/read_dataset.md b/docs/examples/read_dataset/read_dataset.md index 2cd4ea6..63c7f25 100644 --- a/docs/examples/read_dataset/read_dataset.md +++ b/docs/examples/read_dataset/read_dataset.md @@ -1,22 +1,10 @@ # Reading a Whole Dataset Folder -A Harp acquisition is usually saved as a **de-multiplexed dataset folder**: one -binary file per register, named `_
.bin`, next to the device's -`device.yml` schema. `harp.data.DatasetReader` reads that whole folder into pandas -DataFrames, driven by a [generated device](../../api/device.md) that describes how -to decode each register. +A Harp acquisition is usually saved as a **de-multiplexed dataset folder**: one binary file per register, named `_
.bin`, next to the `device.yml` schema for the device. `harp.data.DatasetReader` reads that whole folder into pandas DataFrames, based on a [device module](../../api/device.md) that describes how to decode each register. -This is the recommended entry point when you have a recorded session on disk. To -decode a single loose `.bin` file instead, see -[Reading Data into a DataFrame](../read_data_to_dataframe/read_data_to_dataframe.md). +This is the recommended entry point for a recorded session on disk. To decode a single loose `.bin` file instead, see [Reading Data into a DataFrame](../read_data_to_dataframe/read_data_to_dataframe.md). -The quickest way in is `create_dataset_reader(folder)`: it finds the `device.yml` -inside the folder, builds the device for you, and returns a reader ready to go. -(If you already have a device class — e.g. from a pre-generated package — construct -`DatasetReader(Device, folder)` directly instead.) You then read a register by -class or by address, or read every register at once with `read_all()`. Timestamps -are detected automatically and placed on the `"Time"` index (float seconds, or an -absolute `DatetimeIndex` when you pass an `epoch`). +The quickest way in is `create_dataset_reader(folder)`. It finds the `device.yml` inside the folder, builds the device module, and returns a reader ready to go. Given a device module already in hand, for example from a pre-generated package, construct `DatasetReader(module, folder)` directly instead. A register is then read by class or by address, or every register at once with `read_all()`. Timestamps are detected automatically and placed on the `"Time"` index, as float seconds or an absolute `DatetimeIndex` when an `epoch` is passed. ```python diff --git a/docs/examples/read_dataset/read_dataset.py b/docs/examples/read_dataset/read_dataset.py index f7d769e..7d0d1de 100644 --- a/docs/examples/read_dataset/read_dataset.py +++ b/docs/examples/read_dataset/read_dataset.py @@ -1,9 +1,9 @@ -from harp.data import REFERENCE_EPOCH, create_dataset_reader -from harp.device.core import OperationControl +from harp import data +from harp.device import core -# A Harp acquisition is usually saved as a de-multiplexed dataset folder — one +# A Harp acquisition is usually saved as a de-multiplexed dataset folder, one # `.bin` file per register, named "_
.bin", next to the -# device's `device.yml` schema: +# `device.yml` schema for the device: # # 📦 session.harp # ┣ 📜 Behavior_0.bin @@ -14,14 +14,15 @@ # `create_dataset_reader` does the right thing: it finds `device.yml` inside the # folder, builds the module of register classes that knows how to decode each # register, and hands back a reader ready to go. -reader = create_dataset_reader("session.harp") +reader = data.create_dataset_reader("session.harp") -# Read one register into a DataFrame — by register class (any register in the -# device's map, including the common ones like `OperationControl`)... -df = reader.read(OperationControl) +# Read one register into a DataFrame by register class, which covers any register +# in the device map, including common ones such as `OperationControl`. +df = reader.read(core.OperationControl) -# ...or by address. Timestamps are auto-detected from the frames, and when present -# they become the DataFrame index, named "Time" (float seconds from device start). +# A register can also be read by address. Timestamps are auto-detected from the +# frames, and when present they become the DataFrame index, named "Time", holding +# float seconds from device start. df = reader.read(44) print(df.head()) @@ -30,17 +31,18 @@ print(list(everything)) # Pass an epoch to turn the "Time" index into an absolute `DatetimeIndex` instead -# of float seconds. `REFERENCE_EPOCH` is time zero of the Harp clock (UTC). -absolute = reader.read(44, epoch=REFERENCE_EPOCH) +# of float seconds. `REFERENCE_EPOCH` is time zero of the Harp clock in UTC. +absolute = reader.read(44, epoch=data.REFERENCE_EPOCH) print(absolute.index[:3]) -# --- Already have a device module? ------------------------------------------- -# A pre-generated device package, or one you built yourself with `create_device_module`, -# can drive the reader directly — construct `DatasetReader(module, folder)`: +# --- Working from a device module already in hand ---------------------------- +# A pre-generated device package, or one built with `create_device_module`, +# can be passed to the reader directly as `DatasetReader(module, folder)`: # -# from harp.data import DatasetReader -# from harp.device.schema import create_device_module # from pathlib import Path # -# behavior = create_device_module((Path("session.harp") / "device.yml").read_bytes()) -# reader = DatasetReader(behavior, "session.harp") +# from harp import data +# from harp.device import schema +# +# behavior = schema.create_device_module((Path("session.harp") / "device.yml").read_bytes()) +# reader = data.DatasetReader(behavior, "session.harp") diff --git a/docs/examples/subscribing_to_events/subscribing_to_events.md b/docs/examples/subscribing_to_events/subscribing_to_events.md index 796c4b2..ce11dcb 100644 --- a/docs/examples/subscribing_to_events/subscribing_to_events.md +++ b/docs/examples/subscribing_to_events/subscribing_to_events.md @@ -1,14 +1,13 @@ # Subscribing to Events -This example demonstrates how to react to messages pushed by the device — e.g. unsolicited `Event` messages — without polling, using two subscription styles: +This example demonstrates how to react to messages pushed by the device, e.g. unsolicited `Event` messages, without polling, using two subscription styles: -- `device.subscribe(register, handler)` — the handler receives a typed, parsed `ParsedHarpMessage` for a single register. -- `device.subscribe_all(handler)` — a catch-all handler that receives the raw `HarpMessage` for every register. +- `device.subscribe(register, handler)`, where the handler receives a typed, parsed `ParsedHarpMessage` for a single register. +- `device.subscribe_all(handler)`, a catch-all handler that receives the raw `HarpMessage` for every register. -Handlers run on a dedicated event thread, so they never block `read()`/`write()`. Both methods return a `Subscription`; call `.unsubscribe()` (or use it as a context manager) to stop receiving events. +Handlers run on a dedicated event thread, so they never block `read()` or `write()`. Both methods return a `Subscription`. Call `.unsubscribe()`, or use it as a context manager, to stop receiving events. -!!! warning - Don't forget to change the `SERIAL_PORT` to the one that corresponds to your device! The `SERIAL_PORT` must be denoted as `/dev/ttyUSBx` in Linux and `COMx` in Windows, where `x` is the number of the serial port. +{% include-markdown "includes/serial-port.md" %} ```python diff --git a/docs/examples/subscribing_to_events/subscribing_to_events.py b/docs/examples/subscribing_to_events/subscribing_to_events.py index 25a98cd..e286e25 100644 --- a/docs/examples/subscribing_to_events/subscribing_to_events.py +++ b/docs/examples/subscribing_to_events/subscribing_to_events.py @@ -1,17 +1,10 @@ import numpy as np -from harp.device.core import ( - EnableFlag, - OperationControl, - OperationControlPayload, - OperationMode, - REGISTER_MAP, - TimestampSeconds, -) -from harp.device.client import Device + +from harp import serial +from harp.device import client, core from harp.protocol import HarpMessage, ParsedHarpMessage -from harp.serial import open_serial_device -SERIAL_PORT = "/dev/ttyUSB0" # or "COMx" in Windows ("x" is the number of the serial port) +SERIAL_PORT = "/dev/ttyUSB0" # or "COMx" in Windows, where "x" is the serial port number def print_timestamp(msg: ParsedHarpMessage[np.uint32]) -> None: @@ -19,27 +12,27 @@ def print_timestamp(msg: ParsedHarpMessage[np.uint32]) -> None: def print_any_event(msg: HarpMessage) -> None: - register = REGISTER_MAP.get(msg.address, None) + register = core.REGISTER_MAP.get(msg.address, None) value = register.parse(msg) if register is not None else msg.payload.hex() print(f"[{msg.address}] {msg.timestamp:.6f} {msg.message_type.name:<5s} {value}") -with open_serial_device(Device, port=SERIAL_PORT) as device: +with serial.open_serial_device(client.Device, port=SERIAL_PORT) as device: # Subscribe to a single, typed register: the handler receives a parsed payload. - timestamp_subscription = device.subscribe(TimestampSeconds, print_timestamp) + timestamp_subscription = device.subscribe(core.TimestampSeconds, print_timestamp) # Subscribe to every register at once: the handler receives the raw message. device.subscribe_all(print_any_event) device.write( - OperationControl, - OperationControlPayload( - operation_mode=OperationMode.ACTIVE, + core.OperationControl, + core.OperationControlPayload( + operation_mode=core.OperationMode.ACTIVE, dump_registers=True, - heartbeat=EnableFlag.ENABLED, + heartbeat=core.EnableFlag.ENABLED, mute_replies=False, - operation_led=EnableFlag.ENABLED, - visual_indicators=EnableFlag.ENABLED, + operation_led=core.EnableFlag.ENABLED, + visual_indicators=core.EnableFlag.ENABLED, ), ) diff --git a/docs/includes/serial-port.md b/docs/includes/serial-port.md new file mode 100644 index 0000000..a57d429 --- /dev/null +++ b/docs/includes/serial-port.md @@ -0,0 +1,2 @@ +!!! warning + Do not forget to change the `SERIAL_PORT` to the one that corresponds to the device in use. The `SERIAL_PORT` must be denoted as `/dev/ttyUSBx` in Linux and `COMx` in Windows, where `x` is the number of the serial port. diff --git a/mkdocs.yml b/mkdocs.yml index 23d980e..3a89b65 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,6 +3,9 @@ repo_url: https://github.com/harp-tech/pyharp repo_name: "harp-tech/pyharp" copyright: Copyright (c) harp-tech and Contributors +exclude_docs: | + includes/ + plugins: - search - autorefs @@ -22,16 +25,12 @@ plugins: markdown_extensions: - - abbr - attr_list - admonition - - pymdownx.details - pymdownx.highlight: anchor_linenums: true line_spans: __span pygments_lang_class: true - - pymdownx.inlinehilite - - pymdownx.snippets - pymdownx.superfences - toc: permalink: "#" diff --git a/src/packages/harp-benchmarks/README.md b/src/packages/harp-benchmarks/README.md index beaf6fb..a772d2c 100644 --- a/src/packages/harp-benchmarks/README.md +++ b/src/packages/harp-benchmarks/README.md @@ -1,57 +1,47 @@ # harp-benchmarks -**Internal** parsing-speed benchmarks for the Harp register/payload API, run over every -register defined in [`register_models.py`](src/harp/benchmarks/register_models.py) (the -device.yml coverage model — also imported by the acceptance tests under `tests/`). +**Internal** parsing-speed benchmarks for the Harp register/payload API, run over every register defined in [`register_models.py`](src/harp/benchmarks/register_models.py), the device.yml coverage model, which is also imported by the acceptance tests under `tests/`. -> This package is **not published to PyPI** (`classifiers = ["Private :: Do Not Upload"]` -> in its `pyproject.toml`). It is a workspace member consumed only in dev/internal builds -> via the repo-root `pyproject.toml` `dev` dependency-group. Its console scripts ship with -> this package, so they never leak into the published `harp` distribution. +> This package is **not published to PyPI**, since its `pyproject.toml` sets `classifiers` to `Private :: Do Not Upload`. It is a workspace member consumed only in dev/internal builds via the repo-root `pyproject.toml` `dev` dependency-group. Its console scripts ship with this package, so they never leak into the published `harp` distribution. ## Layout | Path | Purpose | | --- | --- | -| `src/harp/benchmarks/register_models.py` | Reference models for every device.yml register (fixtures shared with the acceptance tests). | -| `src/harp/benchmarks/_registers.py` | Registry: each register + a representative sample value; artifact paths. | -| `src/harp/benchmarks/generate.py` | Writes `./benchmark/data/_.bin`; exposes `ensure_corpus` (cache-aware). | +| `src/harp/benchmarks/register_models.py` | Reference models for every device.yml register, with fixtures shared with the acceptance tests. | +| `src/harp/benchmarks/_registers.py` | Registry: each register plus a representative sample value, and artifact paths. | +| `src/harp/benchmarks/generate.py` | Writes `./benchmark/data/_.bin`, and exposes a cache-aware `ensure_corpus`. | | `src/harp/benchmarks/benchmark.py` | Ensures corpora exist, then times `parse_bulk`, `parse_to_dataframe`, `payload_as_columns`; writes `./benchmark/report.md`. | -All generated artifacts (corpora + report) are written under **`./benchmark`** in the -current working directory — git-ignored and fully regenerable. +All generated artifacts, both corpora and report, are written under **`./benchmark`** in the current working directory, git-ignored and fully regenerable. ## Usage -Console scripts are declared in this package's `pyproject.toml`: +Console scripts are declared in the `pyproject.toml` of this package: ```bash -# One command: generate-if-needed (cache honored), then benchmark + write the report. +# One command: generate if needed, honoring the cache, then benchmark and write the report. uv run harp-benchmark uv run harp-benchmark --runs 20 uv run harp-benchmark --entries 100000 --force # rebuild smaller corpora uv run harp-benchmark --only Version ComplexConfiguration -# Generate corpora explicitly (optional — harp-benchmark does this for you). +# Generate corpora explicitly. Optional, since harp-benchmark does this automatically. uv run harp-benchmark-generate uv run harp-benchmark-generate --entries 100000 ``` -Equivalent module invocations: `uv run python -m harp.benchmarks.benchmark` / -`uv run python -m harp.benchmarks.generate`. +Equivalent module invocations: `uv run python -m harp.benchmarks.benchmark` / `uv run python -m harp.benchmarks.generate`. ## What is measured -- **`parse_bulk`** — the core zero-copy strided-view parse into a `Batch` payload. - This is **lazy**: it builds strided views only and runs **no** converters. -- **`parse_to_dataframe`** — the full path to a pandas `DataFrame` (`copy=False`). -- **`payload_as_columns`** (decode only) — `parse_bulk` views built once up front, then only - `payload.payload_as_columns()` timed. This is where each field's `converter.decode_batch` - actually runs, with no file read and no pandas construction. +- **`parse_bulk`**, the core zero-copy strided-view parse into a `Batch` payload. This is **lazy**: it builds strided views only and runs **no** converters. +- **`parse_to_dataframe`**, the full path to a pandas `DataFrame`, with `copy=False`. +- **`payload_as_columns`**, decode only, with `parse_bulk` views built once up front and then only `payload.payload_as_columns()` timed. This is where the `converter.decode_batch` of each field actually runs, with no file read and no pandas construction. `parse_bulk` and `parse_to_dataframe` are each timed in two modes: -- **pre-read** — file read once up front; only deserialization is timed (isolates library speed). -- **re-read** — file re-read from disk on every run (real-world "load a dump" path, includes disk). +- **pre-read**, file read once up front, so only deserialization is timed. This isolates library speed. +- **re-read**, file re-read from disk on every run, the real-world "load a dump" path, which includes disk. -The report also decomposes `parse_to_dataframe ≈ parse_bulk + payload_as_columns + pandas overhead`. +The report also decomposes `parse_to_dataframe` into `parse_bulk` plus `payload_as_columns` plus pandas overhead. diff --git a/src/packages/harp-data/README.md b/src/packages/harp-data/README.md index 89d0677..accb8b4 100644 --- a/src/packages/harp-data/README.md +++ b/src/packages/harp-data/README.md @@ -1,19 +1,15 @@ # harp-data -Load Harp register data into pandas DataFrames. This is the package that pulls -in `pandas` — [`harp-protocol`](../harp-protocol) stays numpy-only and exposes a -pandas-free `ColumnData` view that this package assembles into a DataFrame. +Load Harp register data into pandas DataFrames. This is the package that pulls in `pandas`. [`harp-protocol`](../harp-protocol) stays numpy-only and exposes a pandas-free `ColumnData` view that this package assembles into a DataFrame. -There are two ways in, depending on what you have on disk: +There are two ways in, depending on what is on disk: -- a whole **dataset folder** (many registers) → `DatasetReader` -- a single **register file** or buffer → `parse_to_dataframe` +- a whole **dataset folder** holding many registers, read with `DatasetReader` +- a single **register file** or buffer, read with `parse_to_dataframe` ## Read a whole dataset folder -A Harp acquisition is usually saved as a de-multiplexed folder — one binary file -per register, named `_
.bin`, alongside the device's -`device.yml` schema: +A Harp acquisition is usually saved as a de-multiplexed folder, one binary file per register, named `_
.bin`, alongside the `device.yml` schema for the device: ```text 📦 session.harp @@ -23,76 +19,64 @@ per register, named `_
.bin`, alongside the device's ┗ 📜 device.yml ``` -Reading is driven by a generated -[`harp.device.client.Device`](../harp-device) that describes how to decode each register. -`create_dataset_reader` does that for you — it finds the `device.yml` in the folder, -builds the device, and returns a ready-to-use reader: +Reading is based on a [device module](../harp-device) that describes how to decode each register. `create_dataset_reader` supplies one automatically. It finds the `device.yml` in the folder, builds the module, and returns a ready-to-use reader: ```python -from harp.data import create_dataset_reader +from harp import data -reader = create_dataset_reader("session.harp") -df = reader.read(AnalogData) # by register class -df = reader.read(44) # by address +reader = data.create_dataset_reader("session.harp") +df = reader.read(AnalogData) # by register class +df = reader.read(44) # by address everything = reader.read_all() # {register_name: DataFrame} ``` -Already have a device module (e.g. a pre-generated package, or one built with -`create_device_module`)? Drive `DatasetReader` with it directly: +Given a device module already in hand, either a pre-generated package or one built with `create_device_module`, pass it to `DatasetReader` directly: ```python from pathlib import Path -from harp.data import DatasetReader -from harp.device.schema import create_device_module -behavior = create_device_module((Path("session.harp") / "device.yml").read_bytes()) -reader = DatasetReader(behavior, "session.harp") +from harp import data +from harp.device import schema + +behavior = schema.create_device_module((Path("session.harp") / "device.yml").read_bytes()) +reader = data.DatasetReader(behavior, "session.harp") ``` -Timestamps are auto-detected per register and placed on the DataFrame index -(named `"Time"`): float seconds by default, or an absolute `DatetimeIndex` when -you pass `epoch=REFERENCE_EPOCH`. Multi-chunk registers logged as -`_
_.bin` are concatenated in filename order; pass a -`resolver` to support an alternative on-disk layout, or `name=` to override the -file prefix. +Timestamps are auto-detected per register and placed on the DataFrame index named `"Time"`: float seconds by default, or an absolute `DatetimeIndex` when `epoch=REFERENCE_EPOCH` is passed. Multi-chunk registers logged as `_
_.bin` are concatenated in filename order; pass a `resolver` to support an alternative on-disk layout, or `name=` to override the file prefix. ## Read a single register file -`parse_to_dataframe` takes a register and a source (path, bytes, or open binary -file) and returns one row per frame: +`parse_to_dataframe` takes a register and a source, either a path, bytes, or an open binary file, and returns one row per frame: ```python -from harp.data import parse_to_dataframe +from harp import data from my_device import AnalogData -df = parse_to_dataframe(AnalogData, "AnalogData.bin") -df = parse_to_dataframe(AnalogData, raw, timestamp=True, message_type=False, decode_enums=True) +df = data.parse_to_dataframe(AnalogData, "AnalogData.bin") +df = data.parse_to_dataframe( + AnalogData, raw, timestamp=True, message_type=False, decode_enums=True +) ``` -With `timestamp=True` (the default) the Harp time becomes the DataFrame index, -named `"Time"` — float seconds, or an absolute `DatetimeIndex` when you also pass -`epoch=REFERENCE_EPOCH`. Enum fields decode to `pd.Categorical` -(`decode_enums=False` keeps raw codes). +With `timestamp=True`, the default, the Harp time becomes the DataFrame index named `"Time"`, as float seconds, or an absolute `DatetimeIndex` when `epoch=REFERENCE_EPOCH` is also passed. Enum fields decode to `pd.Categorical`, and `decode_enums=False` keeps raw codes. ## From an already-parsed payload -If you already have a batched payload (e.g. from `register.parse_bulk`), convert -it directly: +Given a batched payload already in hand, for example from `register.parse_bulk`, convert it directly: ```python -from harp.data import payload_to_dataframe +from harp import data _data, timestamps, _msg, payload = AnalogData.parse_bulk(raw) -df = payload_to_dataframe(payload) +df = data.payload_to_dataframe(payload) ``` ## Write data back out -`to_file` / `to_buffer` are the inverse of the readers — encode values as Harp -frames. Handy for round-tripping data or generating test corpora: +`to_file` and `to_buffer` are the inverse of the readers, encoding values as Harp frames. Useful for round-tripping data or generating test corpora: ```python -from harp.data import to_file +from harp import data -to_file(AnalogData, values, "AnalogData.bin", timestamps=seconds) +data.to_file(AnalogData, values, "AnalogData.bin", timestamps=seconds) ``` diff --git a/src/packages/harp-device/README.md b/src/packages/harp-device/README.md index 4f16f99..d0dcf0d 100644 --- a/src/packages/harp-device/README.md +++ b/src/packages/harp-device/README.md @@ -1,28 +1,22 @@ # harp-device -The transport-agnostic device layer for the Harp protocol: the common Harp -registers and a `Device` base that handles framing, request/reply and register -access. It depends only on [`harp-protocol`](../harp-protocol) — no transport -dependencies. Pair it with a transport (e.g. [`harp-serial`](../harp-serial)). +The transport-agnostic device layer for the Harp protocol: the common Harp registers and a `Device` base that handles framing, request/reply and register access. It depends only on [`harp-protocol`](../harp-protocol), with no transport dependencies. Pair it with a transport such as [`harp-serial`](../harp-serial). ## Read/write registers -A `Device` is driven over a transport; `read`/`write` take a register class: +A `Device` operates over a transport. `read` and `write` take a register class: ```python -from harp.device.core import OperationControl, WhoAmI -from harp.device.client import Device +from harp.device import core -# `device` is a Device opened over some transport (see harp-serial) -who = device.read(WhoAmI).parsed # -> np.uint16 -device.write(OperationControl, payload) # write a register +# `device` is a Device opened over some transport, see harp-serial +who = device.read(core.WhoAmI).parsed # -> np.uint16 +device.write(core.OperationControl, payload) # write a register ``` ## Extending for a specific device -A device is described by a module. Downstream, often generated, packages record the -device identity as `WHO_AM_I`, declare the register classes at module level, and expand -the core `REGISTER_MAP` beside them: +A device is described by a module. Downstream, often generated, packages record the device identity as `WHO_AM_I`, declare the register classes at module level, and expand the core `REGISTER_MAP` beside them: ```python from harp.device.core import REGISTER_MAP as _CORE_REGISTER_MAP @@ -31,17 +25,11 @@ WHO_AM_I: int = 1216 REGISTER_MAP = {**_CORE_REGISTER_MAP, 32: DigitalInputState, ...} ``` -This is the same structure `create_device_module` builds from a schema, so a device -reads the same way whether it was generated ahead of time or compiled at runtime. A -`WHO_AM_I` of `0` marks an unregistered device, used while a device is in development -or outside the official registry, and identity checks are skipped for it. +This is the same structure `create_device_module` builds from a schema, so a device reads the same way whether it was generated ahead of time or compiled at runtime. A `WHO_AM_I` of `0` marks an unregistered device, used while a device is in development or outside the official registry, and identity checks are skipped for it. -A device module names only the registers its schema declares, so `REGISTER_MAP` is the -device address space while the module namespace is what the device adds to it. The -common registers have a single definition, in `harp.device.core`, and -are not a device, so the core register set carries no `WHO_AM_I`. +A device module names only the registers its schema declares, so `REGISTER_MAP` is the device address space while the module namespace is what the device adds to it. The common registers have a single definition, in `harp.device.core`, and are not a device, so the core register set carries no `WHO_AM_I`. -Pass the module to `Device` (or `open_serial_device`) to validate identity on open: +Pass the module to `Device`, or to `open_serial_device`, to validate identity on open: ```python from harp.device import behavior, client, core @@ -51,44 +39,30 @@ with client.Device(transport, behavior) as device: device.read(behavior.DigitalInputState) # declared by the schema ``` -`WHO_AM_I` in the module drives the check; `0` skips it. Omitting the module skips -validation. The module is not otherwise consulted: registers reach `read`, `write` and -`subscribe` as arguments either way, and only a subscribed register is parsed on -arrival. Common registers such as `WhoAmI` and `OperationControl` come from -`harp.device.core` and are read the same way. +The `WHO_AM_I` in the module determines the check, and `0` skips it. Omitting the module skips validation. The module is not otherwise consulted: registers reach `read`, `write` and `subscribe` as arguments either way, and only a subscribed register is parsed on arrival. Common registers such as `WhoAmI` and `OperationControl` come from `harp.device.core` and are read the same way. -A new transport is just an object implementing the `ITransport` protocol -(`open`/`write`/`read`/`close`). +A new transport is just an object implementing the `ITransport` protocol, with `open`, `write`, `read` and `close`. ## Generating registers from a `device.yml` -Without a pre-generated device package, `create_device_module` builds the same -structure at runtime from Harp `device.yml` text: register classes at module level, a -`REGISTER_MAP` beside them, and the identity declared by the schema as `WHO_AM_I`. -Identifiers match a generated package name for name: register, enum, and payload class -names come from the yml verbatim, payload fields are `snake_case`, and enum members are -`SCREAMING_SNAKE_CASE`. +Without a pre-generated device package, `create_device_module` builds the same structure at runtime from Harp `device.yml` text: register classes at module level, a `REGISTER_MAP` beside them, and the identity declared by the schema as `WHO_AM_I`. Identifiers match a generated package name for name: register, enum, and payload class names come from the yml verbatim, payload fields are `snake_case`, and enum members are `SCREAMING_SNAKE_CASE`. ```python from pathlib import Path -from harp.device.schema import create_device_module -behavior = create_device_module(Path("device.yml").read_bytes()) +from harp.device import schema + +behavior = schema.create_device_module(Path("device.yml").read_bytes()) reg = behavior.AnalogData # by name reg = behavior.REGISTER_MAP[44] # or by address ``` -The module is not registered in `sys.modules`, so it has to be bound rather than -imported. Names come from the schema at runtime, so they don't autocomplete and -aren't statically checked. A generated package on disk gives both. +The module is not registered in `sys.modules`, so it has to be bound rather than imported. Names come from the schema at runtime, so they don't autocomplete and aren't statically checked. A generated package on disk gives both. -For a custom `interfaceType`, pass its converter via `converters=` (keyed by -`{InterfaceType}Converter` / `{MemberName}Converter`); an unresolved custom type -raises `UnknownConverterError`, or pass `strict=False` to decode it natively: +For a custom `interfaceType`, pass its converter via `converters=`, keyed by `{InterfaceType}Converter` or `{MemberName}Converter`. An unresolved custom type raises `UnknownConverterError`, or pass `strict=False` to decode it natively: ```python -create_device_module(yml_text, converters={"DataConverter": DataConverter()}) +schema.create_device_module(yml_text, converters={"DataConverter": DataConverter()}) ``` -`parse_device_schema(yml_text)` is also public, returning the parsed schema model -without a module: registers, masks, and optional device identity. +`parse_device_schema(yml_text)` is also public, returning the parsed schema model without a module: registers, masks, and optional device identity. diff --git a/src/packages/harp-device/src/harp/device/core/_register_map.py b/src/packages/harp-device/src/harp/device/core/_register_map.py index bc532dc..560bd83 100644 --- a/src/packages/harp-device/src/harp/device/core/_register_map.py +++ b/src/packages/harp-device/src/harp/device/core/_register_map.py @@ -1,8 +1,8 @@ -"""Address → register-class map for the core Harp registers. +"""Address to register-class map for the core Harp registers. Downstream device packages spread this into their own map:: - from harp.device import REGISTER_MAP as _CORE_REGISTER_MAP + from harp.device.core import REGISTER_MAP as _CORE_REGISTER_MAP REGISTER_MAP = {**_CORE_REGISTER_MAP, 32: DigitalInputState, ...} """ diff --git a/src/packages/harp-protocol/README.md b/src/packages/harp-protocol/README.md index 8f7b044..395955e 100644 --- a/src/packages/harp-protocol/README.md +++ b/src/packages/harp-protocol/README.md @@ -4,7 +4,7 @@ The Harp Protocol is a binary communication protocol created in order to facilitate and unify the interaction between different devices. It was designed with efficiency and ease of parsing in mind. -For more detail please check Harp Tech's official documentation [here](https://harp-tech.org/protocol/BinaryProtocol-8bit.html). +For more detail please check the [official Harp Tech documentation](https://harp-tech.org/protocol/BinaryProtocol-8bit.html). `harp-protocol` provides the building blocks: message framing and the typed register/payload DSL. Each register knows how to build (`format`) and decode (`parse`) its frames. @@ -30,4 +30,4 @@ np.uint16(65535) + 1 # RuntimeWarning: overflow encountered in scalar add Numpy scalars behave like plain Python numbers in arithmetic, comparison and formatting. Use `int()` or `float()` where a built-in type is required. -It carries no transport or device logic — see [`harp-device`](../harp-device) for the device layer. +It carries no transport or device logic. See [`harp-device`](../harp-device) for the device layer. diff --git a/src/packages/harp-serial/README.md b/src/packages/harp-serial/README.md index b98b2af..2507e47 100644 --- a/src/packages/harp-serial/README.md +++ b/src/packages/harp-serial/README.md @@ -1,22 +1,19 @@ # harp-serial -Serial transport for [`harp-device`](../harp-device). Provides `SerialTransport` -and the `open_serial_device` factory, which pairs a `Device` class with a serial -port. This is the package that pulls in `pyserial`. +Serial transport for [`harp-device`](../harp-device). Provides `SerialTransport` and the `open_serial_device` factory, which pairs a device module or a `Device` class with a serial port. This is the package that pulls in `pyserial`. ## Usage -Like the builtin `open`, the returned device is connected and ready; use it in a -`with` block for guaranteed cleanup: +Like the builtin `open`, the returned device is connected and ready. Use it in a `with` block for guaranteed cleanup: ```python -from harp.device.core import WhoAmI -from harp.device.client import Device -from harp.serial import open_serial_device +from harp import serial +from harp.device import behavior, core -with open_serial_device(Device, port="COM3", baudrate=1_000_000) as dev: - print(dev.read(WhoAmI).parsed) +# Use "COMx" on Windows, "/dev/ttyUSBx" on Linux. +with serial.open_serial_device(behavior, port="COM3") as device: + print(device.read(core.WhoAmI).parsed) # a common register + print(device.read(behavior.AnalogData).parsed) # a device register ``` -Pass any `Device` subclass (e.g. a generated device class) instead of the base -`Device` to talk to a specific device. +Passing a device module validates the device identity on open. Pass a `Device` subclass instead to preserve its own type, or omit the argument entirely for schema-free access, which skips the identity check. From f367da6890acb03432c03d698c9eba4c670bf4c9 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Fri, 14 Aug 2026 12:29:29 +0100 Subject: [PATCH 3/4] Document declarations and sweep source prose The benchmark report is now ASCII, so the UTF-8 console shim that both entry points carried is removed, along with the sys import it needed in generate. The report is written correctly under the platform default console encoding rather than depending on stdout being reconfigured. Declaration comments become docstrings below the name they document, replacing Sphinx #: markers and comments placed above. Griffe reads the docstring form and neither of the others, so these now reach the API pages, editor tooltips and the generated JSON schema. The device.yml model also gains a description on every field, taken from the upstream protocol schema. Docstrings and comments across the five packages and the test suite drop em-dashes, noun possessives, parenthetical asides and second-person address. The state-machine separators in the framer become ASCII rules. --- src/packages/harp-benchmarks/pyproject.toml | 6 +- .../src/harp/benchmarks/_registers.py | 4 +- .../src/harp/benchmarks/benchmark.py | 39 ++-- .../src/harp/benchmarks/generate.py | 17 +- .../src/harp/benchmarks/register_models.py | 24 +-- .../harp-data/src/harp/data/_dataset.py | 20 +- src/packages/harp-data/src/harp/data/_read.py | 4 +- .../harp-data/src/harp/data/_reader.py | 2 +- .../harp-data/src/harp/data/_write.py | 4 +- .../src/harp/device/client/_device.py | 10 +- .../src/harp/device/client/_framer.py | 8 +- .../src/harp/device/schema/_emit.py | 55 +++--- .../src/harp/device/schema/_model.py | 181 ++++++++++++++---- .../src/harp/device/schema/_module.py | 13 +- .../src/harp/device/schema/_naming.py | 18 +- .../src/harp/protocol/_constants.py | 14 +- .../src/harp/protocol/_message_type.py | 3 +- .../src/harp/protocol/_payload.py | 145 +++++++------- .../src/harp/protocol/_payload_converters.py | 4 +- .../src/harp/protocol/_register.py | 41 ++-- tests/conftest.py | 2 +- tests/device/test_emit.py | 32 ++-- tests/device/test_naming.py | 6 +- tests/protocol/test_converter.py | 10 +- tests/protocol/test_framer.py | 2 +- tests/protocol/test_payload.py | 4 +- tests/protocol/test_register.py | 8 +- 27 files changed, 386 insertions(+), 290 deletions(-) diff --git a/src/packages/harp-benchmarks/pyproject.toml b/src/packages/harp-benchmarks/pyproject.toml index 7ff7391..f0a1b6c 100644 --- a/src/packages/harp-benchmarks/pyproject.toml +++ b/src/packages/harp-benchmarks/pyproject.toml @@ -3,9 +3,9 @@ name = "harp-benchmarks" dynamic = ["version"] description = "Internal parsing-speed benchmarks for the Harp register/payload API." requires-python = ">=3.11" -# INTERNAL ONLY — never published to PyPI. The "Private :: Do Not Upload" trove -# classifier is rejected by PyPI (and twine/uv publish), so an accidental upload -# of this distribution fails fast. +# INTERNAL ONLY, never published to PyPI. The "Private :: Do Not Upload" trove +# classifier is rejected by PyPI, and by twine and uv publish, so an accidental +# upload of this distribution fails fast. classifiers = ["Private :: Do Not Upload"] dependencies = [ "harp-protocol", diff --git a/src/packages/harp-benchmarks/src/harp/benchmarks/_registers.py b/src/packages/harp-benchmarks/src/harp/benchmarks/_registers.py index 461cb58..7e59e79 100644 --- a/src/packages/harp-benchmarks/src/harp/benchmarks/_registers.py +++ b/src/packages/harp-benchmarks/src/harp/benchmarks/_registers.py @@ -4,7 +4,7 @@ Both ``generate.py`` (writes the .bin corpora) and ``benchmark.py`` (times parsing) import :data:`BENCHMARK_REGISTERS` from here so the two stay in lock-step. Payloads are synthesized as random bytes per frame at generation time (see ``generate.py``), -so no sample values live here — only the register class and its frame shape. +so no sample values live here, only the register class and its frame shape. All generated artifacts live under ``./benchmark`` in the current working directory. """ @@ -53,7 +53,7 @@ def filename(self) -> str: def _base_registers() -> list[BenchmarkedRegister]: - """One (timestamped) fixture per register — :func:`_build` derives the untimestamped twin. + """One timestamped fixture per register, from which :func:`_build` derives the untimestamped twin. The set spans the full spread of payload shapes the Harp protocol allows: trivial scalars, struct payloads with byte gaps, masked sub-fields, custom converters, and diff --git a/src/packages/harp-benchmarks/src/harp/benchmarks/benchmark.py b/src/packages/harp-benchmarks/src/harp/benchmarks/benchmark.py index da3c684..196e9e3 100644 --- a/src/packages/harp-benchmarks/src/harp/benchmarks/benchmark.py +++ b/src/packages/harp-benchmarks/src/harp/benchmarks/benchmark.py @@ -42,7 +42,7 @@ def mib_per_s(self) -> float: def _time(fn: Callable[[], object], *, runs: int, frames: int, file_bytes: int) -> TimingStats: - fn() # warm-up (imports, caches, first-touch pages) — not measured + fn() # warm-up for imports, caches and first-touch pages, not measured samples: list[float] = [] for _ in range(runs): t0 = perf_counter() @@ -99,9 +99,9 @@ def benchmark_register(reg: BenchmarkedRegister, path: Path, *, runs: int) -> Re frames=frames, file_bytes=file_bytes, ) - # Decode only: pre-parse the bulk views once, then time payload_as_columns() alone — - # this is where every converter's decode_batch runs, with no file read and no - # pandas DataFrame construction. Matches parse_to_dataframe's decode options. + # Decode only: pre-parse the bulk views once, then time payload_as_columns() alone, + # this is where the decode_batch of every converter runs, with no file read and no + # pandas DataFrame construction. Matches the decode options of parse_to_dataframe. _, _, _, payload = register.parse_bulk(raw, parse_timestamp=True) cols = _time( lambda: payload.payload_as_columns(decode_enums=True, demux_bit_masks=False), @@ -197,8 +197,8 @@ def _table( _table( "`parse_bulk` (core zero-copy parse)", "`pre` = parse a pre-read buffer; `re` = re-read the file from disk each run. " - "Note `parse_bulk` only builds lazy strided views — it does **not** run " - "converters — so timings are near-uniform regardless of payload shape.", + "Note `parse_bulk` only builds lazy strided views and does **not** run " + "converters, so timings are near-uniform regardless of payload shape.", lambda r: (r.bulk_preread, r.bulk_reread), ) _table( @@ -207,14 +207,14 @@ def _table( lambda r: (r.df_preread, r.df_reread), ) - # Decode-only table (single mode): payload_as_columns() runs every field's + # Decode-only table (single mode): payload_as_columns() runs the converter of every field, # converter.decode_batch, with no file read and no pandas construction. - lines.append("## `payload_as_columns` (decode only — where converters run)\n") + lines.append("## `payload_as_columns`, decode only, where converters run\n") lines.append( "Isolates the decode step: `parse_bulk` views are built once up front, then " - "only `payload.payload_as_columns()` is timed. This is where each field's " + "only `payload.payload_as_columns()` is timed. This is where the " "`converter.decode_batch` executes. Registers whose converters loop in Python " - "(`HarpVersionConverter`, `StringConverter`, `BytesToIntConverter` → object " + "(`HarpVersionConverter`, `StringConverter`, `BytesToIntConverter` -> object " "dtype) dominate here; vectorized converters stay cheap.\n" ) lines.append("| Register | Frames | mean (ms) | min (ms) | stdev (ms) | Mframes/s | MiB/s |") @@ -227,11 +227,12 @@ def _table( ) lines.append("") - # Decomposition: parse_to_dataframe(pre) ≈ parse_bulk(pre) + payload_as_columns + pandas. + # Decomposition: parse_to_dataframe(pre) is about parse_bulk(pre) + payload_as_columns + pandas. lines.append("## Decomposition (pre-read means, ms)\n") lines.append( - "`parse_to_dataframe` ≈ `parse_bulk` (build views) + `payload_as_columns` (decode) + " - "pandas DataFrame construction. The residual column is `df − bulk − payload_as_columns`, " + "`parse_to_dataframe` is approximately `parse_bulk` to build views, plus " + "`payload_as_columns` to decode, plus " + "pandas DataFrame construction. The residual column is `df - bulk - payload_as_columns`, " "i.e. the pandas/column-assembly overhead. Note the three terms are timed in " "separate loops, so for converter-dominated registers (large mean, large stdev) " "the residual is within noise and can even go slightly negative.\n" @@ -275,17 +276,7 @@ def _prepare_corpora(selected, *, entries: int, force: bool, data_dir: Path) -> print() -def _use_utf8_console() -> None: - """Best-effort: make console output UTF-8 (the docstrings/report use ≈, →, −).""" - for stream in (sys.stdout, sys.stderr): - try: - stream.reconfigure(encoding="utf-8") # type: ignore[union-attr] - except (AttributeError, ValueError): - pass - - def main() -> None: - _use_utf8_console() parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--runs", type=int, default=10, help="repeats per measurement (default: 10)" @@ -312,7 +303,7 @@ def main() -> None: parser.add_argument( "--head", action="store_true", - help="print head(5) of each register's DataFrame to stdout (not saved to the report)", + help="print head(5) of the DataFrame of each register to stdout (not saved to the report)", ) args = parser.parse_args() diff --git a/src/packages/harp-benchmarks/src/harp/benchmarks/generate.py b/src/packages/harp-benchmarks/src/harp/benchmarks/generate.py index 222dd58..e222df6 100644 --- a/src/packages/harp-benchmarks/src/harp/benchmarks/generate.py +++ b/src/packages/harp-benchmarks/src/harp/benchmarks/generate.py @@ -1,5 +1,4 @@ import argparse -import sys from pathlib import Path import numpy as np @@ -18,8 +17,8 @@ def _frames(reg: BenchmarkedRegister, entries: int) -> np.ndarray: """Build ``entries`` frames of ``reg`` with a random per-frame payload. Bytes are held to the ASCII range (0..127) so every field varies while staying - valid for any ``StringConverter`` member and free of float NaN/inf — the corpus is - decoded (``to_columns`` / ``parse_to_dataframe``) during the benchmark. Timestamps, + valid for any ``StringConverter`` member and free of float NaN/inf, since the corpus is + decoded through ``to_columns`` and ``parse_to_dataframe`` during the benchmark. Timestamps, when present, are a monotonic ramp. Returns the flat uint8 wire buffer. """ dtype = reg.register.payload_class.payload_dtype @@ -45,7 +44,7 @@ def ensure_corpus( ) -> tuple[object, bool]: """Generate ``reg``'s corpus unless a matching cached file already exists. - The cache is honored only when the existing file's size matches ``entries`` + The cache is honored only when the size of the existing file matches ``entries`` exactly (stride * entries); a stale file (different entry count) is rebuilt. Returns (path, generated). """ @@ -58,15 +57,6 @@ def ensure_corpus( return path, True -def _use_utf8_console() -> None: - """Best-effort: make console output UTF-8 (docstrings use non-ASCII glyphs).""" - for stream in (sys.stdout, sys.stderr): - try: - stream.reconfigure(encoding="utf-8") # type: ignore[union-attr] - except (AttributeError, ValueError): - pass - - def _select(only): selected = BENCHMARK_REGISTERS if only: @@ -95,7 +85,6 @@ def main() -> None: default=DATA_DIR, help=f"directory to write corpus files into (default: {DATA_DIR})", ) - _use_utf8_console() args = parser.parse_args() selected = _select(args.only) diff --git a/src/packages/harp-benchmarks/src/harp/benchmarks/register_models.py b/src/packages/harp-benchmarks/src/harp/benchmarks/register_models.py index 510a18c..f8c1c2e 100644 --- a/src/packages/harp-benchmarks/src/harp/benchmarks/register_models.py +++ b/src/packages/harp-benchmarks/src/harp/benchmarks/register_models.py @@ -34,7 +34,7 @@ class PortDigitalIOS(enum.IntFlag): - """device.yml bitMasks.PortDigitalIOS (bits up to 0x800 — see PortDIOSet).""" + """device.yml bitMasks.PortDigitalIOS, with bits up to 0x800. See PortDIOSet.""" DIO0 = 0x1 DIO1 = 0x2 @@ -63,7 +63,7 @@ class EncoderModeMask(enum.IntEnum): # =========================================================================== -# Custom interfaceType converters — byte-based, register-element-agnostic. +# Custom interfaceType converters, byte-based and register-element-agnostic. # =========================================================================== @@ -101,7 +101,7 @@ class DigitalInputs(RegisterU8): # =========================================================================== -# 33 AnalogData : Float[6], Event — named sub-views + a 3-float sub-array. +# 33 AnalogData : Float[6], Event. Named sub-views plus a 3-float sub-array. # =========================================================================== @@ -121,7 +121,7 @@ class AnalogData(RegisterBase[AnalogDataPayload]): # =========================================================================== -# 34 ComplexConfiguration : U8[17], Write — byte gap at bytes 1..3. +# 34 ComplexConfiguration : U8[17], Write. Byte gap at bytes 1..3. # =========================================================================== @@ -140,7 +140,7 @@ class ComplexConfiguration(RegisterBase[ComplexConfigurationPayload]): # =========================================================================== -# 35 Version : U8[32], Event — HarpVersion x3 (3-byte) + string + raw hash. +# 35 Version : U8[32], Event. HarpVersion x3 (3-byte), string, and raw hash. # =========================================================================== @@ -161,7 +161,7 @@ class Version(RegisterBase[VersionPayload]): # =========================================================================== -# 36 / 37 CustomPayload / CustomRawPayload : U32[3] — register-level +# 36 / 37 CustomPayload / CustomRawPayload : U32[3]. Register-level # interfaceType HarpVersion. Single full-span member -> parse() unwraps. # =========================================================================== @@ -187,7 +187,7 @@ class CustomRawPayload(RegisterBase[HarpVersion]): # =========================================================================== -# 38 CustomMemberConverter : U8[3], Read — Header (uint8) + Data (2 bytes -> int). +# 38 CustomMemberConverter : U8[3], Read. Header (uint8) and Data (2 bytes -> int). # =========================================================================== @@ -203,7 +203,7 @@ class CustomMemberConverter(RegisterBase[CustomMemberConverterPayload]): # =========================================================================== -# 39 BitmaskSplitter : U8, Write — Low (mask 0xF, int) + High (mask 0xF0, int). +# 39 BitmaskSplitter : U8, Write. Low (mask 0xF, int) and High (mask 0xF0, int). # =========================================================================== @@ -228,7 +228,7 @@ class Counter0(RegisterS32): # =========================================================================== -# 41 PortDIOSet : U8, Write — bitMask PortDigitalIOS. A single BitMask over the +# 41 PortDIOSet : U8, Write. bitMask PortDigitalIOS, a single BitMask over the # whole byte; bits >= 0x100 can't fit a U8 so they are dropped. Single-member # -> parse() unwraps to a bare PortDigitalIOS. # =========================================================================== @@ -258,7 +258,7 @@ class PulseDO0(RegisterU16): # =========================================================================== -# 100 StartPulse : U16, Write — two overlapping views of one word. +# 100 StartPulse : U16, Write. Two overlapping views of one word. # =========================================================================== @@ -274,7 +274,7 @@ class StartPulse(RegisterBase[StartPulsePayload]): # =========================================================================== -# 101 StartPulseTrain : U16[2], Write — 4 masked members across two words. +# 101 StartPulseTrain : U16[2], Write. 4 masked members across two words. # =========================================================================== @@ -294,7 +294,7 @@ class StartPulseTrain(RegisterBase[StartPulseTrainPayload]): # =========================================================================== -# 103 EncoderMode : U8, Write — whole-register groupMask. +# 103 EncoderMode : U8, Write. Whole-register groupMask. # =========================================================================== diff --git a/src/packages/harp-data/src/harp/data/_dataset.py b/src/packages/harp-data/src/harp/data/_dataset.py index 26094f2..bb8a49b 100644 --- a/src/packages/harp-data/src/harp/data/_dataset.py +++ b/src/packages/harp-data/src/harp/data/_dataset.py @@ -16,8 +16,8 @@ FileNameResolver = Callable[[Path, str], Mapping[int, list[Path]]] -#: Default filename of the device schema looked up inside a dataset folder. DEVICE_SCHEMA_FILENAME = "device.yml" +"""Default filename of the device schema looked up inside a dataset folder.""" def default_file_resolver(root: Path, name: str) -> dict[int, list[Path]]: @@ -34,8 +34,8 @@ def default_file_resolver(root: Path, name: str) -> dict[int, list[Path]]: class DatasetReader: """Reader over a de-multiplexed Harp dataset folder. - Construct from a device module and a dataset folder, then read a register's - frames into a DataFrame by register class or by address:: + Construct from a device module and a dataset folder, then read the + frames of a register into a DataFrame by register class or by address:: reader = DatasetReader(behavior, "session.harp") df = reader.read(behavior.AnalogData) # by register class @@ -103,12 +103,12 @@ def read( decode_enums: bool = True, demux_bit_masks: bool = False, ) -> pd.DataFrame: - """Read one register's data into a DataFrame. + """Read the data of one register into a DataFrame. ``register`` is a register class or an address. ``suffix`` selects a single ``_
_.bin`` chunk (default: concatenate every chunk - for the address). ``timestamp`` defaults to ``None`` — auto-detect from the - frame's payload-type bit; pass ``True``/``False`` to force. ``epoch`` makes + for the address). ``timestamp`` defaults to ``None``, auto-detecting from the + payload-type bit of the frame; pass ``True``/``False`` to force. ``epoch`` makes the ``"Time"`` index absolute (e.g. :data:`~harp.data.REFERENCE_EPOCH`). The remaining options match :func:`~harp.data.parse_to_dataframe`. """ @@ -137,7 +137,7 @@ def read_all( ) -> dict[str, pd.DataFrame]: """Read every register that has a file present, keyed by register name. - Files whose address is not in the device's registers are skipped. + Files whose address is not among the device registers are skipped. Options are forwarded to :meth:`read`. """ registers = self.registers @@ -161,7 +161,7 @@ def _resolve(self, register: RegisterKey) -> tuple[type[RegisterBase[Any]], int] return register, register.address cls = self.registers.get(register) if cls is None: - raise KeyError(f"No register at address {register} in this device's map.") + raise KeyError(f"No register at address {register} in the map of this device.") return cls, register def _resolve_files(self, address: int, suffix: str | None) -> list[Path]: @@ -206,8 +206,8 @@ def create_dataset_reader( ``schema`` points at the schema file explicitly when it isn't ``root/device.yml``. ``converters`` and ``strict`` are forwarded to :func:`~harp.device.schema.create_device_module` for custom ``interfaceType`` decoding; ``name`` and ``resolver`` are forwarded to - :class:`DatasetReader`. Use ``DatasetReader(device_module, root)`` directly when you - already have a (e.g. pre-generated) device module. + :class:`DatasetReader`. Use ``DatasetReader(device_module, root)`` directly given a + device module already in hand, for example a pre-generated one. """ root_path = Path(root) schema_path = Path(schema) if schema is not None else root_path / DEVICE_SCHEMA_FILENAME diff --git a/src/packages/harp-data/src/harp/data/_read.py b/src/packages/harp-data/src/harp/data/_read.py index fb7973f..594b401 100644 --- a/src/packages/harp-data/src/harp/data/_read.py +++ b/src/packages/harp-data/src/harp/data/_read.py @@ -54,7 +54,7 @@ def _infer_native_register(raw: bytes) -> type[RegisterBase[Any]]: - """Build a native register class from the first frame's header. + """Build a native register class from the header of the first frame. Reads the payload-type byte, the length byte and the timestamp flag to derive the element type and element count, then returns the matching scalar or array @@ -79,7 +79,7 @@ def read( timestamp: bool = True, message_type: bool = False, ) -> pd.DataFrame: - """Read a single register's binary data, inferring its native layout. + """Read the binary data of a single register, inferring its native layout. ``source`` may be a file path, raw bytes, or an open binary file. The element type, length and timestamp presence are read from the first frame; values diff --git a/src/packages/harp-data/src/harp/data/_reader.py b/src/packages/harp-data/src/harp/data/_reader.py index 855da48..18bd880 100644 --- a/src/packages/harp-data/src/harp/data/_reader.py +++ b/src/packages/harp-data/src/harp/data/_reader.py @@ -13,8 +13,8 @@ _MSG_NAMES = np.array(["_NONE", "Read", "Write", "Event"]) -#: Harp reference epoch — time zero of the Harp clock (UTC). REFERENCE_EPOCH = datetime(1904, 1, 1) +"""Harp reference epoch, time zero of the Harp clock in UTC.""" _TIME_INDEX_NAME = "Time" diff --git a/src/packages/harp-data/src/harp/data/_write.py b/src/packages/harp-data/src/harp/data/_write.py index 8da9b03..2ff3e45 100644 --- a/src/packages/harp-data/src/harp/data/_write.py +++ b/src/packages/harp-data/src/harp/data/_write.py @@ -1,4 +1,4 @@ -"""Write Harp register data to a binary buffer/file — the inverse of the readers. +"""Write Harp register data to a binary buffer or file, the inverse of the readers. Thin wrappers over :meth:`RegisterBase.format_bulk` giving a pandas-package home and a file sink. Useful for round-tripping data and generating typed test corpora. @@ -22,7 +22,7 @@ def to_buffer( ) -> NDArray[np.uint8]: """Encode ``values`` as a flat buffer of ``register`` frames. - ``values`` is a payload (scalar or batch) or an ndarray of the register's + ``values`` is a payload (scalar or batch) or an ndarray of the ``payload_class.payload_dtype``; ``timestamps`` (length-N seconds) makes every frame timestamped; ``message_type`` is one :class:`MessageType` or a length-N array (e.g. the msgtype view from ``parse_bulk``). diff --git a/src/packages/harp-device/src/harp/device/client/_device.py b/src/packages/harp-device/src/harp/device/client/_device.py index 3f778f9..6734ddc 100644 --- a/src/packages/harp-device/src/harp/device/client/_device.py +++ b/src/packages/harp-device/src/harp/device/client/_device.py @@ -23,14 +23,14 @@ _logger = logging.getLogger(__name__) -#: A callback receiving a typed, parsed event for a specific register. EventHandler = Callable[[ParsedHarpMessage[P]], None] +"""A callback receiving a typed, parsed event for a specific register.""" -#: Message types a subscription reacts to, as a single type or an iterable. MessageTypeFilter = MessageType | Iterable[MessageType] +"""Message types a subscription reacts to, as a single type or an iterable.""" -#: Default filter for :meth:`Device.subscribe`: unsolicited events only. _DEFAULT_MESSAGE_TYPES: frozenset[MessageType] = frozenset({MessageType.Event}) +"""Default filter for :meth:`Device.subscribe`: unsolicited events only.""" def _normalize_message_types(message_types: MessageTypeFilter) -> frozenset[MessageType]: @@ -157,7 +157,7 @@ def open(self) -> Self: return self def _validate_whoami(self) -> None: - """Check the device's ``WhoAmI`` against the module (skipped if no module or ``WHO_AM_I == 0x0``).""" + """Check the reported ``WhoAmI`` against the module (skipped if no module or ``WHO_AM_I == 0x0``).""" module = self._device_module if module is None: return @@ -250,7 +250,7 @@ def subscribe( because that thread is shared, handlers are invoked **sequentially, in subscription order, one message at a time**: a slow handler delays every other subscriber and backs up later messages. Keep handlers quick, and - offload heavy work to your own thread or queue. + offload heavy work to a separate thread or queue. Returns a :class:`Subscription`; call :meth:`Subscription.unsubscribe` to stop. diff --git a/src/packages/harp-device/src/harp/device/client/_framer.py b/src/packages/harp-device/src/harp/device/client/_framer.py index b7ec28f..a9f1387 100644 --- a/src/packages/harp-device/src/harp/device/client/_framer.py +++ b/src/packages/harp-device/src/harp/device/client/_framer.py @@ -13,7 +13,7 @@ class HarpFramer: sources (e.g. serial ports) where data arrives in chunks. Recovery: on checksum or PayloadType failure, the framer skips exactly the - bad MessageType byte and retries from the next byte — matching the C# + bad MessageType byte and retries from the next byte, matching the C# StreamTransport resynchronisation strategy. """ @@ -39,7 +39,7 @@ def next_frame(self) -> HarpMessage | None: pos = self._pos while pos < len(buf): - # ── State 1: Seek ────────────────────────────────────────────── + # --- State 1: Seek ---------------------------------------------- # Find a byte that looks like a valid MessageType. try: _validate_message_type(buf[pos]) @@ -49,7 +49,7 @@ def next_frame(self) -> HarpMessage | None: msg_type_pos = pos - # ── State 2: ReadLength ──────────────────────────────────────── + # --- State 2: ReadLength --------------------------------------- if pos + 1 >= len(buf): break # need more data @@ -59,7 +59,7 @@ def next_frame(self) -> HarpMessage | None: pos += 1 continue - # ── State 3: ReadBody ────────────────────────────────────────── + # --- State 3: ReadBody ----------------------------------------- frame_end = pos + 2 + length if frame_end > len(buf): break # frame not yet complete diff --git a/src/packages/harp-device/src/harp/device/schema/_emit.py b/src/packages/harp-device/src/harp/device/schema/_emit.py index 8377cbc..b085ba4 100644 --- a/src/packages/harp-device/src/harp/device/schema/_emit.py +++ b/src/packages/harp-device/src/harp/device/schema/_emit.py @@ -44,7 +44,6 @@ from ._model import DeviceModel, PayloadMember, PayloadType, Register, Registers, Visibility from ._naming import enum_member_name, field_name -# Register base element: schema PayloadType -> numpy scalar type (byte size via np.dtype). _ELEMENT: dict[PayloadType, type[np.generic]] = { PayloadType.U8: np.uint8, PayloadType.S8: np.int8, @@ -56,6 +55,8 @@ PayloadType.S64: np.int64, PayloadType.Float: np.float32, } +"""Register base element: schema PayloadType to numpy scalar type, byte size via np.dtype.""" + _SCALAR_REGISTER: dict[PayloadType, Any] = { PayloadType.U8: RegisterU8, PayloadType.S8: RegisterS8, @@ -67,6 +68,7 @@ PayloadType.S64: RegisterS64, PayloadType.Float: RegisterFloat, } + _ARRAY_REGISTER: dict[PayloadType, Any] = { PayloadType.U8: RegisterU8Array, PayloadType.S8: RegisterS8Array, @@ -82,10 +84,10 @@ @dataclass(frozen=True) class ConverterContext: - """A payload value's schema definition, resolved against its register context. + """The schema definition of a payload value, resolved against its register context. Handed to every converter factory so it can construct the converter with the - right arguments — e.g. ``StringConverter(span)``, ``HarpVersionConverter(element)``, + right arguments, for example ``StringConverter(span)``, ``HarpVersionConverter(element)``, or ``IdentityConverter(dtype)``. """ @@ -93,8 +95,8 @@ class ConverterContext: interface_type: Optional[str] # the DSL interfaceType (None = raw/native) mask: Optional[int] # bit mask, when the value is bit-packed length: int # element count this value spans (0 = unset -> scalar) - element: np.dtype # the register's base element dtype (from PayloadType) - element_size: int # the register's base element byte size + element: np.dtype # base element dtype of the register, from PayloadType + element_size: int # base element byte size of the register @property def span(self) -> int: @@ -103,7 +105,7 @@ def span(self) -> int: @property def member_dtype(self) -> np.dtype: - """The value's own numpy dtype — a native primitive interfaceType overrides the element.""" + """The numpy dtype of the value itself. A native primitive interfaceType overrides the element.""" if self.interface_type is not None: entry = _INTERFACES.get(self.interface_type) if entry is not None and entry.native_dtype is not None: @@ -112,21 +114,24 @@ def member_dtype(self) -> np.dtype: @property def raw_dtype(self) -> np.dtype: - """Native passthrough dtype — a sub-array when the value spans >1 element.""" + """Native passthrough dtype, a sub-array when the value spans more than one element.""" if self.length > 1: return np.dtype((self.element.type, (self.length,))) return self.element -# A converter factory builds a converter from a field's DSL context. ConverterFactory = Callable[[ConverterContext], Converter[Any]] -# A user-supplied converter: a ready instance, or a factory that builds one from context. +"""A converter factory builds a converter from the DSL context of a field.""" + ConverterValue = Union[Converter[Any], ConverterFactory] -# Internal built-in factory — may decline (return None) when the DSL type doesn't -# actually fit (e.g. a primitive whose declared byte span isn't its native size). +"""A user-supplied converter: a ready instance, or a factory that builds one from context.""" + _InterfaceFactory = Callable[[ConverterContext], Optional[Converter[Any]]] -# Coerces a field's yml numeric default into its typed value (``_NO_DEFAULT`` = skip). +"""Internal built-in factory, which may decline by returning None when the DSL type does +not actually fit, for example a primitive whose declared byte span is not its native size.""" + _DefaultCoercer = Callable[[float, ConverterContext], Any] +"""Coerces a yml numeric default into its typed value, where ``_NO_DEFAULT`` skips.""" _NO_DEFAULT = Sentinel("_NO_DEFAULT") # this interface has no numeric default representation @@ -165,10 +170,6 @@ class _Interface: native_dtype: Optional[type[np.generic]] = None -# Every interfaceType the library handles natively, in one uniform table: the -# fixed-width primitives (identity passthrough, carrying their numpy scalar as -# ``native_dtype``) beside string/bool/HarpVersion. Custom interfaceTypes are -# supplied by the caller (see ``converters=``). _INTERFACES: dict[str, _Interface] = { "byte": _Interface(_native(np.uint8), _numpy_default, np.uint8), "sbyte": _Interface(_native(np.int8), _numpy_default, np.int8), @@ -183,6 +184,12 @@ class _Interface: "bool": _Interface(lambda ctx: BoolConverter(), _bool_default), "HarpVersion": _Interface(lambda ctx: HarpVersionConverter(ctx.element), _skip_default), } +"""Every interfaceType the library handles natively, in one uniform table. + +The fixed-width primitives are identity passthroughs carrying their numpy scalar as +``native_dtype``, beside string, bool and HarpVersion. Custom interfaceTypes are +supplied by the caller through ``converters=``. +""" def _materialize(value: ConverterValue, ctx: ConverterContext) -> Converter[Any]: @@ -198,7 +205,7 @@ class NameCollisionError(ValueError): """Two schema identifiers collapse to one Python name, or one shadows a reserved name. Casing is not significant to the generator naming convention, so distinct yml - keys (``DIO0`` / ``Dio0``) can converge — which would silently alias an enum + keys such as ``DIO0`` and ``Dio0`` can converge, which would silently alias an enum member or drop a payload field. """ @@ -233,7 +240,7 @@ def __init__( self.bit_masks = device.bitMasks or {} self.enums = self._build_enums() # Payload classes are cached by name so registers sharing an ``interfaceType`` - # share one class, as the generator's module-level payload list does. + # share one class, as the module-level payload list of the generator does. self.payloads: dict[str, type] = {} # -- naming ----------------------------------------------------------- @@ -273,7 +280,7 @@ def _rename( # -- enums ------------------------------------------------------------ def _build_enums(self) -> dict[str, Any]: - # Enum type names stay verbatim; members take the generator's SCREAMING_SNAKE. + # Enum type names stay verbatim; members take the SCREAMING_SNAKE of the generator. enums: dict[str, Any] = {} for name, spec in self.bit_masks.items(): # IntFlag has no zero-valued member; drop it if present. @@ -317,7 +324,7 @@ def _extension(self, symbol: str, ctx: ConverterContext) -> Converter[Any]: # -- defaults --------------------------------------------------------- def _default(self, member: PayloadMember, type_name: str, ctx: ConverterContext) -> Any: - """The field's typed default value, or ``_NO_DEFAULT`` when it has none.""" + """The typed default value of the field, or ``_NO_DEFAULT`` when it has none.""" _default_value = member.defaultValue if member.defaultValue is not None else member.minValue if _default_value is None or (member.length or 0) > 1: return _NO_DEFAULT @@ -428,7 +435,7 @@ def _new_payload(self, class_name: str, owner: str, reg: Register) -> type: # -- registers -------------------------------------------------------- def _class_name(self, name: str, reg: Register) -> str: - """A private register's class is underscore-prefixed; its payload class is not.""" + """The class of a private register is underscore-prefixed; its payload class is not.""" return f"_{name}" if reg.visibility is Visibility.private else name def _build_register(self, name: str, class_name: str, reg: Register) -> type[RegisterBase[Any]]: @@ -474,7 +481,7 @@ def parse_device_schema(text: str | bytes) -> DeviceModel: """Parse a Harp ``device.yml`` (or a header-less fragment) into a :class:`DeviceModel`. A header-less fragment (just ``registers`` / ``bitMasks`` / ``groupMasks``) - parses fine — the identity fields (``device`` / ``whoAmI`` / ...) are simply + parses fine, and the identity fields such as ``device`` and ``whoAmI`` are simply ``None``. Read files yourself, e.g. ``parse_device_schema(Path("device.yml").read_bytes())``. Prefer reading bytes: a YAML stream declares its own encoding, so the parser decodes it, whereas @@ -502,11 +509,11 @@ def create_registers( ``SCREAMING_SNAKE_CASE``. ``converters`` supplies custom converters keyed by symbol name (e.g. ``{"DataConverter": ...}``); a value is either a ready :class:`~harp.protocol.Converter` instance or a factory - ``(ctx: ConverterContext) -> Converter`` that builds one from the field's DSL + ``(ctx: ConverterContext) -> Converter`` that builds one from the DSL context. A custom type with no matching converter raises ``UnknownConverterError`` when ``strict`` (the default); ``strict=False`` decodes it as its native element type instead. ``exclude_private=True`` drops - registers whose DSL ``visibility`` is ``private``; when kept, a private register's + registers whose DSL ``visibility`` is ``private``; when kept, a private register class is underscore-prefixed (``_Reserved0``). Note that the converter symbol for a payload field derives from its *verbatim* yml key, not the renamed field. """ diff --git a/src/packages/harp-device/src/harp/device/schema/_model.py b/src/packages/harp-device/src/harp/device/schema/_model.py index 0d11f28..1ab7076 100644 --- a/src/packages/harp-device/src/harp/device/schema/_model.py +++ b/src/packages/harp-device/src/harp/device/schema/_model.py @@ -25,26 +25,47 @@ class PayloadType(str, Enum): class Access(Enum): + """The operations which can be used to access register data.""" + Read = "Read" + """The register will accept a request to read the payload value.""" + Write = "Write" + """The register will accept a request to write the payload value.""" + Event = "Event" + """The device may send messages to the controller reporting the register contents.""" class Visibility(Enum): + """Whether a register is exposed in the high-level interface.""" + public = "public" + """The register is exposed to the high-level interface.""" + private = "private" + """The register is hidden from the high-level interface.""" class Converter(Enum): + """A custom converter used to parse or format a payload or payload member value.""" + None_ = "None" + """No custom conversion is required.""" + Payload = "Payload" + """The custom converter operates on the specified payload type.""" + RawPayload = "RawPayload" + """The custom converter operates directly on raw payload bytes.""" class MaskValueItem(BaseModel): model_config = ConfigDict(extra="forbid") - value: int = Field(..., description="Specifies the numerical mask value.") - description: Optional[str] = Field(None, description="Summary of the mask value function.") + value: int = Field(..., description="The numerical mask value.") + description: Optional[str] = Field( + None, description="A summary description of the mask value function." + ) def __int__(self) -> int: return self.value @@ -58,13 +79,21 @@ def __int__(self) -> int: class BitMask(BaseModel): - description: Optional[str] = Field(None, description="Summary of the bit mask function.") - bits: Dict[str, MaskValue] + """A bit mask used for reading or writing specific registers.""" + + description: Optional[str] = Field( + None, description="A summary description of the bit mask function." + ) + bits: Dict[str, MaskValue] = Field(..., description="The collection of bit mask values.") class GroupMask(BaseModel): - description: Optional[str] = Field(None, description="Summary of the group mask function.") - values: Dict[str, MaskValue] + """A group mask used for reading or writing specific registers.""" + + description: Optional[str] = Field( + None, description="A summary description of the group mask function." + ) + values: Dict[str, MaskValue] = Field(..., description="The collection of group mask values.") class MaskType(RootModel[str]): @@ -88,58 +117,128 @@ class DefaultValue(RootModel[float]): class PayloadMember(BaseModel): - mask: Optional[int] = Field(None, description="Mask used to read/write this member.") - offset: Optional[int] = Field(None, description="Payload array offset of this member.") - length: Optional[int] = Field(None, description="Number of base elements this member spans.") - description: Optional[str] = Field(None, description="Summary of the payload member.") - minValue: Optional[MinValue] = None - maxValue: Optional[MaxValue] = None - defaultValue: Optional[DefaultValue] = None - maskType: Optional[MaskType] = None - interfaceType: Optional[InterfaceType] = None - converter: Optional[Converter] = None + """A named member of a structured register payload.""" + + mask: Optional[int] = Field( + None, description="The mask used to read and write this payload member." + ) + offset: Optional[int] = Field( + None, + description="The zero-based index at which encoding of this payload member starts.", + ) + length: Optional[int] = Field( + None, description="The number of elements used to encode this payload member." + ) + description: Optional[str] = Field( + None, description="A summary description of this payload member." + ) + minValue: Optional[MinValue] = Field( + None, description="The minimum allowable value for the payload member." + ) + maxValue: Optional[MaxValue] = Field( + None, description="The maximum allowable value for the payload member." + ) + defaultValue: Optional[DefaultValue] = Field( + None, description="The default value for the payload member." + ) + maskType: Optional[MaskType] = Field( + None, + description="The name of the bit mask or group mask used to represent this payload member.", + ) + interfaceType: Optional[InterfaceType] = Field( + None, + description=( + "The name of the type used to represent this payload member " + "in the high-level interface." + ), + ) + converter: Optional[Converter] = Field( + None, + description="A custom converter used to parse or format this payload member.", + ) class Register(BaseModel): - address: Annotated[int, Field(le=255, description="Unique 8-bit register address.")] - type: PayloadType - length: Annotated[Optional[int], Field(ge=1, default=1, description="Payload length.")] - access: Union[Access, List[Access]] = Field(..., description="Expected use of the register.") - description: Optional[str] = Field(None, description="Summary of the register function.") - minValue: Optional[MinValue] = None - maxValue: Optional[MaxValue] = None - defaultValue: Optional[DefaultValue] = None - maskType: Optional[MaskType] = None + """The functionality and operation of a specific register.""" + + address: Annotated[int, Field(le=255, description="The unique 8-bit address of the register.")] + type: PayloadType = Field(..., description="The type of the register payload.") + length: Annotated[ + Optional[int], Field(ge=1, default=1, description="The length of the register payload.") + ] + access: Union[Access, List[Access]] = Field( + ..., description="The expected use of the register." + ) + description: Optional[str] = Field( + None, description="A summary description of the register function." + ) + minValue: Optional[MinValue] = Field( + None, description="The minimum allowable value for the payload." + ) + maxValue: Optional[MaxValue] = Field( + None, description="The maximum allowable value for the payload." + ) + defaultValue: Optional[DefaultValue] = Field( + None, description="The default value for the payload." + ) + maskType: Optional[MaskType] = Field( + None, + description="The name of the bit mask or group mask used to represent the payload value.", + ) visibility: Optional[Visibility] = Field( - None, description="Exposed in the high-level interface." + None, description="Whether the register function is exposed in the high-level interface." + ) + volatile: Optional[bool] = Field( + None, description="Whether register values can be saved in non-volatile memory." + ) + payloadSpec: Optional[Dict[str, PayloadMember]] = Field( + None, + description=( + "A collection of payload members describing the contents of the raw payload value." + ), + ) + interfaceType: Optional[InterfaceType] = Field( + None, + description=( + "The name of the type used to represent the payload value in the high-level interface." + ), + ) + converter: Optional[Converter] = Field( + None, description="A custom converter used to parse or format the payload value." ) - volatile: Optional[bool] = Field(None, description="Value can be saved in non-volatile memory.") - payloadSpec: Optional[Dict[str, PayloadMember]] = None - interfaceType: Optional[InterfaceType] = None - converter: Optional[Converter] = None class Registers(BaseModel): - """A bare register collection — a header-less ``device.yml`` fragment.""" + """A bare register collection, a header-less ``device.yml`` fragment.""" - registers: Dict[str, Register] = Field(..., description="The device's registers.") - bitMasks: Optional[Dict[str, BitMask]] = None - groupMasks: Optional[Dict[str, GroupMask]] = None + registers: Dict[str, Register] = Field( + ..., description="The collection of registers implementing the device function." + ) + bitMasks: Optional[Dict[str, BitMask]] = Field( + None, + description="The collection of masks available to be used with the different registers.", + ) + groupMasks: Optional[Dict[str, GroupMask]] = Field( + None, + description=( + "The collection of group masks available to be used with the different registers." + ), + ) class DeviceModel(Registers): - """A device schema: a `Registers` collection plus (optional) device identity. + """A device schema: a `Registers` collection plus optional device identity. - Every identity field is optional, so a header-less fragment (just ``registers`` - / ``bitMasks`` / ``groupMasks``) is simply a ``DeviceModel`` with them all None - — parsing never needs to branch on "fragment vs full document". + Every identity field is optional, so a header-less fragment, carrying + just ``registers``, ``bitMasks`` or ``groupMasks``, is simply a ``DeviceModel`` with + them all None, and parsing never needs to branch on "fragment vs full document". """ device: Optional[str] = Field(None, description="The name of the device.") - whoAmI: Optional[int] = Field(None, description="Unique identifier for this device type.") + whoAmI: Optional[int] = Field(None, description="The unique identifier for this device type.") firmwareVersion: Optional[str] = Field( - None, description="Semantic version of the device firmware." + None, description="The version of the device firmware, as ``major.minor``." ) hardwareTargets: Optional[str] = Field( - None, description="Semantic version of the device hardware." + None, description="The version of the device hardware, as ``major.minor``." ) diff --git a/src/packages/harp-device/src/harp/device/schema/_module.py b/src/packages/harp-device/src/harp/device/schema/_module.py index e7c542e..9c0ed04 100644 --- a/src/packages/harp-device/src/harp/device/schema/_module.py +++ b/src/packages/harp-device/src/harp/device/schema/_module.py @@ -15,8 +15,8 @@ from harp.device.core import REGISTER_MAP as CORE_REGISTER_MAP from ._emit import ConverterValue, create_registers, parse_device_schema -#: Module name used when the schema carries no ``device`` header. _DEFAULT_NAME = "Device" +"""Module name used when the schema carries no ``device`` header.""" @runtime_checkable @@ -44,10 +44,11 @@ class DeviceModule(types.ModuleType): none of this, since its registers are written out. """ - #: Address -> register class, the common Harp registers merged with the schema's. REGISTER_MAP: dict[int, type[RegisterBase[Any]]] - #: The device identity declared by the schema; ``0`` when absent. + """Address -> register class, the common Harp registers merged with those of the schema.""" + WHO_AM_I: int + """The device identity declared by the schema. ``0`` when absent.""" def __getattr__(self, name: str) -> type[RegisterBase[Any]]: raise AttributeError(f"module {self.__name__!r} has no register named {name!r}") @@ -69,14 +70,14 @@ def create_device_module( * ``REGISTER_MAP``, the device address space, so the common registers are present here even though the module does not name them; - * ``WHO_AM_I``, the schema's identity (``0`` for an unregistered device); - * ``__name__``, the schema's ``device`` name, or ``name`` when given + * ``WHO_AM_I``, the identity declared by the schema (``0`` for an unregistered device); + * ``__name__``, the ``device`` name of the schema, or ``name`` when given (``"Device"`` for a header-less register fragment). Because the names come from the schema at runtime they don't autocomplete, and each resolves as ``type[RegisterBase[Any]]`` rather than its own register type; a generated device package is a real module on disk and gives both. On an - address clash the device's register replaces the common one in ``REGISTER_MAP``. + address clash the device register replaces the common one in ``REGISTER_MAP``. ``exclude_private=True`` drops registers whose DSL ``visibility`` is ``private``. ``text`` is the schema itself rather than a path to it, matching diff --git a/src/packages/harp-device/src/harp/device/schema/_naming.py b/src/packages/harp-device/src/harp/device/schema/_naming.py index b7106fc..6185320 100644 --- a/src/packages/harp-device/src/harp/device/schema/_naming.py +++ b/src/packages/harp-device/src/harp/device/schema/_naming.py @@ -7,9 +7,9 @@ * payload fields -> :func:`field_name` (``DutyCycle`` -> ``duty_cycle``) Type-level identifiers (register classes, enum classes, ``{Name}Payload``) are -*not* transformed — the generator keeps those verbatim from the yml too. +*not* transformed, and the generator keeps those verbatim from the yml too. -See the upstream generator's package for more information: +See the upstream generator package for more information: https://github.com/harp-tech/generators """ @@ -17,10 +17,12 @@ _SEPARATOR = "_" -# The generator's regex: an uppercase letter, optionally preceded by a separator. -# The separator is part of the match, so a match starting on ``_``/``-`` has its -# index on the separator rather than on the letter (mirrored in ``_replace``). _BOUNDARY = re.compile(r"(?P[_\-])?(?P[A-Z])") +"""The generator regex: an uppercase letter, optionally preceded by a separator. + +The separator is part of the match, so a match starting on ``_`` or ``-`` has its +index on the separator rather than on the letter, which ``_replace`` mirrors. +""" def _screaming_snake(value: str) -> str: @@ -53,8 +55,8 @@ def _replace(match: "re.Match[str]") -> str: run = index - previous_match previous_match = index char = match.group("char").lower() - # Separate unless this capital continues a run of capitals — and a run's - # final capital still separates when it starts a new lowercase word. + # Separate unless this capital continues a run of capitals, though the final + # capital of a run still separates when it starts a new lowercase word. follower = index + 1 separate = run != 1 or (follower < len(value) and value[follower].islower()) return _SEPARATOR + char if separate else char @@ -75,6 +77,6 @@ def enum_member_name(value: str) -> str: def field_name(value: str) -> str: """The Python payload field name for a yml ``payloadSpec`` key. - ``DutyCycle`` -> ``duty_cycle``. Matches the generator's ``GetPythonFieldName``. + ``DutyCycle`` -> ``duty_cycle``. Matches ``GetPythonFieldName`` in the generator. """ return _screaming_snake(value).lower() diff --git a/src/packages/harp-protocol/src/harp/protocol/_constants.py b/src/packages/harp-protocol/src/harp/protocol/_constants.py index 0b41c43..e2bdc0c 100644 --- a/src/packages/harp-protocol/src/harp/protocol/_constants.py +++ b/src/packages/harp-protocol/src/harp/protocol/_constants.py @@ -1,22 +1,22 @@ """Package-wide Harp protocol constants.""" -# Harp timestamp clock tick period in seconds (32 µs/tick). _TICK_PERIOD_S: float = 32e-6 +"""Harp timestamp clock tick period in seconds, 32 microseconds per tick.""" -# Payload-type byte bit that signals a timestamp is present in the frame. _TIMESTAMP_FLAG: int = 0x10 +"""Payload-type byte bit that signals a timestamp is present in the frame.""" -# Default Harp port value (broadcast). _DEFAULT_PORT: int = 0xFF +"""Default Harp port value, meaning broadcast.""" -# Fixed header size in bytes: msg_type + length + address + port + payload_type. _HEADER_LEN: int = 5 +"""Fixed header size in bytes: msg_type + length + address + port + payload_type.""" -# Timestamp field size in bytes: 4-byte seconds (u32) + 2-byte microseconds (u16). _TIMESTAMP_LEN: int = 6 +"""Timestamp field size in bytes: 4-byte seconds as u32 plus 2-byte microseconds as u16.""" -# Byte offset of the timestamp microseconds field (_HEADER_LEN + 4). _TS_MICROS_OFFSET: int = 9 +"""Byte offset of the timestamp microseconds field, which is ``_HEADER_LEN + 4``.""" -# Byte offset of the payload when a timestamp is present (_HEADER_LEN + _TIMESTAMP_LEN). _TIMESTAMPED_PAYLOAD_OFFSET: int = 11 +"""Byte offset of the payload when a timestamp is present, ``_HEADER_LEN + _TIMESTAMP_LEN``.""" diff --git a/src/packages/harp-protocol/src/harp/protocol/_message_type.py b/src/packages/harp-protocol/src/harp/protocol/_message_type.py index 6ac97dd..d23e997 100644 --- a/src/packages/harp-protocol/src/harp/protocol/_message_type.py +++ b/src/packages/harp-protocol/src/harp/protocol/_message_type.py @@ -9,8 +9,9 @@ class MessageType(IntEnum): Event = 3 -# Bits 7,6,5,4,2 must be 0; bit 3 is error; bits 1:0 are type. _RESERVED_MASK = 0b11110100 +"""Bits 7, 6, 5, 4 and 2 must be 0. Bit 3 is error and bits 1:0 are the type.""" + _VALID_TYPES = frozenset(t.value for t in MessageType) diff --git a/src/packages/harp-protocol/src/harp/protocol/_payload.py b/src/packages/harp-protocol/src/harp/protocol/_payload.py index 12bebcc..1f92e66 100644 --- a/src/packages/harp-protocol/src/harp/protocol/_payload.py +++ b/src/packages/harp-protocol/src/harp/protocol/_payload.py @@ -38,8 +38,8 @@ class Column: *codes* and ``categories`` the ordered labels, so a consumer can map codes to labels without copying. - ``eq=False`` keeps identity comparison — field-wise equality would hit - numpy's ambiguous-truth-value error on the ``data`` array. + ``eq=False`` keeps identity comparison, since field-wise equality would hit + the numpy ambiguous-truth-value error on the ``data`` array. ``name`` is ``None`` for an anonymous single value """ @@ -58,7 +58,7 @@ class _FieldSlot: def _mask_trailing_zeros(mask: int) -> int: - """Number of trailing zero bits in ``mask`` — the right-shift that aligns a + """Number of trailing zero bits in ``mask``, the right-shift that aligns a masked field to bit 0.""" if mask == 0: return 0 @@ -66,7 +66,7 @@ def _mask_trailing_zeros(mask: int) -> int: # --------------------------------------------------------------------------- -# Descriptors — scalar variants (return Python / 0-D types) +# Descriptors, scalar variants returning Python or 0-D types # --------------------------------------------------------------------------- @@ -75,18 +75,19 @@ class Field(Generic[T]): Two modes, selected by ``mask``: - * **Whole-element** (``mask=None``, the default) — the view reads - ``converter.dtype.itemsize`` bytes starting at ``offset`` (in base-element - units; see :class:`StructPayload`) and runs them through ``converter``. The - converter owns its own ``dtype`` (byte layout) and is independent of the - payload's base element type, so the same converter works under any register - width. - * **Masked sub-field** (``mask`` set) — the raw value is extracted as - ``(element & mask) >> shift`` from the payload's *base element* at ``offset`` - and then run through ``converter`` (which dictates the output type). The - right-shift is derived from ``mask`` (its trailing-zero count). Several masked - fields at the same offset share the element slot automatically, and may share - it with a :class:`GroupMask` or :class:`BitMask` on the same word. + * **Whole-element**, with ``mask=None`` as the default. The view reads + ``converter.dtype.itemsize`` bytes starting at ``offset``, in base-element + units as described in :class:`StructPayload`, and runs them through + ``converter``. The converter owns its own ``dtype``, and so its own byte + layout, and is independent of the base element type of the payload, so the + same converter works under any register width. + * **Masked sub-field**, with ``mask`` set. The raw value is extracted as + ``(element & mask) >> shift`` from the *base element* of the payload at + ``offset`` and then run through ``converter``, which dictates the output + type. The right-shift is derived from the trailing-zero count of ``mask``. + Several masked fields at the same offset share the element slot + automatically, and may share it with a :class:`GroupMask` or + :class:`BitMask` on the same word. ``offset`` defaults to ``0``. Omitting it suits a payload with a single member; when a payload has several distinct slots, each must declare an @@ -197,14 +198,14 @@ class GroupMask(Generic[E]): """Descriptor for a masked, shifted enum sub-field of a payload element. Syntactic sugar over a masked :class:`Field`: the raw value is extracted as - ``(element & mask) >> shift`` and mapped strictly to an ``enum.IntEnum`` member - (an unknown code raises). ``enum=`` is required; for masked *numeric* fields use + ``(element & mask) >> shift`` and mapped strictly to an ``enum.IntEnum`` member, + and an unknown code raises. ``enum=`` is required. For masked *numeric* fields use ``Field(converter=..., mask=...)`` instead. - The right-shift is always derived from ``mask`` (its trailing-zero count, so the - field aligns to bit 0); ``offset`` defaults to ``0``. The element width and - storage slot are derived from the payload's base element type, so several masked - fields at the same offset share storage automatically. + The right-shift is always derived from the trailing-zero count of ``mask``, so the + field aligns to bit 0, and ``offset`` defaults to ``0``. The element width and + storage slot are derived from the base element type of the payload, so several + masked fields at the same offset share storage automatically. """ if TYPE_CHECKING: @@ -239,8 +240,9 @@ def __init__( self._lookup_safe = (mask >> self._shift) < len(self._code_lookup) def _decode_raw(self, raw: Any) -> Any: - """Map an extracted (masked + shifted) integer to its enum member, preserving an - undefined code as its raw int (permissive, like C#'s unchecked enum cast).""" + """Map an extracted masked and shifted integer to its enum member, preserving an + undefined code as its raw int. Decoding is permissive, like the unchecked enum + cast in C#.""" value = int(raw) try: return self._enum(value) @@ -278,13 +280,14 @@ def _to_batch(self) -> "_GroupMaskBatch[E]": def _columns( self, arr: "NDArray[Any]", name: "str | None", *, decode_enums: bool, demux_bit_masks: bool ) -> "list[Column]": - """One enum column: category codes + labels (``decode_enums``) or raw codes.""" + """One enum column: category codes and labels under ``decode_enums``, or raw codes.""" raw = (arr[self._slot] & self._mask) >> self._shift if not decode_enums: return [Column(name, raw)] lookup = self._code_lookup - # ``_lookup_safe`` (the field's raw range fits the table) skips the bounds guard; - # an in-range gap still maps to -1, so the undefined branch below runs regardless. + # ``_lookup_safe`` means the raw range of the field fits the table, so the bounds + # guard is skipped. An in-range gap still maps to -1, so the undefined branch + # below runs regardless. if self._lookup_safe: codes = lookup[raw] else: @@ -292,8 +295,9 @@ def _columns( undefined = codes < 0 if not undefined.any(): return [Column(name, codes, self._categories)] - # An undefined code (an in-range gap, or a value past the enum's range) is kept - # as its raw integer — an extra category — matching the scalar decode and C#'s + # An undefined code, either an in-range gap or a value past the range of the + # enum, is kept as its raw integer and becomes an extra category, matching the + # scalar decode and the unchecked cast in C#. codes = codes.astype(np.intp) extras = np.unique(raw[undefined]) codes[undefined] = len(self._categories) + np.searchsorted(extras, raw[undefined]) @@ -304,17 +308,17 @@ class BitMask(Generic[F]): """Descriptor for a masked ``enum.IntFlag`` view of a payload element. The flag counterpart of :class:`GroupMask`: the raw value is extracted as - ``element & mask`` and mapped to an ``enum.IntFlag`` member (decoding is - *permissive* — combined flag values such as ``A | B`` are valid, matching the - C# generator's unchecked cast). ``enum=`` is required and must be an + ``element & mask`` and mapped to an ``enum.IntFlag`` member. Decoding is + *permissive*, so combined flag values such as ``A | B`` are valid, matching the + unchecked cast of the C# generator. ``enum=`` is required and must be an ``IntFlag`` subclass. Unlike :class:`GroupMask` there is **no shift**: ``IntFlag`` member values are absolute bit positions, so the flags are read and written in place. ``mask`` - defaults to the full base element (the common whole-register bitMask case) and + defaults to the full base element, the common whole-register bitMask case, and may be narrowed to embed a flag set inside a wider element. The element width - and storage slot are derived from the payload's base element type, so several - masked fields at the same offset share storage automatically. + and storage slot are derived from the base element type of the payload, so + several masked fields at the same offset share storage automatically. """ if TYPE_CHECKING: @@ -396,7 +400,7 @@ def _columns( # --------------------------------------------------------------------------- -# Descriptors — batch variants (return ndarray views) +# Descriptors, batch variants returning ndarray views # These are mostly used for batch operations like `to_dataframe` # --------------------------------------------------------------------------- @@ -500,8 +504,8 @@ class Batch(Protocol[_PT]): than a single record. At runtime, the value is the auto-derived ``P._PayloadBatchType`` sibling whose descriptors return ``NDArray`` views. - Per-field dtype precision is intentionally dropped — every declared - field reports ``NDArray[Any]`` — to keep ``RegisterBase[P]`` + Per-field dtype precision is intentionally dropped, with every declared + field reporting ``NDArray[Any]``, to keep ``RegisterBase[P]`` parameterized by a single TypeVar. """ @@ -517,16 +521,16 @@ def payload_as_columns( # type: ignore[empty-body] def __getattr__(self, name: str) -> "NDArray[Any]": ... # type: ignore[empty-body] -# Helpers for type checking using isinstance() +# Descriptor type tuples used for isinstance checks over payload declarations. _SCALAR_DECLARATION_TYPES = (Field, GroupMask, BitMask) _BATCH_DECLARATION_TYPES = (_FieldBatch, _GroupMaskBatch, _BitMaskBatch) _DECLARATION_TYPES = _SCALAR_DECLARATION_TYPES + _BATCH_DECLARATION_TYPES -#: Every member the payload classes own carries one of these prefixes, so a field name -#: is barred from them rather than from a list of the members themselves. Dunders are -#: exempt because ``__value__`` is how a single-slot payload declares its root field. _RESERVED_FIELD_PREFIXES = ("_", "payload_") +"""Every member the payload classes own carries one of these prefixes, so a field name +is barred from them rather than from a list of the members themselves. Dunders are +exempt because ``__value__`` is how a single-slot payload declares its root field.""" def _reserved_field_reason(name: str) -> "str | None": @@ -554,7 +558,7 @@ def _batch_init_disabled(self: "PayloadBase", *args: object, **kwargs: object) - def _resolve_element_dtype(cls: type) -> np.dtype: """Resolve the base element dtype from the ``StructPayload[...]`` type arg. - Only used for offset→byte arithmetic and the masked-read integer width. + Only used for offset-to-byte arithmetic and the masked-read integer width. Defaults to uint8 (byte) when the payload is not parameterized, which makes byte-offset layouts of heterogeneous consecutive fields work out of the box. """ @@ -606,8 +610,8 @@ def _build_struct_dtype( ) -> np.dtype: """Build the numpy structured dtype from field declarations. - Each descriptor binds itself to its numpy slot via ``_bind_slot``; this - function resolves only cross-field layout — which masked fields share a slot, + Each descriptor binds itself to its numpy slot via ``_bind_slot``. This + function resolves only cross-field layout: which masked fields share a slot, plus offsets, overlap, and itemsize.""" elem = cls._elem_dtype elem_size = elem.itemsize @@ -616,15 +620,15 @@ def _build_struct_dtype( for attr_name, val in declarations: byte_offset = val._offset * elem_size - # A plain Field (no mask=) is the only whole-element view; everything else - # (GroupMask, BitMask, or a Field with mask=) is a masked sub-field. The + # A plain Field with no mask= is the only whole-element view. Everything else, + # a GroupMask, a BitMask, or a Field with mask=, is a masked sub-field. The # isinstance form lets the type checker narrow `val` to access ``_converter``. - if isinstance(val, Field) and val._mask is None: # whole-element Field — own slot + if isinstance(val, Field) and val._mask is None: # whole-element Field, own slot if attr_name in slots: raise TypeError(f"{cls.__name__}: duplicate field name {attr_name!r}") val._bind_slot(attr_name, elem) slots[attr_name] = _FieldSlot(val._converter.dtype, byte_offset) - else: # masked sub-field — shares the base-element slot at its offset + else: # masked sub-field, shares the base-element slot at its offset owner = mask_slot_by_byte_offset.setdefault(byte_offset, attr_name) val._bind_slot(owner, elem) assert val._mask is not None # _bind_slot ensures every masked field has a mask @@ -666,7 +670,7 @@ class PayloadBase(Generic[NpStructT]): _scalar_cls: ClassVar["type[PayloadBase]"] # The batch twin of this class (identity until the Batch sibling is generated). _batch_cls: ClassVar["type[PayloadBase]"] - # Cached map of attribute name → default value for fields that declare one. + # Cached map of attribute name to default value for fields that declare one. _defaults: ClassVar[dict[str, Any]] # Auto-generated sibling class whose descriptors return NDArray views instead of scalars. _PayloadBatchType: ClassVar["type[PayloadBase]"] @@ -704,8 +708,8 @@ def __init__(self, *args: object, **kwargs: object) -> None: arr = np.zeros((), dtype=self.payload_dtype) # Route each kwarg by its descriptor kind, not by whether its name happens - # to match a numpy slot — masked descriptors may share a slot whose name - # collides with the first masked field's attribute name. + # to match a numpy slot, since masked descriptors may share a slot whose + # name collides with the attribute name of the first masked field. for attr_name, value in kwargs.items(): desc = cls._mro_descriptor(attr_name) if isinstance(desc, Field) and desc._mask is None: # whole-element Field @@ -768,7 +772,7 @@ def __init_subclass__( if _batch_of is not None: # Auto-generated Batch sibling: borrow dtype/_repr_fields from its - # scalar twin and wire the scalar↔batch pointers. + # scalar twin and wire the pointers between scalar and batch. cls.payload_dtype = _batch_of.payload_dtype cls._repr_fields = _batch_of._repr_fields cls._elem_dtype = _batch_of._elem_dtype @@ -841,12 +845,13 @@ def payload_as_columns( ) -> list[Column]: """Returns a list of Column where each member represents a field from a payload across multiple messages. - ``decode_enums`` controls whether ``GroupMask`` (enum) columns become - category codes + labels (True) or raw integer codes (False) — a - shape-preserving relabel. ``demux_bit_masks`` controls whether a - ``BitMask`` (flag) column is expanded into one boolean column per flag - member (True) or kept as a single raw-integer column (False) — a shape - change. The two are orthogonal and apply to different descriptor kinds. + ``decode_enums`` controls whether ``GroupMask`` enum columns become + category codes and labels when ``True``, or raw integer codes when + ``False``, which is a shape-preserving relabel. ``demux_bit_masks`` + controls whether a ``BitMask`` flag column is expanded into one boolean + column per flag member when ``True``, or kept as a single raw-integer + column when ``False``, which is a shape change. The two are orthogonal + and apply to different descriptor kinds. """ arr = np.atleast_1d(self._arr) # Each descriptor renders its own column(s); resolve via the scalar twin. @@ -878,9 +883,9 @@ def _unwrap(cls, arr: "np.ndarray") -> Any: Struct payloads always return a typed wrapper so descriptors like ``payload.Channel0`` work. Anonymous payloads override this to return the - raw numpy scalar/ndarray directly, or — for an ``AnonymousPayload`` root — - the unwrapped ``__value__`` (the single-member branch below, reached via - the override's ``super()`` call). A struct payload never auto-unwraps. + raw numpy scalar or ndarray directly, or, for an ``AnonymousPayload`` root, + the unwrapped ``__value__`` through the single-member branch below, reached + via the ``super()`` call of the override. A struct payload never auto-unwraps. """ obj = cls._from_array(arr) if cls._single_member is not None and arr.ndim == 0: @@ -889,7 +894,7 @@ def _unwrap(cls, arr: "np.ndarray") -> Any: # --------------------------------------------------------------------------- -# StructPayload — base for named-field (struct) register payloads +# StructPayload, the base for named-field struct register payloads # --------------------------------------------------------------------------- @@ -919,7 +924,7 @@ class MyPayload(StructPayload[np.uint8]): # --------------------------------------------------------------------------- -# AnonymousPayload — single unnamed slot, no user-facing descriptors +# AnonymousPayload, a single unnamed slot with no user-facing descriptors # --------------------------------------------------------------------------- @@ -932,7 +937,7 @@ class PayloadU16(AnonymousPayload, scalar_dtype=" None: def _unwrap(cls, arr: "np.ndarray") -> Any: if cls._root: return super()._unwrap(arr) # PayloadBase single-member unwrap (.__value__) - # 0-D → numpy scalar via item-like access (preserves dtype). - # 1-D / sub-array → return the ndarray as-is. + # 0-D becomes a numpy scalar via item-like access, preserving dtype. + # 1-D or sub-array returns the ndarray as-is. return arr if arr.ndim > 0 else arr[()] def _repr_kwargs(self) -> str: @@ -1105,7 +1110,7 @@ class PayloadFloat(AnonymousPayload[np.float32], scalar_dtype=np.dtype(" None: class IdentityConverter(Converter[NpScalarT]): - """Pass-through converter — the raw numpy scalar is returned as-is.""" + """Pass-through converter, returning the raw numpy scalar as-is.""" def __init__(self, dtype: "np.dtype[NpScalarT] | str | type[NpScalarT]") -> None: self.dtype = np.dtype(dtype) @@ -124,7 +124,7 @@ def __init__(self) -> None: class BoolConverter(Converter[bool]): """Whole-element ``interfaceType: bool`` (or a single masked bit via ``Field(BoolConverter(), mask=...)``). - The element is non-zero → ``True``. Operates on a single base element. + A non-zero element becomes ``True``. Operates on a single base element. """ def __init__(self, dtype: "np.dtype | str | type" = np.uint8) -> None: diff --git a/src/packages/harp-protocol/src/harp/protocol/_register.py b/src/packages/harp-protocol/src/harp/protocol/_register.py index 21ce8ee..8245757 100644 --- a/src/packages/harp-protocol/src/harp/protocol/_register.py +++ b/src/packages/harp-protocol/src/harp/protocol/_register.py @@ -66,11 +66,11 @@ def _encode_message_types(message_type: MessageType | ArrayLike, nrows: int) -> class _LazyTimestamps: """Seconds + microseconds timestamp views, combined into float64 on first use. - Combining the raw views costs an O(n) pass over every frame (two ``astype`` - casts plus a multiply-add), independent of the register's payload — so eagerly + Combining the raw views costs an O(n) pass over every frame, two ``astype`` + casts plus a multiply-add, independent of the register payload, so eagerly computing it in ``parse_bulk`` taxes every call even when the caller never reads the timestamps. Deferring the combine until the array is actually - accessed (and caching the result) avoids that cost in the common case where + accessed, and caching the result, avoids that cost in the common case where only the payload is needed. """ @@ -118,17 +118,17 @@ def __call__(cls: "type[_R]", address: int) -> "type[_R]": class RegisterBase(ABC, Generic[U]): """Abstract base for all typed Harp registers. - The generic parameter ``U`` is the static return type of :meth:`parse` — the - user-facing value, *not* necessarily ``payload_class`` (that is the wire - encoding). The two coincide only for multi-member struct payloads: + The generic parameter ``U`` is the static return type of :meth:`parse`, the + user-facing value, *not* necessarily ``payload_class``, which is the wire + encoding. The two coincide only for multi-member struct payloads: - * scalar registers → a numpy scalar (e.g. ``np.uint16``); - * array registers → ``NDArray[…]`` of fixed length; - * multi-member struct registers → the payload class itself; - * single-member registers that unwrap on parse → the inner value type, e.g. - ``RegisterBase[str]`` (DeviceName), ``RegisterBase[HarpVersion]``, or - ``RegisterBase[ClockConfigurationFlags]`` for a whole-register ``BitMask`` - / ``GroupMask`` — even though each still has a ``payload_class``. + * scalar registers -> a numpy scalar, for example ``np.uint16``; + * array registers -> ``NDArray[...]`` of fixed length; + * multi-member struct registers -> the payload class itself; + * single-member registers that unwrap on parse -> the inner value type, for + example ``RegisterBase[str]`` for DeviceName, ``RegisterBase[HarpVersion]``, + or ``RegisterBase[ClockConfigurationFlags]`` for a whole-register + ``BitMask`` or ``GroupMask``, even though each still has a ``payload_class``. Subclasses must define ``address``, ``payload_type``, and ``payload_class`` as ``ClassVar``s. @@ -207,14 +207,15 @@ def format_bulk( message_type: MessageType | ArrayLike = MessageType.Event, port: int = _DEFAULT_PORT, ) -> NDArray[np.uint8]: - """Build a flat buffer of N frames of this register type — the inverse of + """Build a flat buffer of N frames of this register type, the inverse of :meth:`parse_bulk`. - ``values`` is a payload (scalar or :class:`Batch`) or an ndarray of the - register's ``payload_class.payload_dtype``. ``timestamps`` (a length-N array of - seconds) makes every frame timestamped. ``message_type`` is one - :class:`MessageType` for all frames, or a length-N array of message-type - bytes / values (e.g. the ``msgtype`` view returned by ``parse_bulk``). + ``values`` is a payload, either scalar or :class:`Batch`, or an ndarray of + the ``payload_class.payload_dtype`` of the register. ``timestamps``, a + length-N array of seconds, makes every frame timestamped. ``message_type`` + is one :class:`MessageType` for all frames, or a length-N array of + message-type bytes or values, for example the ``msgtype`` view returned by + ``parse_bulk``. """ payload_cls = cls.payload_class itemsize = payload_cls.payload_dtype.itemsize @@ -300,7 +301,7 @@ def format( timestamp: float | None = None, port: int = _DEFAULT_PORT, ) -> bytes: - """Build a Harp frame for this register. No value → Read; with value → Write.""" + """Build a Harp frame for this register. No value gives a Read, a value gives a Write.""" if value is _MISSING: mt = MessageType.Read if message_type is None else message_type return build_message_frame( diff --git a/tests/conftest.py b/tests/conftest.py index b7fe583..ed31099 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,4 @@ -"""Pytest configuration — shared fixtures and helpers for all suites.""" +"""Pytest configuration, shared fixtures and helpers for all suites.""" from pathlib import Path diff --git a/tests/device/test_emit.py b/tests/device/test_emit.py index 73a89fc..f741f3b 100644 --- a/tests/device/test_emit.py +++ b/tests/device/test_emit.py @@ -27,8 +27,8 @@ def _device_registers(): def _layout(dt): """Full structural signature: field name + element dtype + offset, and itemsize. - Name-exact — the emitter applies the same naming convention as the generator, so - the golden comparison covers identifiers as well as byte layout. + Name-exact, since the emitter applies the same naming convention as the generator, + so the reference comparison covers identifiers as well as byte layout. """ if dt.names is None: return ("scalar", dt.str, dt.shape, dt.itemsize) @@ -36,7 +36,7 @@ def _layout(dt): # --------------------------------------------------------------------------- -# Device golden — layout/type parity with generator output +# Device reference output, layout and type parity with generator output # --------------------------------------------------------------------------- @@ -56,7 +56,7 @@ def test_device_emits_all_registers(device_registers): # --------------------------------------------------------------------------- -# Naming — identical to the statically generated device package +# Naming, identical to the statically generated device package # --------------------------------------------------------------------------- @@ -113,7 +113,7 @@ def test_register_and_payload_class_names_stay_verbatim(device_registers): def test_enum_names_match_generator_for_every_enum(device_registers): - """Every enum the golden module declares has identical members in the emitter.""" + """Every enum the reference module declares has identical members in the emitter.""" for name, reg in _device_registers().items(): payload = reg.payload_class if payload.payload_dtype.names is None: @@ -131,7 +131,7 @@ def test_enum_names_match_generator_for_every_enum(device_registers): # --------------------------------------------------------------------------- -# Core golden — from protocol common.yml +# Core reference output, from protocol common.yml # --------------------------------------------------------------------------- @@ -150,8 +150,8 @@ def test_core_register_structural(name, common_yml): == expected.payload_class.payload_dtype.itemsize ) if name == "DeviceName": - # Generator enriches DeviceName to interfaceType: string; protocol's - # common.yml does not, so only the layout size matches here. + # Generator enriches DeviceName to interfaceType: string, while the + # common.yml of the protocol does not, so only the layout size matches here. return assert _layout(emitted.payload_class.payload_dtype) == _layout( expected.payload_class.payload_dtype @@ -187,7 +187,7 @@ def test_struct_masked_members_roundtrip(device_registers): payload_cls = reg.payload_class pwm = payload_cls._mro_descriptor("digital_output")._enum # digital_output is a 2-bit field (mask 0xC00); only PWM0/PWM1 fit it. This - # matches the generator's output verbatim (GroupMask(enum=PwmPort, mask=0xC00)). + # matches the generator output verbatim, as GroupMask(enum=PwmPort, mask=0xC00). payload = payload_cls(digital_output=pwm["PWM1"], pulse_width=np.uint16(300)) parsed = _roundtrip(reg, payload) assert parsed.digital_output == pwm["PWM1"] @@ -232,8 +232,8 @@ def factory(ctx): regs["CustomMemberConverter"].payload_class(header=np.uint8(1), data=42), ) assert int(parsed.data) == 42 - # The factory was handed the Data field's resolved DSL context, keyed by the - # verbatim yml name — the converter symbol derives from that, not from "data". + # The factory was handed the resolved DSL context of the Data field, keyed by the + # verbatim yml name, since the converter symbol derives from that, not from "data". assert seen == {"name": "Data", "span": 2, "interface_type": "int"} @@ -250,7 +250,7 @@ def factory(ctx): def test_exclude_private_drops_private_registers(): - # Kept by default; a private register's class is underscore-prefixed, as the + # Kept by default. The class of a private register is underscore-prefixed, as the # generator emits it. assert set(create_registers(_VISIBILITY_YML)) == {"Pub", "_Priv"} assert set(create_registers(_VISIBILITY_YML, exclude_private=True)) == {"Pub"} @@ -279,7 +279,7 @@ def test_private_payload_class_is_not_prefixed(): # --------------------------------------------------------------------------- -# Payload class sharing — a structured register with an interfaceType names its +# Payload class sharing, where a structured register with an interfaceType names its # payload after that type, so registers sharing the type share one class. # --------------------------------------------------------------------------- @@ -303,7 +303,7 @@ def test_structured_register_payload_named_after_interface_type(): " Foo: {offset: 0}\n" ) assert regs["A"].payload_class.__name__ == "Shared" - # One class, reused — not two structurally identical copies. + # One class, reused, not two structurally identical copies. assert regs["A"].payload_class is regs["B"].payload_class @@ -418,7 +418,7 @@ def test_field_name_renaming_to_keyword_raises(key): # --------------------------------------------------------------------------- -# Golden bulk round-trip — the emitted register and the generator oracle are +# Reference bulk round-trip, where the emitted register and the generator oracle are # wire- and dataframe-compatible for the same payload bytes (cross read/write). # --------------------------------------------------------------------------- @@ -448,7 +448,7 @@ def test_emitted_register_bulk_matches_oracle(name, device_registers): assert buf == bytes(oracle.format_bulk(records)) # Cross-read via harp.data: the shared bytes decode to equal frames through - # either class — including column names and decoded enum labels, which now agree. + # either class, including column names and decoded enum labels, which now agree. df_emitted = parse_to_dataframe(emitted, buf, timestamp=False) df_oracle = parse_to_dataframe(oracle, buf, timestamp=False) assert list(df_emitted.columns) == list(df_oracle.columns) diff --git a/tests/device/test_naming.py b/tests/device/test_naming.py index 8795061..5f1cc4a 100644 --- a/tests/device/test_naming.py +++ b/tests/device/test_naming.py @@ -1,6 +1,6 @@ """The naming convention must match ``FirmwareNamingConvention`` in harp-tech/generators. -Every pair below is taken from the generator's own committed expected output +Every pair below is taken from the committed expected output of the generator (``tests/ExpectedOutput/{core,device}.py`` against ``tests/Metadata/{core,device}.yml``), so these lock the port to the C# behaviour rather than to a re-derivation of it. """ @@ -70,7 +70,7 @@ ("DigitalOutput", "digital_output"), ("PulseWidth", "pulse_width"), ("PulseCount", "pulse_count"), - # core.yml — a trailing capital run collapses either way it is spelled. + # core.yml, where a trailing capital run collapses either way it is spelled. ("OperationMode", "operation_mode"), ("DumpRegisters", "dump_registers"), ("MuteReplies", "mute_replies"), @@ -99,7 +99,7 @@ def test_both_conventions_share_one_casing_pass(): def test_already_converted_names_are_stable(): - # The generator's own output is a fixed point, so regenerating never drifts. + # The generator output is a fixed point, so regenerating never drifts. for _, generated in ENUM_MEMBERS: assert enum_member_name(generated) == generated for _, generated in PAYLOAD_FIELDS: diff --git a/tests/protocol/test_converter.py b/tests/protocol/test_converter.py index 9809c5d..b58fa20 100644 --- a/tests/protocol/test_converter.py +++ b/tests/protocol/test_converter.py @@ -2,7 +2,7 @@ Covers: * IdentityConverter via auto-generated _Field (parity with previous _Field). -* StringConverter (sub-array uint8 ↔ str). +* StringConverter, sub-array uint8 to str and back. * EnumConverter (full-byte enum decoding). * Declarations-build-dtype direction (no _dtype on the subclass). * Reserved-name collision check. @@ -39,7 +39,7 @@ class _Flag(enum.IntFlag): # --------------------------------------------------------------------------- -# IdentityConverter — pass-through field +# IdentityConverter, pass-through field # --------------------------------------------------------------------------- @@ -127,7 +127,7 @@ def test_string_converter_to_dataframe(): rec2 = _NamedPayload(name="bye", delta=2).payload_array.tobytes() batch = _NamedPayload.payload_from_buffer(rec1 + rec2) df = payload_to_dataframe(batch) - # Non-identity converter produces one column per field — no sub-array + # Non-identity converter produces one column per field, with no sub-array # expansion for the string field. assert list(df.columns) == ["name", "delta"] assert df["name"].tolist() == ["hi", "bye"] @@ -174,7 +174,7 @@ class _Bad(PayloadBase): def test_value_field_name_allowed(): - # ``value`` is intentionally overridable — the descriptor wins via MRO. + # ``value`` is intentionally overridable, since the descriptor wins via MRO. class _Single(PayloadBase): value = Field(converter=_StringConverter(4)) @@ -200,7 +200,7 @@ class _Flags(PayloadBase): assert scalar.flag is _Flag.A assert scalar.group is _Color.Green - # 1-D batch — Batch sibling, ndarray-typed accessors. + # 1-D batch, the Batch sibling with ndarray-typed accessors. batch = _Flags.payload_from_buffer(bytes([0x01, 0x02])) assert type(batch) is _Flags._PayloadBatchType assert isinstance(batch, _Flags) diff --git a/tests/protocol/test_framer.py b/tests/protocol/test_framer.py index a90041a..182a2bf 100644 --- a/tests/protocol/test_framer.py +++ b/tests/protocol/test_framer.py @@ -53,7 +53,7 @@ def test_bad_checksum_skipped_recovery(): def test_truncated_stream_returns_empty(): frame = make_frame_from_raw(0x01, 8, 0xFF, 0x04, b"") - # Feed only the first 3 bytes — not enough for a complete frame. + # Only the first 3 bytes, not enough for a complete frame. msgs = HarpFramer.parse_bytes(frame[:3]) assert msgs == [] diff --git a/tests/protocol/test_payload.py b/tests/protocol/test_payload.py index d4e0855..da75c91 100644 --- a/tests/protocol/test_payload.py +++ b/tests/protocol/test_payload.py @@ -65,7 +65,7 @@ def test_to_dataframe_override(): def test_from_buffer_zero_copy(): data = _make_simple_bytes(4) p = SimplePayload.payload_from_buffer(data) - # np.frombuffer returns a read-only view — writes should raise + # np.frombuffer returns a read-only view, so writes should raise with pytest.raises((ValueError, TypeError)): p.payload_array["x"][0] = 999 @@ -88,7 +88,7 @@ class _SparseModePayload(AnonymousPayload[np.uint8]): def test_groupmask_undefined_code_preserves_raw(): # Codes: defined (0->Low, 2->High), an in-range gap (1), and out-of-range (90, 255). - # Every undefined code is preserved as its raw int (like C#'s unchecked cast) — + # Every undefined code is preserved as its raw int, like the unchecked cast in C#, batch = _SparseModePayload.payload_from_buffer( np.array([0, 2, 1, 90, 255], dtype=np.uint8).tobytes() ) diff --git a/tests/protocol/test_register.py b/tests/protocol/test_register.py index 4309193..2bf5c55 100644 --- a/tests/protocol/test_register.py +++ b/tests/protocol/test_register.py @@ -168,7 +168,7 @@ def test_factory_different_addresses_are_independent(): ], ) def test_format_with_payload_instance(reg_cls, payload_cls, value): - """Passing a PayloadXxx instance to format() uses the instance's bytes directly.""" + """Passing a PayloadXxx instance to format() uses the bytes of the instance directly.""" reg = reg_cls(0x08) payload = payload_cls(value) frame = reg.format(payload) @@ -441,7 +441,7 @@ def test_parse_returns_numpy_scalar(): def test_parse_does_not_overrun_buffer(): """parse() reads exactly one record even if the buffer is larger.""" - # Two-record buffer in raw form (no Harp header — exercise raw-bytes path). + # Two-record buffer in raw form, with no Harp header, exercising the raw-bytes path. raw = np.array([42, 99], dtype=np.dtype(" Date: Fri, 14 Aug 2026 18:14:02 +0100 Subject: [PATCH 4/4] Read by register class in the dataset example The dataset quickstart now reads by register class alongside the address, taking the class from reader.device_module so the example uses one device module throughout. --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f5e292c..52f2f5f 100644 --- a/README.md +++ b/README.md @@ -71,8 +71,10 @@ from harp import data # Finds device.yml in the folder, builds the device, returns a ready-to-use reader. reader = data.create_dataset_reader("session.harp") -df = reader.read(44) # one register, by address, or pass its class -everything = reader.read_all() # {register_name: DataFrame} +behavior = reader.device_module +df = reader.read(behavior.AnalogData) # by register class +df = reader.read(44) # or by address +everything = reader.read_all() # {register_name: DataFrame} ``` Both paths are based on a device schema. Given only a `device.yml` and no pre-generated package, `create_device_module` compiles it into a module of register classes at runtime, with no code-generation step. This is exactly what `create_dataset_reader` does internally: