From a574c8cd36539b5b71da86d42fa47cd640c6fb0d Mon Sep 17 00:00:00 2001 From: glopesdev Date: Mon, 17 Aug 2026 00:02:29 +0100 Subject: [PATCH 1/5] Remove read_all from the dataset reader Loading every register of a session is now written as a comprehension over the registers that have files, calling read for each, rather than through a single call on the reader. --- README.md | 1 - docs/examples/read_dataset/read_dataset.md | 2 +- docs/examples/read_dataset/read_dataset.py | 4 --- src/packages/harp-data/README.md | 1 - .../harp-data/src/harp/data/_dataset.py | 31 ----------------- tests/data/test_dataset.py | 33 +++++++------------ 6 files changed, 12 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 52f2f5f..15ee575 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,6 @@ reader = data.create_dataset_reader("session.harp") behavior = reader.device_module df = reader.read(behavior.AnalogData) # by register class df = reader.read(44) # or by address -everything = reader.read_all() # {register_name: DataFrame} ``` Both paths are based on a device schema. Given only a `device.yml` and no pre-generated package, `create_device_module` compiles it into a module of register classes at runtime, with no code-generation step. This is exactly what `create_dataset_reader` does internally: diff --git a/docs/examples/read_dataset/read_dataset.md b/docs/examples/read_dataset/read_dataset.md index 63c7f25..62a9c11 100644 --- a/docs/examples/read_dataset/read_dataset.md +++ b/docs/examples/read_dataset/read_dataset.md @@ -4,7 +4,7 @@ A Harp acquisition is usually saved as a **de-multiplexed dataset folder**: one This is the recommended entry point for a recorded session on disk. To decode a single loose `.bin` file instead, see [Reading Data into a DataFrame](../read_data_to_dataframe/read_data_to_dataframe.md). -The quickest way in is `create_dataset_reader(folder)`. It finds the `device.yml` inside the folder, builds the device module, and returns a reader ready to go. Given a device module already in hand, for example from a pre-generated package, construct `DatasetReader(module, folder)` directly instead. A register is then read by class or by address, or every register at once with `read_all()`. Timestamps are detected automatically and placed on the `"Time"` index, as float seconds or an absolute `DatetimeIndex` when an `epoch` is passed. +The quickest way in is `create_dataset_reader(folder)`. It finds the `device.yml` inside the folder, builds the device module, and returns a reader ready to go. Given a device module already in hand, for example from a pre-generated package, construct `DatasetReader(module, folder)` directly instead. A register is then read by class or by address. Timestamps are detected automatically and placed on the `"Time"` index, as float seconds or an absolute `DatetimeIndex` when an `epoch` is passed. ```python diff --git a/docs/examples/read_dataset/read_dataset.py b/docs/examples/read_dataset/read_dataset.py index 7d0d1de..54633cd 100644 --- a/docs/examples/read_dataset/read_dataset.py +++ b/docs/examples/read_dataset/read_dataset.py @@ -26,10 +26,6 @@ df = reader.read(44) print(df.head()) -# Read every register that has a file on disk at once, keyed by register name. -everything = reader.read_all() -print(list(everything)) - # Pass an epoch to turn the "Time" index into an absolute `DatetimeIndex` instead # of float seconds. `REFERENCE_EPOCH` is time zero of the Harp clock in UTC. absolute = reader.read(44, epoch=data.REFERENCE_EPOCH) diff --git a/src/packages/harp-data/README.md b/src/packages/harp-data/README.md index 6dd4c62..262ec5f 100644 --- a/src/packages/harp-data/README.md +++ b/src/packages/harp-data/README.md @@ -28,7 +28,6 @@ reader = data.create_dataset_reader("session.harp") behavior = reader.device_module df = reader.read(behavior.AnalogData) # by register class df = reader.read(44) # by address -everything = reader.read_all() # {register_name: DataFrame} ``` Given a device module already in hand, either a pre-generated package or one built with `create_device_module`, pass it to `DatasetReader` directly: diff --git a/src/packages/harp-data/src/harp/data/_dataset.py b/src/packages/harp-data/src/harp/data/_dataset.py index 06f013c..ad8b1c8 100644 --- a/src/packages/harp-data/src/harp/data/_dataset.py +++ b/src/packages/harp-data/src/harp/data/_dataset.py @@ -47,7 +47,6 @@ class DatasetReader(Generic[M]): reader = DatasetReader(behavior, "session.harp") df = reader.read(behavior.AnalogData) # by register class df = reader.read(44) # by address - everything = reader.read_all() # {register_name: DataFrame} ``device_module`` is a device module -- a generated device package, or one built from a schema with :func:`~harp.device.schema.create_device_module`. Its ``REGISTER_MAP`` is @@ -183,36 +182,6 @@ def read( demux_bit_masks=demux_bit_masks, ) - def read_all( - self, - *, - timestamp: bool | None = None, - epoch: datetime | None = None, - message_type: bool = False, - decode_enums: bool = True, - demux_bit_masks: bool = False, - ) -> dict[str, pd.DataFrame]: - """Read every register that has a file present, keyed by register name. - - Files whose address is not among the device registers are skipped. - Options are forwarded to :meth:`read`. - """ - registers = self.registers - out: dict[str, pd.DataFrame] = {} - for address in sorted(self._files): - cls = registers.get(address) - if cls is None: - continue - out[cls.__name__] = self.read( - address, - timestamp=timestamp, - epoch=epoch, - message_type=message_type, - decode_enums=decode_enums, - demux_bit_masks=demux_bit_masks, - ) - return out - def _resolve(self, register: RegisterKey) -> tuple[type[RegisterBase[Any]], int]: if isinstance(register, type): return register, register.address diff --git a/tests/data/test_dataset.py b/tests/data/test_dataset.py index 8b042b1..ffd41a5 100644 --- a/tests/data/test_dataset.py +++ b/tests/data/test_dataset.py @@ -74,10 +74,13 @@ def test_reads_common_registers_not_named_by_module(emitted_module, tmp_path): (tmp_path / f"{mod.DEVICE_NAME}_{cls.address}.bin").write_bytes(buf) reader = DatasetReader(mod, tmp_path) - # By address, and by the class imported from harp.device, and in read_all. + # By address, and by the class imported from harp.device. assert len(reader.read(WhoAmI.address)) == 4 assert reader.read(WhoAmI).equals(reader.read(WhoAmI.address)) - assert set(reader.read_all()) == {"WhoAmI", "TimestampSeconds"} + assert {reader.registers[a].__name__ for a in reader.files} == { + "WhoAmI", + "TimestampSeconds", + } def test_timestamp_is_auto_detected(dataset): @@ -110,15 +113,6 @@ def test_epoch_gives_absolute_datetime_index(dataset): assert df.index[2] == pd.Timestamp(REFERENCE_EPOCH) + pd.Timedelta(seconds=2) -def test_read_all_keyed_by_register_name(dataset): - mod, _name, root, specs = dataset - reader = DatasetReader(mod, root) - frames = reader.read_all() - assert set(frames) == {cls.__name__ for cls, _ts, _buf in specs.values()} - for cls, _timestamped, _buf in specs.values(): - assert frames[cls.__name__].equals(reader.read(cls.address)) - - def test_suffix_chunks_are_concatenated(emitted_module, tmp_path): # Chunk suffixes in this test are ISO 8601 UTC timestamps in basic format, so # filename order is chronological order. Written newest first to test the sorting. @@ -170,7 +164,6 @@ def test_empty_dataset_reads_empty(emitted_module, tmp_path): reader = DatasetReader(emitted_module, tmp_path) assert reader.name == emitted_module.DEVICE_NAME assert reader.files == {} - assert reader.read_all() == {} def test_nameless_module_raises_on_construction(dataset): @@ -219,10 +212,8 @@ def resolver(root, _name): reader = DatasetReader(mod, tmp_path, resolver=resolver) assert set(reader.files) == set(addresses) - frames = reader.read_all() - assert set(frames) == set(expected) - for register_name, df in frames.items(): - assert df.equals(expected[register_name]) + for address in addresses: + assert reader.read(address).equals(expected[mod.REGISTER_MAP[address].__name__]) def test_files_property_lists_discovered_bins(dataset): @@ -231,7 +222,7 @@ def test_files_property_lists_discovered_bins(dataset): assert set(reader.files) == set(specs) -def test_read_all_registers_of_mock_device(emitted_module, tmp_path): +def test_every_register_round_trips(emitted_module, tmp_path): """Write one .bin per register of the device.yml device, then read them all back.""" mod = emitted_module name = mod.DEVICE_NAME @@ -246,14 +237,12 @@ def test_read_all_registers_of_mock_device(emitted_module, tmp_path): expected[cls.__name__] = parse_to_dataframe(cls, buf, timestamp=timestamped) reader = DatasetReader(mod, tmp_path) - frames = reader.read_all() assert set(reader.files) == set(mod.REGISTER_MAP) - assert set(frames) == set(expected) - assert len(frames) == len(mod.REGISTER_MAP) - for register_name, df in frames.items(): + for address, cls in mod.REGISTER_MAP.items(): + df = reader.read(address) assert len(df) == 4 - assert df.equals(expected[register_name]) + assert df.equals(expected[cls.__name__]) def test_reader_derives_name_and_registers_from_module(dataset): From f5f12a558c80753f6510ce278f4a4423caa4e8b3 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Mon, 17 Aug 2026 03:58:23 +0100 Subject: [PATCH 2/5] Add a register repr and restrict re-addressing A register now renders as its name and address rather than as , so a register map prints legibly. The repr sits on the metaclass of RegisterBase, which is what reaches structured-payload registers too, since those do not derive from the scalar bases. A base declaring no address keeps the default. Calling a register that already declares an address raises TypeError instead of returning a copy at the new address. Declaring one from a base, RegisterU32(0x08), is unchanged. --- .../src/harp/protocol/_register.py | 44 +++++++++++++------ tests/protocol/test_register.py | 19 ++++++++ 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/src/packages/harp-protocol/src/harp/protocol/_register.py b/src/packages/harp-protocol/src/harp/protocol/_register.py index 8245757..129897f 100644 --- a/src/packages/harp-protocol/src/harp/protocol/_register.py +++ b/src/packages/harp-protocol/src/harp/protocol/_register.py @@ -105,17 +105,34 @@ def __repr__(self) -> str: return repr(self._resolve()) -class _RegisterMeta(ABCMeta): - """Calling a register class with an address creates a one-off subclass: ``RegisterU32(0x08)``.""" +class _RegisterBaseMeta(ABCMeta): + """The metaclass of every register, rendering its name and address.""" + + def __repr__(cls) -> str: + address = getattr(cls, "address", None) + return super().__repr__() if address is None else f"<{cls.__name__} @{address}>" + + +def _require_no_address(cls: Any) -> None: + address = getattr(cls, "address", None) + if address is not None: + raise TypeError( + f"{cls.__name__} already declares address {address} and cannot be reassigned." + ) + + +class _ScalarRegisterMeta(_RegisterBaseMeta): + """Calling a register base with an address creates a one-off subclass: ``RegisterU32(0x08)``.""" def __call__(cls: "type[_R]", address: int) -> "type[_R]": + _require_no_address(cls) return cast( "type[_R]", type(f"{cls.__name__}_{address:#04x}", (cls,), {"address": address}), ) -class RegisterBase(ABC, Generic[U]): +class RegisterBase(ABC, Generic[U], metaclass=_RegisterBaseMeta): """Abstract base for all typed Harp registers. The generic parameter ``U`` is the static return type of :meth:`parse`, the @@ -323,73 +340,74 @@ def format( ) -class RegisterU8(RegisterBase[np.uint8], metaclass=_RegisterMeta): +class RegisterU8(RegisterBase[np.uint8], metaclass=_ScalarRegisterMeta): """A simple scalar register with a uint8 payload. ``parse()`` returns ``np.uint8``.""" payload_type: ClassVar[PayloadType] = PayloadType.U8 payload_class = PayloadU8 -class RegisterU16(RegisterBase[np.uint16], metaclass=_RegisterMeta): +class RegisterU16(RegisterBase[np.uint16], metaclass=_ScalarRegisterMeta): """A simple scalar register with a uint16 payload. ``parse()`` returns ``np.uint16``.""" payload_type: ClassVar[PayloadType] = PayloadType.U16 payload_class = PayloadU16 -class RegisterU32(RegisterBase[np.uint32], metaclass=_RegisterMeta): +class RegisterU32(RegisterBase[np.uint32], metaclass=_ScalarRegisterMeta): """A simple scalar register with a uint32 payload. ``parse()`` returns ``np.uint32``.""" payload_type: ClassVar[PayloadType] = PayloadType.U32 payload_class = PayloadU32 -class RegisterU64(RegisterBase[np.uint64], metaclass=_RegisterMeta): +class RegisterU64(RegisterBase[np.uint64], metaclass=_ScalarRegisterMeta): """A simple scalar register with a uint64 payload. ``parse()`` returns ``np.uint64``.""" payload_type: ClassVar[PayloadType] = PayloadType.U64 payload_class = PayloadU64 -class RegisterS8(RegisterBase[np.int8], metaclass=_RegisterMeta): +class RegisterS8(RegisterBase[np.int8], metaclass=_ScalarRegisterMeta): """A simple scalar register with a int8 payload. ``parse()`` returns ``np.int8``.""" payload_type: ClassVar[PayloadType] = PayloadType.S8 payload_class = PayloadS8 -class RegisterS16(RegisterBase[np.int16], metaclass=_RegisterMeta): +class RegisterS16(RegisterBase[np.int16], metaclass=_ScalarRegisterMeta): """A simple scalar register with a int16 payload. ``parse()`` returns ``np.int16``.""" payload_type: ClassVar[PayloadType] = PayloadType.S16 payload_class = PayloadS16 -class RegisterS32(RegisterBase[np.int32], metaclass=_RegisterMeta): +class RegisterS32(RegisterBase[np.int32], metaclass=_ScalarRegisterMeta): """A simple scalar register with a int32 payload. ``parse()`` returns ``np.int32``.""" payload_type: ClassVar[PayloadType] = PayloadType.S32 payload_class = PayloadS32 -class RegisterS64(RegisterBase[np.int64], metaclass=_RegisterMeta): +class RegisterS64(RegisterBase[np.int64], metaclass=_ScalarRegisterMeta): """A simple scalar register with a int64 payload. ``parse()`` returns ``np.int64``.""" payload_type: ClassVar[PayloadType] = PayloadType.S64 payload_class = PayloadS64 -class RegisterFloat(RegisterBase[np.float32], metaclass=_RegisterMeta): +class RegisterFloat(RegisterBase[np.float32], metaclass=_ScalarRegisterMeta): """A simple scalar register with a float32 payload. ``parse()`` returns ``np.float32``.""" payload_type: ClassVar[PayloadType] = PayloadType.Float payload_class = PayloadFloat -class _ArrayRegisterMeta(ABCMeta): +class _ArrayRegisterMeta(_RegisterBaseMeta): """A base metaclass for array registers. Calling with address and length creates a concrete subclass: ``RegisterU16Array(0x28, length=3)``.""" def __call__(cls: "type[_AR]", address: int, *, length: int) -> "type[_AR]": # type: ignore[override, misc] + _require_no_address(cls) base_payload = cls.payload_class # type: ignore[attr-defined] # Anonymous payloads carry a plain (non-structured) dtype. The array # variant uses a sub-dtype (inner_dtype, (length,)) so a single buffer diff --git a/tests/protocol/test_register.py b/tests/protocol/test_register.py index 2bf5c55..5bf2a16 100644 --- a/tests/protocol/test_register.py +++ b/tests/protocol/test_register.py @@ -157,6 +157,25 @@ def test_factory_different_addresses_are_independent(): assert r1 is not r2 +def test_declared_register_raises_type_error(): + # A declared address cannot be reassigned. + declared = RegisterU32(0x08) + with pytest.raises(TypeError, match="already declares address"): + declared(0x09) + + +def test_declared_array_register_raises_type_error(): + declared = RegisterU32Array(0x28, length=3) + with pytest.raises(TypeError, match="already declares address"): + declared(0x29, length=3) + + +def test_register_repr_shows_name_and_address(): + assert repr(RegisterU32(0x08)) == "" + # A base declares no address, so it keeps the default class repr. + assert repr(RegisterU32).startswith(" Date: Mon, 17 Aug 2026 04:51:05 +0100 Subject: [PATCH 3/5] Replace create_dataset_reader with open_dataset open_dataset replaces create_dataset_reader and takes an optional device module as its second argument, so a folder with its own device.yml and a pre-generated package both reach the same reader. Passing schema= or converters= beside a module now raises TypeError. Identity is checked only when the module was not built from the folder schema. read accepts a register name alongside a class and an address, resolved through the device register map rather than the module namespace. contents maps register name to address for every register with data in the folder, so what a dataset holds and what read takes are the same key. The registers property is gone, since it only repeated device_module.REGISTER_MAP, and files becomes paths. The default resolver now returns addresses in numeric order. --- README.md | 24 ++- docs/api/data.md | 2 +- docs/examples/index.md | 2 +- .../read_data_to_dataframe.md | 2 +- .../read_data_to_dataframe.py | 6 +- docs/examples/read_dataset/read_dataset.md | 2 +- docs/examples/read_dataset/read_dataset.py | 33 ++-- src/packages/harp-data/README.md | 31 ++-- .../harp-data/src/harp/data/__init__.py | 4 +- .../harp-data/src/harp/data/_dataset.py | 142 +++++++++++----- tests/conformance.py | 9 +- tests/data/test_dataset.py | 151 ++++++++++++++---- 12 files changed, 301 insertions(+), 107 deletions(-) diff --git a/README.md b/README.md index 15ee575..e6e0a4c 100644 --- a/README.md +++ b/README.md @@ -69,14 +69,26 @@ with serial.open_serial_device(behavior, port="COM3") as device: ```python from harp import data -# Finds device.yml in the folder, builds the device, returns a ready-to-use reader. -reader = data.create_dataset_reader("session.harp") -behavior = reader.device_module -df = reader.read(behavior.AnalogData) # by register class -df = reader.read(44) # or by address +# Finds device.yml in the folder, builds the device, returns a ready-to-use reader +reader = data.open_dataset("session.harp") +df = reader.read("AnalogData") # by name +df = reader.read(44) # or by address + +# `contents` names every register the folder holds +frames = {name: reader.read(name) for name in reader.contents} +``` + +Given a device package already in hand, pass it as the second argument and read by register class. This is the form that type-checks, and it also checks the device identity against the `device.yml` in the folder: + +```python +from harp import data +from harp.device import behavior + +reader = data.open_dataset("session.harp", behavior) +df = reader.read(behavior.AnalogData) ``` -Both paths are based on a device schema. Given only a `device.yml` and no pre-generated package, `create_device_module` compiles it into a module of register classes at runtime, with no code-generation step. This is exactly what `create_dataset_reader` does internally: +Both paths are based on a device schema. Given only a `device.yml` and no pre-generated package, `create_device_module` compiles it into a module of register classes at runtime, with no code-generation step. This is exactly what `open_dataset` does internally: ```python from pathlib import Path diff --git a/docs/api/data.md b/docs/api/data.md index c66a43f..bc80888 100644 --- a/docs/api/data.md +++ b/docs/api/data.md @@ -2,7 +2,7 @@ --- -::: harp.data.create_dataset_reader +::: harp.data.open_dataset ::: harp.data.DatasetReader ::: harp.data.default_file_resolver ::: harp.data.parse_to_dataframe diff --git a/docs/examples/index.md b/docs/examples/index.md index cf3ca62..fa005ab 100644 --- a/docs/examples/index.md +++ b/docs/examples/index.md @@ -14,5 +14,5 @@ Talking to a device: Reading recorded data: -- [Reading a Whole Dataset Folder](./read_dataset/read_dataset.md) - load an entire recorded session folder into pandas DataFrames with `DatasetReader`. +- [Reading a Whole Dataset Folder](./read_dataset/read_dataset.md) - read registers from a recorded session folder into pandas DataFrames, decoded against the device schema. - [Reading Data into a DataFrame](./read_data_to_dataframe/read_data_to_dataframe.md) - decode the binary file of a single register into a pandas DataFrame. diff --git a/docs/examples/read_data_to_dataframe/read_data_to_dataframe.md b/docs/examples/read_data_to_dataframe/read_data_to_dataframe.md index 5f47f0b..94e1479 100644 --- a/docs/examples/read_data_to_dataframe/read_data_to_dataframe.md +++ b/docs/examples/read_data_to_dataframe/read_data_to_dataframe.md @@ -3,7 +3,7 @@ This example demonstrates how to load the binary data file of a **single** Harp register into a pandas DataFrame using `harp.data`. The register definition tells `parse_to_dataframe` how to decode each frame, so the result carries named columns and decoded enums. !!! tip - For a whole recorded session folder rather than one loose file, use [`DatasetReader`](../read_dataset/read_dataset.md), which reads every register in a dataset folder based on the device schema. + For a recorded session folder rather than one loose file, use [`open_dataset`](../read_dataset/read_dataset.md), which resolves each register against the device schema so any of them can be read by class, by name, or by address. ```python diff --git a/docs/examples/read_data_to_dataframe/read_data_to_dataframe.py b/docs/examples/read_data_to_dataframe/read_data_to_dataframe.py index d236fa7..2c24804 100644 --- a/docs/examples/read_data_to_dataframe/read_data_to_dataframe.py +++ b/docs/examples/read_data_to_dataframe/read_data_to_dataframe.py @@ -15,6 +15,6 @@ with open("OperationControl.bin", "rb") as f: df = data.parse_to_dataframe(core.OperationControl, f) -# To read a whole recorded session folder at once, covering many registers based on -# the device schema, use `harp.data.DatasetReader`. See the "Reading a Whole Dataset -# Folder" example. +# To read registers from a recorded session folder, resolved against the device +# schema, use `harp.data.open_dataset`. See the "Reading a Whole Dataset Folder" +# example. diff --git a/docs/examples/read_dataset/read_dataset.md b/docs/examples/read_dataset/read_dataset.md index 62a9c11..7764092 100644 --- a/docs/examples/read_dataset/read_dataset.md +++ b/docs/examples/read_dataset/read_dataset.md @@ -4,7 +4,7 @@ A Harp acquisition is usually saved as a **de-multiplexed dataset folder**: one This is the recommended entry point for a recorded session on disk. To decode a single loose `.bin` file instead, see [Reading Data into a DataFrame](../read_data_to_dataframe/read_data_to_dataframe.md). -The quickest way in is `create_dataset_reader(folder)`. It finds the `device.yml` inside the folder, builds the device module, and returns a reader ready to go. Given a device module already in hand, for example from a pre-generated package, construct `DatasetReader(module, folder)` directly instead. A register is then read by class or by address. Timestamps are detected automatically and placed on the `"Time"` index, as float seconds or an absolute `DatetimeIndex` when an `epoch` is passed. +The quickest way in is `open_dataset(folder)`. It finds the `device.yml` inside the folder, builds the device module, and returns a reader ready to go. Given a device module already in hand, for example from a pre-generated package, pass it as the second argument, `open_dataset(folder, module)`. A register is then read by class, by name, or by address. Timestamps are detected automatically and placed on the `"Time"` index, as float seconds or an absolute `DatetimeIndex` when an `epoch` is passed. ```python diff --git a/docs/examples/read_dataset/read_dataset.py b/docs/examples/read_dataset/read_dataset.py index 54633cd..6b48c06 100644 --- a/docs/examples/read_dataset/read_dataset.py +++ b/docs/examples/read_dataset/read_dataset.py @@ -11,29 +11,41 @@ # ┣ ... # ┗ 📜 device.yml # -# `create_dataset_reader` does the right thing: it finds `device.yml` inside the -# folder, builds the module of register classes that knows how to decode each -# register, and hands back a reader ready to go. -reader = data.create_dataset_reader("session.harp") +# `open_dataset` does the right thing: it finds `device.yml` inside the folder, +# builds the module of register classes that knows how to decode each register, +# and hands back a reader ready to go. +reader = data.open_dataset("session.harp") # Read one register into a DataFrame by register class, which covers any register # in the device map, including common ones such as `OperationControl`. df = reader.read(core.OperationControl) -# A register can also be read by address. Timestamps are auto-detected from the -# frames, and when present they become the DataFrame index, named "Time", holding -# float seconds from device start. +# A register can also be read by name. Names resolve through the device register +# map rather than the module namespace, so common registers are reachable too. +df = reader.read("OperationControl") + +# Or by address. Timestamps are auto-detected from the frames, and when present +# they become the DataFrame index, named "Time", holding float seconds from +# device start. df = reader.read(44) print(df.head()) +# `contents` names every register that has data in this folder. Most datasets log +# every register by default. +frames = {name: reader.read(name) for name in reader.contents} +print(list(frames)) + # Pass an epoch to turn the "Time" index into an absolute `DatetimeIndex` instead # of float seconds. `REFERENCE_EPOCH` is time zero of the Harp clock in UTC. absolute = reader.read(44, epoch=data.REFERENCE_EPOCH) print(absolute.index[:3]) # --- Working from a device module already in hand ---------------------------- -# A pre-generated device package, or one built with `create_device_module`, -# can be passed to the reader directly as `DatasetReader(module, folder)`: +# A pre-generated device package, or one built with `create_device_module`, is +# passed as the second argument. Either way the device identity is checked against +# the `device.yml` in the folder, so a module paired with the wrong session fails +# here rather than decoding against the wrong register map. A generated package +# adds register classes a type checker can verify: # # from pathlib import Path # @@ -41,4 +53,5 @@ # from harp.device import schema # # behavior = schema.create_device_module((Path("session.harp") / "device.yml").read_bytes()) -# reader = data.DatasetReader(behavior, "session.harp") +# reader = data.open_dataset("session.harp", behavior) +# df = reader.read(behavior.AnalogData) diff --git a/src/packages/harp-data/README.md b/src/packages/harp-data/README.md index 262ec5f..a2d803a 100644 --- a/src/packages/harp-data/README.md +++ b/src/packages/harp-data/README.md @@ -19,32 +19,43 @@ A Harp acquisition is usually saved as a de-multiplexed folder, one binary file ┗ 📜 device.yml ``` -Reading is based on a [device module](../harp-device) that describes how to decode each register. `create_dataset_reader` supplies one automatically. It finds the `device.yml` in the folder, builds the module, and returns a ready-to-use reader: +Reading is based on a [device module](../harp-device) that describes how to decode each register. `open_dataset` supplies one automatically. It finds the `device.yml` in the folder, builds the module, and returns a ready-to-use reader: ```python from harp import data -reader = data.create_dataset_reader("session.harp") -behavior = reader.device_module -df = reader.read(behavior.AnalogData) # by register class -df = reader.read(44) # by address +reader = data.open_dataset("session.harp") +df = reader.read("AnalogData") # by name +df = reader.read(44) # by address ``` -Given a device module already in hand, either a pre-generated package or one built with `create_device_module`, pass it to `DatasetReader` directly: +`contents` maps every register with data in the folder to its address, keyed by register name. It is the place to start on an unfamiliar dataset, and since its keys are exactly what `read` takes, loading a whole dataset can be done with a comprehension: + +```python +reader.contents # {'WhoAmI': 0, 'AnalogData': 33, ...} + +frames = {name: reader.read(name) for name in reader.contents} +``` + +A name is resolved through the device register map rather than the module namespace, so the common registers are reachable by name too. + +Given a device module already in hand, either a pre-generated package or one built with `create_device_module`, pass it as the second argument: ```python from harp import data from harp.device import behavior -reader = data.DatasetReader(behavior, "session.harp") -df = reader.read(behavior.AnalogData) +reader = data.open_dataset("session.harp", behavior) +df = reader.read(behavior.AnalogData) # by register class ``` -Timestamps are auto-detected per register and placed on the DataFrame index named `"Time"`: float seconds by default, or an absolute `DatetimeIndex` when `epoch=REFERENCE_EPOCH` is passed. Multi-chunk registers logged as `_
_.bin` are concatenated in filename order; pass a `resolver` to support an alternative on-disk layout. +Prefer the register class where a generated package supplies one, since it is the only form that type-checks and a misspelling is caught before the folder is read. A module built by `create_device_module` resolves its registers as `Any`, so there the class verifies no more than the name does. + +Timestamps are auto-detected per register and placed on the DataFrame index named `"Time"`: float seconds by default, or an absolute `DatetimeIndex` when `epoch=REFERENCE_EPOCH` is passed. Multi-chunk registers logged as `_
_.bin` are concatenated in filename order; pass a `resolver` to support an alternative on-disk layout. `paths` reports what the resolver found, keyed by address, which is where a custom layout or a chunked register can be checked. The `` prefix comes from the `DEVICE_NAME` declared by the device module. Pass `name=` to override it, or to supply one when the module declares an empty name. -When the folder carries a `device.yml` and the module declares an identity, their `whoAmI` values are checked against each other. Reusing a module across sessions and reaching the wrong folder then fails on construction rather than decoding the files against the wrong register map. Pass `validate=False` to turn off every check the reader performs, so a folder whose `device.yml` is damaged can be read with a module obtained elsewhere. +When a device module declaring an identity is supplied and the folder carries a `device.yml`, their `whoAmI` values are checked against each other. Reusing a module across sessions and reaching the wrong folder then fails on construction rather than decoding the files against the wrong register map. Pass `validate=False` to turn off every check the reader performs, so a folder whose `device.yml` is damaged can be read with a module obtained elsewhere. ## Read a single register file diff --git a/src/packages/harp-data/src/harp/data/__init__.py b/src/packages/harp-data/src/harp/data/__init__.py index 39adede..ea4b57f 100644 --- a/src/packages/harp-data/src/harp/data/__init__.py +++ b/src/packages/harp-data/src/harp/data/__init__.py @@ -1,4 +1,4 @@ -from ._dataset import DatasetReader, create_dataset_reader, default_file_resolver +from ._dataset import DatasetReader, default_file_resolver, open_dataset from ._read import read from ._reader import REFERENCE_EPOCH, parse_to_dataframe, payload_to_dataframe from ._write import to_buffer, to_file @@ -10,7 +10,7 @@ "to_buffer", "to_file", "DatasetReader", - "create_dataset_reader", + "open_dataset", "default_file_resolver", "REFERENCE_EPOCH", ] diff --git a/src/packages/harp-data/src/harp/data/_dataset.py b/src/packages/harp-data/src/harp/data/_dataset.py index ad8b1c8..71c8841 100644 --- a/src/packages/harp-data/src/harp/data/_dataset.py +++ b/src/packages/harp-data/src/harp/data/_dataset.py @@ -3,7 +3,7 @@ from datetime import datetime from os import PathLike from pathlib import Path -from typing import Any, Generic, TypeVar +from typing import Any, Generic, TypeVar, overload import pandas as pd from harp.device.schema import ( @@ -19,7 +19,7 @@ M = TypeVar("M", bound=DeviceModuleLike) -RegisterKey = type[RegisterBase[Any]] | int +RegisterKey = type[RegisterBase[Any]] | int | str FileNameResolver = Callable[[Path, str], Mapping[int, list[Path]]] @@ -35,22 +35,26 @@ def default_file_resolver(root: Path, name: str) -> dict[int, list[Path]]: match = pattern.match(path.stem) if match is not None: files.setdefault(int(match.group(1)), []).append(path) - return files + return dict(sorted(files.items())) class DatasetReader(Generic[M]): """Reader over a de-multiplexed Harp dataset folder. - Construct from a device module and a dataset folder, then read the - frames of a register into a DataFrame by register class or by address:: + Construct from a device module and a dataset folder, then read the frames of a + register into a DataFrame by register class, by name, or by address:: reader = DatasetReader(behavior, "session.harp") df = reader.read(behavior.AnalogData) # by register class + df = reader.read("AnalogData") # by name df = reader.read(44) # by address + :func:`open_dataset` builds one for a folder that carries its own ``device.yml``. + :attr:`contents` lists what was recorded, keyed by register name. + ``device_module`` is a device module -- a generated device package, or one built from a schema with :func:`~harp.device.schema.create_device_module`. Its ``REGISTER_MAP`` is - read on demand. + read at construction. The files are matched by a ```` prefix, taken from the ``DEVICE_NAME`` declared by the module. Pass ``name`` to override it, or to supply one when the module @@ -85,7 +89,9 @@ def __init__( self._name_override = name self._resolver = resolver self._name = self._resolve_name() - self._files = dict(self._resolver(self._root, self._name)) + self._paths = dict(self._resolver(self._root, self._name)) + registers = device_module.REGISTER_MAP + self._name_map = {registers[address].__name__: address for address in sorted(registers)} if validate: self._validate_whoami() @@ -139,14 +145,14 @@ def _resolve_name(self) -> str: ) @property - def registers(self) -> Mapping[int, type[RegisterBase[Any]]]: - """The address -> register-class map the module carries as ``REGISTER_MAP``.""" - return self._device_module.REGISTER_MAP + def contents(self) -> Mapping[str, int]: + """The mapping from register name to address for registers with data under :attr:`root`.""" + return {name: address for name, address in self._name_map.items() if address in self._paths} @property - def files(self) -> Mapping[int, list[Path]]: - """The discovered address -> binary file(s) present under :attr:`root`.""" - return self._files + def paths(self) -> Mapping[int, list[Path]]: + """The mapping from address to binary files discovered under :attr:`root`.""" + return self._paths def read( self, @@ -161,15 +167,22 @@ def read( ) -> pd.DataFrame: """Read the data of one register into a DataFrame. - ``register`` is a register class or an address. ``suffix`` selects a single - ``_
_.bin`` chunk (default: concatenate every chunk - for the address). ``timestamp`` defaults to ``None``, auto-detecting from the - payload-type bit of the frame; pass ``True``/``False`` to force. ``epoch`` makes - the ``"Time"`` index absolute (e.g. :data:`~harp.data.REFERENCE_EPOCH`). The - remaining options match :func:`~harp.data.parse_to_dataframe`. + ``register`` is a register class, a register name, or an address. Names are + resolved through the device register map rather than the module namespace, which + declares no common registers. Prefer the class where a generated package supplies + one, since it is the only form a type checker can verify. A module built by + :func:`~harp.device.schema.create_device_module` resolves its registers as + ``Any``, so there the generated module verifies no more than the name does. + + ``suffix`` selects a single ``_
_.bin`` chunk (default: + concatenate every chunk for the address). ``timestamp`` defaults to ``None``, + auto-detecting from the payload-type bit of the frame; pass ``True``/``False`` + to force. ``epoch`` makes the ``"Time"`` index absolute (e.g. + :data:`~harp.data.REFERENCE_EPOCH`). The remaining options match + :func:`~harp.data.parse_to_dataframe`. """ cls, address = self._resolve(register) - paths = self._resolve_files(address, suffix) + paths = self._resolve_paths(address, suffix) raw = b"".join(p.read_bytes() for p in paths) ts = self._first_frame_timestamped(raw) if timestamp is None else timestamp return parse_to_dataframe( @@ -185,13 +198,19 @@ def read( def _resolve(self, register: RegisterKey) -> tuple[type[RegisterBase[Any]], int]: if isinstance(register, type): return register, register.address - cls = self.registers.get(register) + registers = self._device_module.REGISTER_MAP + if isinstance(register, str): + address = self._name_map.get(register) + if address is None: + raise KeyError(f"No register named {register!r} in the map of this device.") + return registers[address], address + cls = registers.get(register) if cls is None: raise KeyError(f"No register at address {register} in the map of this device.") return cls, register - def _resolve_files(self, address: int, suffix: str | None) -> list[Path]: - paths = self._files.get(address) + def _resolve_paths(self, address: int, suffix: str | None) -> list[Path]: + paths = self._paths.get(address) if not paths: raise FileNotFoundError( f"No data file for register address {address} under {self._root} " @@ -211,8 +230,34 @@ def _first_frame_timestamped(raw: bytes) -> bool: return len(raw) > 4 and bool(raw[4] & _TIMESTAMP_FLAG) -def create_dataset_reader( +@overload +def open_dataset( + root: str | PathLike[str], + device_module: M, + *, + name: str | None = ..., + resolver: FileNameResolver = ..., + validate: bool = ..., +) -> DatasetReader[M]: ... + + +@overload +def open_dataset( + root: str | PathLike[str], + device_module: None = ..., + *, + schema: str | PathLike[str] | None = ..., + name: str | None = ..., + resolver: FileNameResolver = ..., + converters: Mapping[str, Any] | None = ..., + require_converters: bool = ..., + validate: bool = ..., +) -> DatasetReader[DeviceModule]: ... + + +def open_dataset( root: str | PathLike[str], + device_module: DeviceModuleLike | None = None, *, schema: str | PathLike[str] | None = None, name: str | None = None, @@ -220,35 +265,50 @@ def create_dataset_reader( converters: Mapping[str, Any] | None = None, require_converters: bool = True, validate: bool = True, -) -> DatasetReader[DeviceModule]: - """Build a :class:`DatasetReader` for a dataset folder, device and all. +) -> DatasetReader: + """Open a de-multiplexed Harp dataset folder and return a :class:`DatasetReader`. - Convenience wrapper that finds the device schema inside ``root`` (``device.yml`` - by default), builds its module with :func:`~harp.device.schema.create_device_module`, and - returns a reader ready to :meth:`~DatasetReader.read`:: + If the device module is omitted, the schema file inside the folder will be used. + The ``device.yml`` inside ``root`` is first built into a module using + :func:`~harp.device.schema.create_device_module`. - reader = create_dataset_reader("session.harp") - df = reader.read(44) + If a device module is provided, its identity class will be used to validate the + dataaset, and a generated package additionally carries register classes a + type checker can verify. - ``schema`` points at the schema file explicitly when it isn't ``root/device.yml``. + ``schema`` points at the schema file when it isn't ``root/device.yml``, and ``converters`` and ``require_converters`` are forwarded to :func:`~harp.device.schema.create_device_module` for custom ``interfaceType`` - decoding; ``name``, ``resolver`` and ``validate`` are forwarded to - :class:`DatasetReader`. Use ``DatasetReader(device_module, root)`` directly given a - device module already in hand, for example a pre-generated one. + decoding. These three parameters describe alternative ways to supply a module, so + they are mutually exclusive, and will raise when more than one is specified. - Note ``validate`` cannot rescue a damaged ``device.yml`` here, since the module is - built from that same file and fails before the reader exists. Reading such a folder - means supplying a module obtained elsewhere. + ``validate`` cannot rescue a corrupt ``device.yml`` if that schema file is also + used to build the module. Reading such a folder always requires supplying a module + obtained elsewhere. """ root_path = Path(root) + if device_module is not None: + if schema is not None or converters is not None: + raise TypeError( + "schema= and converters= describe how to build a device module, so they " + "do not apply when one is given. Drop them, or drop the device module." + ) + return DatasetReader( + device_module, root_path, name=name, resolver=resolver, validate=validate + ) schema_path = Path(schema) if schema is not None else root_path / DEVICE_SCHEMA_FILENAME if not schema_path.is_file(): raise FileNotFoundError( f"No device schema at '{schema_path}'. Pass schema= to point at a device.yml, " - f"or build the device module yourself and use DatasetReader(device_module, root)." + f"or pass the device module itself as open_dataset(root, device_module)." ) - device_module = create_device_module( + built = create_device_module( schema_path.read_text(), converters=converters, require_converters=require_converters ) - return DatasetReader(device_module, root_path, name=name, resolver=resolver, validate=validate) + return DatasetReader( + built, + root_path, + name=name, + resolver=resolver, + validate=validate and schema is not None, + ) diff --git a/tests/conformance.py b/tests/conformance.py index 4fe587c..fce5e5c 100644 --- a/tests/conformance.py +++ b/tests/conformance.py @@ -8,7 +8,7 @@ from typing import Any, ClassVar, assert_type import numpy as np -from harp.data import DatasetReader, create_dataset_reader +from harp.data import DatasetReader, open_dataset from harp.device.client import Device, ITransport from harp.device.core import OperationControl, OperationControlPayload, WhoAmI from harp.device.schema import DeviceModule, DeviceModuleLike, create_device_module @@ -104,12 +104,17 @@ def dataset_reader_registers_resolve_through_the_module() -> None: the ceiling here is the one :func:`create_device_module` documents. A generated package carries its own declarations and resolves each to its own class. """ - reader = create_dataset_reader("session.harp") + reader = open_dataset("session.harp") assert_type(reader, DatasetReader[DeviceModule]) assert_type(reader.device_module.AnalogData, Any) reader.read(reader.device_module.AnalogData) +def open_dataset_keeps_supplied_module_type(generated: DeviceModuleLike) -> None: + """A module passed through the entry point types the reader on itself.""" + assert_type(open_dataset("session.harp", generated), DatasetReader[DeviceModuleLike]) + + def open_serial_device_prefers_the_subclass_overload() -> None: """A Device subclass is matched as a subclass even when it looks like a module. diff --git a/tests/data/test_dataset.py b/tests/data/test_dataset.py index ffd41a5..d5f0d1c 100644 --- a/tests/data/test_dataset.py +++ b/tests/data/test_dataset.py @@ -6,7 +6,7 @@ from harp.data import ( REFERENCE_EPOCH, DatasetReader, - create_dataset_reader, + open_dataset, parse_to_dataframe, ) from harp.device.core import TimestampSeconds, WhoAmI @@ -54,7 +54,7 @@ def test_read_by_class_and_by_address(dataset): assert reader.read(address).equals(expected) -def test_read_by_name_from_module(dataset): +def test_read_by_class_from_module_namespace(dataset): mod, _name, root, specs = dataset reader = DatasetReader(mod, root) for address, (cls, _timestamped, _buf) in specs.items(): @@ -62,9 +62,23 @@ def test_read_by_name_from_module(dataset): assert reader.read(getattr(mod, cls.__name__)).equals(reader.read(address)) +def test_read_by_register_name(dataset): + mod, _name, root, specs = dataset + reader = DatasetReader(mod, root) + for address, (cls, _timestamped, _buf) in specs.items(): + assert reader.read(cls.__name__).equals(reader.read(address)) + + +def test_unknown_name_raises_key_error(dataset): + mod, _name, root, _specs = dataset + reader = DatasetReader(mod, root) + with pytest.raises(KeyError): + reader.read("NotARegister") + + def test_reads_common_registers_not_named_by_module(emitted_module, tmp_path): - """A device module names only its own registers, but a session folder also holds - files for the common ones, so the reader must still decode those.""" + # A device module names only its own registers, but a session folder also holds + # files for the common ones, so the reader must still decode those. mod = emitted_module assert not hasattr(mod, "WhoAmI") # imported from harp.device, not re-exported @@ -74,13 +88,12 @@ def test_reads_common_registers_not_named_by_module(emitted_module, tmp_path): (tmp_path / f"{mod.DEVICE_NAME}_{cls.address}.bin").write_bytes(buf) reader = DatasetReader(mod, tmp_path) - # By address, and by the class imported from harp.device. + # By address, by the class imported from harp.device, and by name, which resolves + # through the register map and so reaches further than the module namespace. assert len(reader.read(WhoAmI.address)) == 4 assert reader.read(WhoAmI).equals(reader.read(WhoAmI.address)) - assert {reader.registers[a].__name__ for a in reader.files} == { - "WhoAmI", - "TimestampSeconds", - } + assert reader.read("WhoAmI").equals(reader.read(WhoAmI.address)) + assert set(reader.contents) == {"WhoAmI", "TimestampSeconds"} def test_timestamp_is_auto_detected(dataset): @@ -135,13 +148,11 @@ def test_suffix_chunks_are_concatenated(emitted_module, tmp_path): assert reader.read(cls, suffix="20260816T090000Z").equals(earliest) -def test_non_module_raises_on_register_access(dataset): +def test_non_module_raises_on_construction(dataset): + # The register map is read at construction, so a module without one is rejected. _mod, name, root, _specs = dataset - # Registers are derived lazily; anything without a REGISTER_MAP fails on access. - # name and validate keep construction from reading the module at all. - reader = DatasetReader(object, root, name=name, validate=False) with pytest.raises(AttributeError, match="REGISTER_MAP"): - _ = reader.registers + DatasetReader(object, root, name=name, validate=False) def test_explicit_name_overrides(dataset): @@ -163,7 +174,7 @@ def test_empty_dataset_reads_empty(emitted_module, tmp_path): # A session that logged nothing is a dataset with no data, not a failure. reader = DatasetReader(emitted_module, tmp_path) assert reader.name == emitted_module.DEVICE_NAME - assert reader.files == {} + assert reader.paths == {} def test_nameless_module_raises_on_construction(dataset): @@ -211,19 +222,65 @@ def resolver(root, _name): return found reader = DatasetReader(mod, tmp_path, resolver=resolver) - assert set(reader.files) == set(addresses) + assert set(reader.paths) == set(addresses) for address in addresses: assert reader.read(address).equals(expected[mod.REGISTER_MAP[address].__name__]) -def test_files_property_lists_discovered_bins(dataset): +def test_paths_property_lists_discovered_bins(dataset): + mod, _name, root, specs = dataset + reader = DatasetReader(mod, root) + assert set(reader.paths) == set(specs) + + +def test_contents_maps_names_to_addresses(dataset): + mod, _name, root, specs = dataset + reader = DatasetReader(mod, root) + # The declared address space is much larger; contents is what this folder holds. + assert reader.contents == {cls.__name__: address for address, (cls, _ts, _buf) in specs.items()} + assert len(mod.REGISTER_MAP) > len(reader.contents) + + +def test_contents_sorted_by_address(dataset): + mod, name, root, _specs = dataset + cls = mod.REGISTER_MAP[0] + (root / f"{name}_0.bin").write_bytes(bytes(cls.format_bulk(_records(cls, 2, seed=0)))) + + reader = DatasetReader(mod, root) + + assert list(reader.contents.values()) == sorted(reader.contents.values()) + assert next(iter(reader.contents)) == "WhoAmI" + + +def test_contents_keys_read_every_register(dataset): + # The comprehension over contents is what replaces a load-everything call, so its + # keys must reach every register with data and produce the same frames as a direct read. mod, _name, root, specs = dataset reader = DatasetReader(mod, root) - assert set(reader.files) == set(specs) + + frames = {name: reader.read(name) for name in reader.contents} + + assert set(frames) == {cls.__name__ for cls, _ts, _buf in specs.values()} + for cls, timestamped, buf in specs.values(): + assert frames[cls.__name__].equals(parse_to_dataframe(cls, buf, timestamp=timestamped)) + + +def test_contents_excludes_undescribed_address(dataset): + # A file at an address not described by the register map cannot be named, but it + # stays visible in paths. + mod, name, root, specs = dataset + undescribed = max(mod.REGISTER_MAP) + 1 + (root / f"{name}_{undescribed}.bin").write_bytes(b"") + + reader = DatasetReader(mod, root) + + assert undescribed in reader.paths + assert undescribed not in reader.contents.values() + assert set(reader.contents) == {cls.__name__ for cls, _ts, _buf in specs.values()} def test_every_register_round_trips(emitted_module, tmp_path): - """Write one .bin per register of the device.yml device, then read them all back.""" + # Write one .bin per register of the device.yml device, then read them all back. mod = emitted_module name = mod.DEVICE_NAME expected = {} @@ -238,27 +295,27 @@ def test_every_register_round_trips(emitted_module, tmp_path): reader = DatasetReader(mod, tmp_path) - assert set(reader.files) == set(mod.REGISTER_MAP) + assert set(reader.paths) == set(mod.REGISTER_MAP) for address, cls in mod.REGISTER_MAP.items(): df = reader.read(address) assert len(df) == 4 assert df.equals(expected[cls.__name__]) -def test_reader_derives_name_and_registers_from_module(dataset): - mod, name, root, _specs = dataset +def test_reader_derives_name_and_contents_from_module(dataset): + mod, name, root, specs = dataset reader = DatasetReader(mod, root) assert reader.device_module is mod assert reader.name == name - assert reader.registers == mod.REGISTER_MAP + assert set(reader.contents.values()) == set(specs) -def test_create_dataset_reader_builds_module_from_device_yml(dataset, device_yml): +def test_open_dataset_builds_module_from_device_yml(dataset, device_yml): mod, _name, root, specs = dataset (root / "device.yml").write_text(device_yml) # require_converters=False mirrors the emitted_module fixture, which does not # inject the custom DataConverter either. - reader = create_dataset_reader(root, require_converters=False) + reader = open_dataset(root, require_converters=False) assert isinstance(reader, DatasetReader) # Reads match a reader built from an explicitly-generated module. reference = DatasetReader(mod, root) @@ -266,15 +323,38 @@ def test_create_dataset_reader_builds_module_from_device_yml(dataset, device_yml assert reader.read(address).equals(reference.read(cls)) -def test_create_dataset_reader_accepts_explicit_schema_path(dataset, device_yml, tmp_path): +def test_open_dataset_accepts_explicit_schema_path(dataset, device_yml, tmp_path): _mod, _name, root, specs = dataset schema_path = tmp_path / "elsewhere.yml" # not inside the dataset folder schema_path.write_text(device_yml) - reader = create_dataset_reader(root, schema=schema_path, require_converters=False) + reader = open_dataset(root, schema=schema_path, require_converters=False) address = next(iter(specs)) assert not reader.read(address).empty +def test_open_dataset_accepts_device_module(dataset): + # The overload taking a module must reach the same reader as constructing one, + # since it is the only route open to a pre-generated package here. + mod, _name, root, specs = dataset + reader = open_dataset(root, mod) + reference = DatasetReader(mod, root) + assert reader.device_module is mod + assert reader.name == reference.name + for address in specs: + assert reader.read(address).equals(reference.read(address)) + + +def test_open_dataset_rejects_schema_beside_module(dataset, device_yml, tmp_path): + # Both describe how to build a module, so accepting them together would ignore one. + mod, _name, root, _specs = dataset + schema_path = tmp_path / "elsewhere.yml" + schema_path.write_text(device_yml) + with pytest.raises(TypeError, match="device module"): + open_dataset(root, mod, schema=schema_path) + with pytest.raises(TypeError, match="device module"): + open_dataset(root, mod, converters={}) + + def _with_whoami(device_yml: str, who_am_i: int) -> str: return f"whoAmI: {who_am_i}\n{device_yml}" @@ -362,7 +442,20 @@ def test_corrupt_schema_is_not_skipped(dataset, device_yml): assert not isinstance(excinfo.value, ValueError) -def test_create_dataset_reader_missing_schema_raises(dataset): +def test_open_dataset_missing_schema_raises_file_not_found(dataset): _mod, _name, root, _specs = dataset # no device.yml written into the folder with pytest.raises(FileNotFoundError, match="device.yml"): - create_dataset_reader(root) + open_dataset(root) + + +def test_external_schema_mismatch_raises_on_construction(dataset, device_yml, tmp_path): + # Building the module from a schema outside the folder leaves the two free to + # disagree, so the check still runs. Omitting schema= builds it from the folder + # itself, where they agree by construction and the check is skipped. + _mod, _name, root, _specs = dataset + (root / "device.yml").write_text(_with_whoami(device_yml, 1234)) + schema_path = tmp_path / "elsewhere.yml" + schema_path.write_text(_with_whoami(device_yml, 1216)) + + with pytest.raises(ValueError, match="WhoAmI mismatch"): + open_dataset(root, schema=schema_path, require_converters=False) From 964156f377d6a7aa86c22097fb188311d69de4b5 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Mon, 17 Aug 2026 15:08:06 +0100 Subject: [PATCH 4/5] Render every sub-array column for an empty buffer The number of columns a sub-array field renders is taken from the dtype rather than inferred from the data, so a buffer carrying no frames now returns an empty DataFrame with the full column set. Previously parse_to_dataframe would fail on any register with a sub-array field when given no frames, since numpy cannot infer a dimension from a zero-length array. --- .../harp-protocol/src/harp/protocol/_payload.py | 3 ++- tests/protocol/test_register.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/packages/harp-protocol/src/harp/protocol/_payload.py b/src/packages/harp-protocol/src/harp/protocol/_payload.py index 1f92e66..1f14871 100644 --- a/src/packages/harp-protocol/src/harp/protocol/_payload.py +++ b/src/packages/harp-protocol/src/harp/protocol/_payload.py @@ -176,7 +176,8 @@ def _columns( return [Column(name, sub)] # sub-array -> one column per element; index is intrinsic identity, so a # nameless (root) array is positional, a named field is prefixed. - flat = sub.reshape(len(arr), -1) + width = int(np.prod(sub.shape[1:])) + flat = sub.reshape(len(arr), width) label = (lambda i: str(i)) if name is None else (lambda i: f"{name}_{i}") return [Column(label(i), flat[:, i]) for i in range(flat.shape[1])] diff --git a/tests/protocol/test_register.py b/tests/protocol/test_register.py index 5bf2a16..f8b823a 100644 --- a/tests/protocol/test_register.py +++ b/tests/protocol/test_register.py @@ -205,6 +205,18 @@ def test_format_with_payload_instance_via_register(): assert parsed == 42 +def test_empty_buffer_keeps_columns(): + # A sub-array renders one column per element, and the count comes from the dtype, + # so a buffer carrying no frames still renders all of them. + reg = RegisterU32Array(0x28, length=3) + records = np.arange(6, dtype=np.uint32).reshape(2, 3) + populated = parse_to_dataframe(reg, bytes(reg.format_bulk(records)), timestamp=False) + empty = parse_to_dataframe(reg, b"", timestamp=False) + assert list(empty.columns) == list(populated.columns) + assert empty.dtypes.equals(populated.dtypes) + assert len(empty) == 0 + + def test_structured_register_format_single_sample(): sample = np.array([(100, 512, -200)], dtype=AnalogDataPayload.payload_dtype) frame = AnalogData.format(sample) From 6495819b22a98918245f1392d221e2f87d74407e Mon Sep 17 00:00:00 2001 From: glopesdev Date: Mon, 17 Aug 2026 19:28:07 +0100 Subject: [PATCH 5/5] Read an unlogged register as an empty frame A register declared in the device register map with no data in the folder now reads as an empty DataFrame carrying the same columns, rather than raising FileNotFoundError. The schema describes the structure of the data regardless of whether anything was recorded, so contents is what distinguishes a register that was never logged from one the device does not declare. read now takes timestamp as a bool defaulting to True, matching parse_to_dataframe, in place of a tri-state that inferred it from the payload-type bit of the first frame. --- docs/examples/read_dataset/read_dataset.md | 2 +- docs/examples/read_dataset/read_dataset.py | 5 +- src/packages/harp-data/README.md | 4 +- .../harp-data/src/harp/data/_dataset.py | 32 ++--- tests/data/test_dataset.py | 124 ++++++++++++------ 5 files changed, 103 insertions(+), 64 deletions(-) diff --git a/docs/examples/read_dataset/read_dataset.md b/docs/examples/read_dataset/read_dataset.md index 7764092..8c15801 100644 --- a/docs/examples/read_dataset/read_dataset.md +++ b/docs/examples/read_dataset/read_dataset.md @@ -4,7 +4,7 @@ A Harp acquisition is usually saved as a **de-multiplexed dataset folder**: one This is the recommended entry point for a recorded session on disk. To decode a single loose `.bin` file instead, see [Reading Data into a DataFrame](../read_data_to_dataframe/read_data_to_dataframe.md). -The quickest way in is `open_dataset(folder)`. It finds the `device.yml` inside the folder, builds the device module, and returns a reader ready to go. Given a device module already in hand, for example from a pre-generated package, pass it as the second argument, `open_dataset(folder, module)`. A register is then read by class, by name, or by address. Timestamps are detected automatically and placed on the `"Time"` index, as float seconds or an absolute `DatetimeIndex` when an `epoch` is passed. +The quickest way in is `open_dataset(folder)`. It finds the `device.yml` inside the folder, builds the device module, and returns a reader ready to go. Given a device module already in hand, for example from a pre-generated package, pass it as the second argument, `open_dataset(folder, module)`. A register is then read by class, by name, or by address. The Harp time becomes the `"Time"` index, as float seconds or an absolute `DatetimeIndex` when an `epoch` is passed. ```python diff --git a/docs/examples/read_dataset/read_dataset.py b/docs/examples/read_dataset/read_dataset.py index 6b48c06..a44f2a4 100644 --- a/docs/examples/read_dataset/read_dataset.py +++ b/docs/examples/read_dataset/read_dataset.py @@ -24,9 +24,8 @@ # map rather than the module namespace, so common registers are reachable too. df = reader.read("OperationControl") -# Or by address. Timestamps are auto-detected from the frames, and when present -# they become the DataFrame index, named "Time", holding float seconds from -# device start. +# Or by address. The Harp time becomes the DataFrame index, named "Time", +# holding float seconds from device start. df = reader.read(44) print(df.head()) diff --git a/src/packages/harp-data/README.md b/src/packages/harp-data/README.md index a2d803a..442f347 100644 --- a/src/packages/harp-data/README.md +++ b/src/packages/harp-data/README.md @@ -39,6 +39,8 @@ frames = {name: reader.read(name) for name in reader.contents} A name is resolved through the device register map rather than the module namespace, so the common registers are reachable by name too. +A register declared in the device register map with no data present in the folder reads as an empty DataFrame carrying the same columns, since the schema describes the structure of the data regardless of whether anything was recorded. `contents` is what tells the two cases apart. A register the device does not declare at all raises `KeyError`. + Given a device module already in hand, either a pre-generated package or one built with `create_device_module`, pass it as the second argument: ```python @@ -51,7 +53,7 @@ df = reader.read(behavior.AnalogData) # by register class Prefer the register class where a generated package supplies one, since it is the only form that type-checks and a misspelling is caught before the folder is read. A module built by `create_device_module` resolves its registers as `Any`, so there the class verifies no more than the name does. -Timestamps are auto-detected per register and placed on the DataFrame index named `"Time"`: float seconds by default, or an absolute `DatetimeIndex` when `epoch=REFERENCE_EPOCH` is passed. Multi-chunk registers logged as `_
_.bin` are concatenated in filename order; pass a `resolver` to support an alternative on-disk layout. `paths` reports what the resolver found, keyed by address, which is where a custom layout or a chunked register can be checked. +The Harp time becomes the DataFrame index named `"Time"`, as float seconds by default or an absolute `DatetimeIndex` when `epoch=REFERENCE_EPOCH` is passed. Data carrying no timestamp raise unless `timestamp=False` is passed. Multi-chunk registers logged as `_
_.bin` are concatenated in filename order; pass a `resolver` to support an alternative on-disk layout. `paths` reports what the resolver found, keyed by address, which is where a custom layout or a chunked register can be checked. The `` prefix comes from the `DEVICE_NAME` declared by the device module. Pass `name=` to override it, or to supply one when the module declares an empty name. diff --git a/src/packages/harp-data/src/harp/data/_dataset.py b/src/packages/harp-data/src/harp/data/_dataset.py index 71c8841..356cd92 100644 --- a/src/packages/harp-data/src/harp/data/_dataset.py +++ b/src/packages/harp-data/src/harp/data/_dataset.py @@ -13,7 +13,6 @@ parse_device_schema, ) from harp.protocol import RegisterBase -from harp.protocol._constants import _TIMESTAMP_FLAG from ._reader import parse_to_dataframe @@ -159,7 +158,7 @@ def read( register: RegisterKey, *, suffix: str | None = None, - timestamp: bool | None = None, + timestamp: bool = True, epoch: datetime | None = None, message_type: bool = False, decode_enums: bool = True, @@ -174,21 +173,24 @@ def read( :func:`~harp.device.schema.create_device_module` resolves its registers as ``Any``, so there the generated module verifies no more than the name does. - ``suffix`` selects a single ``_
_.bin`` chunk (default: - concatenate every chunk for the address). ``timestamp`` defaults to ``None``, - auto-detecting from the payload-type bit of the frame; pass ``True``/``False`` - to force. ``epoch`` makes the ``"Time"`` index absolute (e.g. - :data:`~harp.data.REFERENCE_EPOCH`). The remaining options match + A register declared in the device register map with no data present in the folder + reads as an empty DataFrame carrying the same columns, since the schema describes + the structure of the data regardless of whether anything was recorded. + :attr:`contents` is what tells the two cases apart. A register the device does not + declare at all raises ``KeyError``. + + ``suffix`` selects a single ``_
_.bin`` chunk, and naming + one that is absent raises ``FileNotFoundError`` (default: concatenate every + chunk for the address). The remaining options match :func:`~harp.data.parse_to_dataframe`. """ cls, address = self._resolve(register) paths = self._resolve_paths(address, suffix) raw = b"".join(p.read_bytes() for p in paths) - ts = self._first_frame_timestamped(raw) if timestamp is None else timestamp return parse_to_dataframe( cls, raw, - timestamp=ts, + timestamp=timestamp, epoch=epoch, message_type=message_type, decode_enums=decode_enums, @@ -210,12 +212,7 @@ def _resolve(self, register: RegisterKey) -> tuple[type[RegisterBase[Any]], int] return cls, register def _resolve_paths(self, address: int, suffix: str | None) -> list[Path]: - paths = self._paths.get(address) - if not paths: - raise FileNotFoundError( - f"No data file for register address {address} under {self._root} " - f"(expected '{self.name}_{address}[_].bin')." - ) + paths = self._paths.get(address) or [] if suffix is not None: paths = [p for p in paths if p.stem.endswith(f"_{suffix}")] if not paths: @@ -224,11 +221,6 @@ def _resolve_paths(self, address: int, suffix: str | None) -> list[Path]: ) return paths - @staticmethod - def _first_frame_timestamped(raw: bytes) -> bool: - """Whether the first frame carries a timestamp (payload-type bit ``0x10``).""" - return len(raw) > 4 and bool(raw[4] & _TIMESTAMP_FLAG) - @overload def open_dataset( diff --git a/tests/data/test_dataset.py b/tests/data/test_dataset.py index d5f0d1c..a33c471 100644 --- a/tests/data/test_dataset.py +++ b/tests/data/test_dataset.py @@ -13,6 +13,10 @@ from harp.device.schema import create_device_module +def _timestamps(n): + return np.arange(n, dtype=np.float64) + + def _records(cls, n, seed): dtype = cls.payload_class.payload_dtype rng = np.random.default_rng(seed) @@ -29,27 +33,26 @@ def emitted_module(device_yml): @pytest.fixture def dataset(emitted_module, tmp_path): - """A dataset folder with three app registers; the first is timestamped.""" + """A dataset folder with three app registers.""" mod = emitted_module name = mod.DEVICE_NAME addresses = [a for a in sorted(mod.REGISTER_MAP) if a >= 32][:3] specs = {} - for i, address in enumerate(addresses): + for address in addresses: cls = mod.REGISTER_MAP[address] records = _records(cls, 5, seed=address) - timestamped = i == 0 - timestamps = np.arange(5, dtype=np.float64) if timestamped else None + timestamps = _timestamps(5) buf = bytes(cls.format_bulk(records, timestamps=timestamps)) (tmp_path / f"{name}_{address}.bin").write_bytes(buf) - specs[address] = (cls, timestamped, buf) + specs[address] = (cls, buf) return mod, name, tmp_path, specs def test_read_by_class_and_by_address(dataset): mod, _name, root, specs = dataset reader = DatasetReader(mod, root) - for address, (cls, timestamped, buf) in specs.items(): - expected = parse_to_dataframe(cls, buf, timestamp=timestamped) + for address, (cls, buf) in specs.items(): + expected = parse_to_dataframe(cls, buf) assert reader.read(cls).equals(expected) assert reader.read(address).equals(expected) @@ -57,7 +60,7 @@ def test_read_by_class_and_by_address(dataset): def test_read_by_class_from_module_namespace(dataset): mod, _name, root, specs = dataset reader = DatasetReader(mod, root) - for address, (cls, _timestamped, _buf) in specs.items(): + for address, (cls, _buf) in specs.items(): # The register reached by name off the module is the one at that address. assert reader.read(getattr(mod, cls.__name__)).equals(reader.read(address)) @@ -65,7 +68,7 @@ def test_read_by_class_from_module_namespace(dataset): def test_read_by_register_name(dataset): mod, _name, root, specs = dataset reader = DatasetReader(mod, root) - for address, (cls, _timestamped, _buf) in specs.items(): + for address, (cls, _buf) in specs.items(): assert reader.read(cls.__name__).equals(reader.read(address)) @@ -84,7 +87,7 @@ def test_reads_common_registers_not_named_by_module(emitted_module, tmp_path): for cls in (WhoAmI, TimestampSeconds): records = _records(cls, 4, seed=cls.address) - buf = bytes(cls.format_bulk(records)) + buf = bytes(cls.format_bulk(records, timestamps=_timestamps(4))) (tmp_path / f"{mod.DEVICE_NAME}_{cls.address}.bin").write_bytes(buf) reader = DatasetReader(mod, tmp_path) @@ -96,19 +99,43 @@ def test_reads_common_registers_not_named_by_module(emitted_module, tmp_path): assert set(reader.contents) == {"WhoAmI", "TimestampSeconds"} -def test_timestamp_is_auto_detected(dataset): +def test_read_gives_time_index(dataset): + # Every register reads with the same index, so frames aggregate across sessions. mod, _name, root, specs = dataset reader = DatasetReader(mod, root) - for address, (_cls, timestamped, _buf) in specs.items(): - df = reader.read(address) - # Timestamped frames get a "Time" index; untimestamped keep a plain RangeIndex. - assert (df.index.name == "Time") is timestamped + for address in specs: + assert reader.read(address).index.name == "Time" + + +def test_timestamp_false_gives_range_index(dataset): + # The diagnostic escape hatch, and the only way to read frames carrying no timestamp. + mod, _name, root, specs = dataset + reader = DatasetReader(mod, root) + df = reader.read(next(iter(specs)), timestamp=False) + assert df.index.name is None + + +def test_untimestamped_frames_raise_value_error(emitted_module, tmp_path): + # A recording every device should be incapable of producing, so it surfaces rather + # than reading back with a silently different index. + mod = emitted_module + address = next(a for a in sorted(mod.REGISTER_MAP) if a >= 32) + cls = mod.REGISTER_MAP[address] + (tmp_path / f"{mod.DEVICE_NAME}_{address}.bin").write_bytes( + bytes(cls.format_bulk(_records(cls, 3, seed=1))) + ) + + reader = DatasetReader(mod, tmp_path) + + with pytest.raises(ValueError, match="no timestamp data"): + reader.read(address) + assert len(reader.read(address, timestamp=False)) == 3 def test_time_index_is_float_seconds_without_epoch(dataset): mod, _name, root, specs = dataset reader = DatasetReader(mod, root) - address = next(a for a, (_c, ts, _b) in specs.items() if ts) # the timestamped register + address = next(iter(specs)) df = reader.read(address) assert df.index.name == "Time" assert list(df.index) == [0.0, 1.0, 2.0, 3.0, 4.0] # arange(5) seconds from the fixture @@ -117,7 +144,7 @@ def test_time_index_is_float_seconds_without_epoch(dataset): def test_epoch_gives_absolute_datetime_index(dataset): mod, _name, root, specs = dataset reader = DatasetReader(mod, root) - address = next(a for a, (_c, ts, _b) in specs.items() if ts) + address = next(iter(specs)) df = reader.read(address, epoch=REFERENCE_EPOCH) assert isinstance(df.index, pd.DatetimeIndex) assert df.index.name == "Time" @@ -134,17 +161,21 @@ def test_suffix_chunks_are_concatenated(emitted_module, tmp_path): address = next(a for a in sorted(mod.REGISTER_MAP) if a >= 32) cls = mod.REGISTER_MAP[address] chunks = { - "20260816T090000Z": bytes(cls.format_bulk(_records(cls, 3, seed=1))), - "20260816T100000Z": bytes(cls.format_bulk(_records(cls, 2, seed=2))), + "20260816T090000Z": bytes( + cls.format_bulk(_records(cls, 3, seed=1), timestamps=_timestamps(3)) + ), + "20260816T100000Z": bytes( + cls.format_bulk(_records(cls, 2, seed=2), timestamps=_timestamps(2)) + ), } for suffix in reversed(list(chunks)): (tmp_path / f"{name}_{address}_{suffix}.bin").write_bytes(chunks[suffix]) reader = DatasetReader(mod, tmp_path) - combined = parse_to_dataframe(cls, b"".join(chunks.values()), timestamp=False) - assert reader.read(cls).reset_index(drop=True).equals(combined) + combined = parse_to_dataframe(cls, b"".join(chunks.values())) + assert reader.read(cls).reset_index(drop=True).equals(combined.reset_index(drop=True)) # A specific chunk can still be selected by suffix. - earliest = parse_to_dataframe(cls, chunks["20260816T090000Z"], timestamp=False) + earliest = parse_to_dataframe(cls, chunks["20260816T090000Z"]) assert reader.read(cls, suffix="20260816T090000Z").equals(earliest) @@ -188,12 +219,30 @@ def test_nameless_module_raises_on_construction(dataset): assert DatasetReader(nameless, root, name=name).name == name -def test_missing_register_file_raises(dataset): +def test_unlogged_register_reads_empty(dataset): + # The schema describes the columns whether or not anything was recorded, so a + # declared register with no file is zero rows rather than a failure. WhoAmI sits + # at address 0, is in the map, and has no file in this dataset. mod, _name, root, _specs = dataset reader = DatasetReader(mod, root) - # WhoAmI (address 0) is in the map but has no file in this dataset. - with pytest.raises(FileNotFoundError): - reader.read(0) + + df = reader.read(0) + buf = bytes(WhoAmI.format_bulk(_records(WhoAmI, 2, seed=0), timestamps=_timestamps(2))) + populated = parse_to_dataframe(WhoAmI, buf) + + assert len(df) == 0 + assert "WhoAmI" not in reader.contents + assert list(df.columns) == list(populated.columns) + assert df.dtypes.equals(populated.dtypes) + assert df.index.name == "Time" + + +def test_unknown_suffix_raises_file_not_found(dataset): + # Naming a chunk that is absent is a mistake about the request, so it still raises. + mod, _name, root, specs = dataset + reader = DatasetReader(mod, root) + with pytest.raises(FileNotFoundError, match="chunk"): + reader.read(next(iter(specs)), suffix="20991231T000000Z") def test_unknown_address_raises(dataset): @@ -209,9 +258,9 @@ def test_custom_file_resolver_supports_alternative_layout(emitted_module, tmp_pa expected = {} for address in addresses: cls = mod.REGISTER_MAP[address] - buf = bytes(cls.format_bulk(_records(cls, 3, seed=address))) + buf = bytes(cls.format_bulk(_records(cls, 3, seed=address), timestamps=_timestamps(3))) (tmp_path / f"reg{address}.bin").write_bytes(buf) # not the Harp layout - expected[cls.__name__] = parse_to_dataframe(cls, buf, timestamp=False) + expected[cls.__name__] = parse_to_dataframe(cls, buf) def resolver(root, _name): found = {} @@ -237,7 +286,7 @@ def test_contents_maps_names_to_addresses(dataset): mod, _name, root, specs = dataset reader = DatasetReader(mod, root) # The declared address space is much larger; contents is what this folder holds. - assert reader.contents == {cls.__name__: address for address, (cls, _ts, _buf) in specs.items()} + assert reader.contents == {cls.__name__: address for address, (cls, _buf) in specs.items()} assert len(mod.REGISTER_MAP) > len(reader.contents) @@ -260,9 +309,9 @@ def test_contents_keys_read_every_register(dataset): frames = {name: reader.read(name) for name in reader.contents} - assert set(frames) == {cls.__name__ for cls, _ts, _buf in specs.values()} - for cls, timestamped, buf in specs.values(): - assert frames[cls.__name__].equals(parse_to_dataframe(cls, buf, timestamp=timestamped)) + assert set(frames) == {cls.__name__ for cls, _buf in specs.values()} + for cls, buf in specs.values(): + assert frames[cls.__name__].equals(parse_to_dataframe(cls, buf)) def test_contents_excludes_undescribed_address(dataset): @@ -276,7 +325,7 @@ def test_contents_excludes_undescribed_address(dataset): assert undescribed in reader.paths assert undescribed not in reader.contents.values() - assert set(reader.contents) == {cls.__name__ for cls, _ts, _buf in specs.values()} + assert set(reader.contents) == {cls.__name__ for cls, _buf in specs.values()} def test_every_register_round_trips(emitted_module, tmp_path): @@ -286,12 +335,9 @@ def test_every_register_round_trips(emitted_module, tmp_path): expected = {} for address, cls in mod.REGISTER_MAP.items(): records = _records(cls, 4, seed=address) - # Alternate timestamped/untimestamped to exercise both parse paths. - timestamped = address % 2 == 0 - timestamps = np.arange(4, dtype=np.float64) if timestamped else None - buf = bytes(cls.format_bulk(records, timestamps=timestamps)) + buf = bytes(cls.format_bulk(records, timestamps=_timestamps(4))) (tmp_path / f"{name}_{address}.bin").write_bytes(buf) - expected[cls.__name__] = parse_to_dataframe(cls, buf, timestamp=timestamped) + expected[cls.__name__] = parse_to_dataframe(cls, buf) reader = DatasetReader(mod, tmp_path) @@ -319,7 +365,7 @@ def test_open_dataset_builds_module_from_device_yml(dataset, device_yml): assert isinstance(reader, DatasetReader) # Reads match a reader built from an explicitly-generated module. reference = DatasetReader(mod, root) - for address, (cls, _timestamped, _buf) in specs.items(): + for address, (cls, _buf) in specs.items(): assert reader.read(address).equals(reference.read(cls))