diff --git a/.github/workflows/needs_release_notes.yml b/.github/workflows/needs_release_notes.yml index fa555d1478..e001e8cd43 100644 --- a/.github/workflows/needs_release_notes.yml +++ b/.github/workflows/needs_release_notes.yml @@ -21,7 +21,7 @@ jobs: pull-requests: write # Required to add labels to PRs runs-on: ubuntu-latest steps: - - uses: actions/labeler@b8dd2d9be0f68b860e7dae5dae7d772984eacd6d # v6.2.0 + - uses: actions/labeler@bf12e9b00b37c5c0ca2b87b79b2daf7891dbda13 # v7.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} sync-labels: true diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 4d460f4a56..fe0d09f300 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -81,7 +81,7 @@ jobs: name: releases path: dist - name: Generate artifact attestation - uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0 with: subject-path: dist/* - name: Publish package to PyPI diff --git a/.github/workflows/zarr-metadata-release.yml b/.github/workflows/zarr-metadata-release.yml index 5021f79d2e..bc9ecf9871 100644 --- a/.github/workflows/zarr-metadata-release.yml +++ b/.github/workflows/zarr-metadata-release.yml @@ -82,7 +82,7 @@ jobs: path: dist - name: Generate artifact attestation - uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0 with: subject-path: dist/* @@ -107,7 +107,7 @@ jobs: path: dist - name: Generate artifact attestation - uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0 with: subject-path: dist/* diff --git a/.github/workflows/zarr-metadata.yml b/.github/workflows/zarr-metadata.yml index 4e5bb0fb1a..df7d96cc1c 100644 --- a/.github/workflows/zarr-metadata.yml +++ b/.github/workflows/zarr-metadata.yml @@ -82,7 +82,10 @@ jobs: - name: Sync test dependency group run: uv sync --group test --python 3.11 - name: Run pyright - run: uv run --group test --with pyright pyright src + # Pinned to the last version that types PEP 661 sentinels in class + # attributes correctly; 1.1.405+ regressed (microsoft/pyright#11115). + # Unpin when the fix lands. + run: uv run --group test --with 'pyright==1.1.404' pyright src zarr-metadata-complete: name: zarr-metadata complete diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 6250426bae..1567bea713 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -32,4 +32,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7 + uses: zizmorcore/zizmor-action@6599ee8b7a49aef6a770f63d261d214911a7ce02 # v0.6.0 diff --git a/changes/3352.bugfix.md b/changes/3352.bugfix.md new file mode 100644 index 0000000000..7461486776 --- /dev/null +++ b/changes/3352.bugfix.md @@ -0,0 +1,3 @@ +Fix `zarr.api.asynchronous.open_like` so it can create a new array by default when the +target path does not already exist. It now defaults to `mode="a"`; when using a read-only +store to open an existing array, pass `mode="r"` explicitly. diff --git a/changes/4128.feature.md b/changes/4128.feature.md new file mode 100644 index 0000000000..c62a615ac2 --- /dev/null +++ b/changes/4128.feature.md @@ -0,0 +1 @@ +Added `Group.get_array`, `Group.get_group`, `AsyncGroup.get_array`, and `AsyncGroup.get_group`: type-safe accessors that return the child array or group at a given path, raising `ArrayNotFoundError` / `GroupNotFoundError` if no node exists there, and `ContainsGroupError` / `ContainsArrayError` if the node is not of the requested kind. Unlike `Group.__getitem__`, which returns `Array | Group`, these methods have precise return types. Nested paths like `"subgroup/subarray"` are supported. diff --git a/changes/4187.feature.md b/changes/4187.feature.md new file mode 100644 index 0000000000..87133e2034 --- /dev/null +++ b/changes/4187.feature.md @@ -0,0 +1,4 @@ +`ZipStore` now accepts an open binary file-like object in place of a path, enabling +zip archives on remote storage (e.g. a file opened with `fsspec` or an +`obstore.ReadableFile`). Operations that require a filesystem location +(`clear`, `move`) raise `NotImplementedError` for file-object-backed stores. diff --git a/changes/4191.doc.md b/changes/4191.doc.md new file mode 100644 index 0000000000..fb01f9141f --- /dev/null +++ b/changes/4191.doc.md @@ -0,0 +1,14 @@ +Added a blog section to the documentation, with a post covering two performance +highlights of the 3.3.0 release: the opt-in `FusedCodecPipeline` and byte-range +coalescing for partial reads of sharded arrays. + +Added two runnable examples that accompany the post: +`examples/codec_pipeline_performance` compares the `BatchedCodecPipeline` and +`FusedCodecPipeline` on a sharded array across two stores and two codec +regimes, showing when the fused pipeline's thread pool helps and when it does +not, and `examples/sharding_coalescing` demonstrates how read coalescing +reduces the number of store requests when reading subregions of a sharded +array. + +Also removed the hardware-specific speedup figures from the `FusedCodecPipeline` +release note, since they depend on the array layout, codec, and machine. diff --git a/docs/blog/.authors.yml b/docs/blog/.authors.yml new file mode 100644 index 0000000000..10ce423cfc --- /dev/null +++ b/docs/blog/.authors.yml @@ -0,0 +1,6 @@ +authors: + d-v-b: + name: Davis Bennett + description: Core developer + avatar: https://github.com/d-v-b.png + url: https://github.com/d-v-b diff --git a/docs/blog/index.md b/docs/blog/index.md new file mode 100644 index 0000000000..fca29e2578 --- /dev/null +++ b/docs/blog/index.md @@ -0,0 +1,3 @@ +# Blog + +News, release highlights, and design notes from the Zarr-Python developers. diff --git a/docs/blog/posts/3.3.0-release.md b/docs/blog/posts/3.3.0-release.md new file mode 100644 index 0000000000..74042e0078 --- /dev/null +++ b/docs/blog/posts/3.3.0-release.md @@ -0,0 +1,169 @@ +--- +date: 2026-07-15 +authors: + - d-v-b +categories: + - Release +--- + +# Zarr-Python 3.3.0 + +We're happy to announce the release of version 3.3.0 of Zarr-Python. It's been a while since our last release ([3.2.1](https://github.com/zarr-developers/zarr-python/releases/tag/v3.2.1) dropped in May of this year), +and we're bringing some exciting additions to the latest version. For the full release notes, see the [3.3.0 release notes](../../release-notes.md), otherwise stick around for an overview of two performance-centric highlights of this release. + + + +## Faster low-latency storage + +Relevant issues and pull requests: + +- [#3524](https://github.com/zarr-developers/zarr-python/issues/3524) -- the performance report that started this work +- [#3885](https://github.com/zarr-developers/zarr-python/pull/3885) -- synchronous codec APIs and the `FusedCodecPipeline` + +### The cost of async overhead + +Zarr-Python 3.x uses async routines for fetching data and decoding chunks. In terms of code, this means our store (data fetching) and codec (chunk decoding) APIs are both async. This makes +I/O against high-latency storage backends like cloud object storage efficient. But for *low-latency* storage, like in-process memory or the file system, async routines add measurable overhead and offer no benefit. Async only adds value when there's work to be done while waiting for I/O to complete, but when I/O latency is low, it completes too quickly to run anything while waiting, and we are left paying the performance bill for obligatory async task scheduling that offered no value. + +This performance problem became acute when Zarr-Python users reported that in-memory array indexing workloads ran *slower* in Zarr-Python 3.1.3 relative to Zarr-Python 2.18.7 ([#3524](https://github.com/zarr-developers/zarr-python/issues/3524)). Fortunately this performance regression had a straightforward fix (I don't say "easy" because it was a lot of work). + +### Synchronous execution restores performance + +If async overhead makes low-latency storage slow, does *removing* that overhead restore performance? Yes, it does! + +In [#3885](https://github.com/zarr-developers/zarr-python/pull/3885) we defined synchronous versions of our storage and codec APIs -- the `SyncByteGetter` and `SyncByteSetter` protocols, plus a `get_ranges_sync` method on the `Store` ABC -- and then combined them in a new codec orchestration class called `FusedCodecPipeline`. The `FusedCodecPipeline` is an opt-in alternative to the default (the `BatchedCodecPipeline`) that gives large speedups for low-latency storage. It is currently marked [experimental](../../user-guide/experimental.md), so we may change it as we learn more; the default pipeline is untouched, and existing code keeps working unless you opt in. + +The win here is *not* a faster compressor. It is the removal of async scheduling overhead (including some [nasty `asyncio.to_thread` overhead](https://github.com/python/cpython/issues/136084)), plus a few vectorized fast paths for dense, uncompressed shards. And we only expect this new pipeline to accelerate workloads targeting a subset of storage backends, namely any store with methods that advertise low latency. + +On this author's 10-core Apple M4 laptop, the `FusedCodecPipeline` delivers the following results against memory-backed arrays: + +- uncompressed writes are *~4 times faster* +- uncompressed reads are *~5 times faster* +- compressed writes are *~2 times faster* +- compressed reads are *~2 times faster* + +These numbers came from a [runnable example](../../user-guide/examples/codec_pipeline_performance.md) that ships with the documentation. Run it yourself to get a sense of how the `FusedCodecPipeline` behaves on your system -- when and how you use it depends on your hardware, your array layout, and how your chunks are compressed. What's certain is that for in-memory arrays, and arrays saved to the local file system, the `FusedCodecPipeline` is worth a try. + +Getting good numbers requires choosing the right level of thread-based parallelism for your workload, which is part of the configuration of the `FusedCodecPipeline`. For uncompressed chunks there's no CPU-bound work to do after fetching a chunk and so +thread-based parallelism is worse than useless and slows things down. But for compressed chunks, threading offers a substantial payoff. + +### How to use it + +Select the pipeline through the [runtime configuration](../../user-guide/config.md) by setting `codec_pipeline.path`. Set it globally to affect every array created or opened afterwards: + +```python exec="true" session="blog-330" source="above" +import zarr + +zarr.config.set( + {"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"} +) +``` + +Or scope it to a block of code by using `zarr.config.set` as a context manager, which is the safer choice if you only want the new pipeline for part of your program: + +```python exec="true" session="blog-330" source="above" result="ansi" +import numpy as np +import zarr +from zarr.storage import MemoryStore + +with zarr.config.set( + {"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"} +): + arr = zarr.create_array( + store=MemoryStore(), + shape=(1000, 1000), + chunks=(100, 100), + shards=(1000, 1000), + dtype="float32", + ) + arr[:] = np.random.random((1000, 1000)).astype("float32") + result = arr[:] + +print(result.shape) +``` + +Thread-based parallelism is configured separately, via `codec_pipeline.max_workers`. It defaults to `None`, meaning a pool sized to `os.cpu_count()`. Note that this setting is read *only* by the `FusedCodecPipeline` -- the default `BatchedCodecPipeline` ignores it, so tuning it without opting in above does nothing. + +As noted, memory-backed and uncompressed workloads often do better with a single worker, which runs everything inline on the calling thread: + +```python exec="true" session="blog-330" source="above" +import zarr + +# No thread pool: run codec compute inline. Often best for uncompressed, +# memory-backed arrays, where there's no CPU-bound work to overlap. +zarr.config.set({"codec_pipeline.max_workers": 1}) + +# A fixed-size thread pool, which pays off once compression is in play. +zarr.config.set({"codec_pipeline.max_workers": 8}) + +# Or back to the default, sized to the number of CPUs. +zarr.config.set({"codec_pipeline.max_workers": None}) +``` + +To return to the default pipeline, set `codec_pipeline.path` back to the batched implementation: + +```python exec="true" session="blog-330" source="above" +import zarr + +zarr.config.set( + {"codec_pipeline.path": "zarr.core.codec_pipeline.BatchedCodecPipeline"} +) +``` + +## Faster sharded reads + +Relevant issues and pull requests: + +- [#3004](https://github.com/zarr-developers/zarr-python/pull/3004) -- optimize partial shard reads +- [#3925](https://github.com/zarr-developers/zarr-python/pull/3925) -- `Store.get_ranges` for concurrent, coalesced multi-range reads +- [#3987](https://github.com/zarr-developers/zarr-python/pull/3987) -- control coalescing through `ArrayConfig` and the runtime config + +### How sharding works + +Chunks encoded with the `sharding_indexed` codec contain a secondary level of chunking, called subchunks. For example, if the `chunk_grid` field of the array metadata declares an "outer chunk" size of, say `(10, 10)`, a `sharding_indexed` codec in the `codecs` field could declare an "inner chunk" size of `(5, 5)`. Readers accessing such a chunk will observe a stored object (a stream of bytes) that decodes to an array with size `(10, 10)` (the "outer chunk"), which is comprised of four separate, contiguous byte ranges that each decode to a `(5, 5)` inner chunk. Each inner chunk occupies its own byte range in the outer chunk. + +A reader can satisfy a request for all four inner chunks by issuing four separate byte-range requests, or by making a *single* request for a byte range that spans all four inner chunks. The latter option is nice because it cuts down on the number of requests we need. Historically Zarr-Python used this optimization when reading entire outer chunks; in 3.3.0, we use this optimization in more cases, resulting in more efficient I/O patterns for sharded reads. + +### Interval equivalence + +Byte ranges, being intervals, obey some combination rules: the values in two half-open intervals `[a, b), [b, c)` can be captured by the single interval `[a, c)`. That means a reader can get multiple inner chunks with *one* byte-range request by requesting a range of bytes starting with the first byte of the first subchunk and ending with the last byte of the last subchunk. When individual requests are expensive, this kind of optimization is worth a lot. + +The requested inner chunks are not necessarily contiguous -- there might be a byte range gap between them. As long as that gap is not too big, its often efficient to fetch the entire byte range, gap included, and pick out the inner chunk byte ranges after I/O is done. + +### Byte range coalescing + +We call this procedure -- merging adjacent byte ranges -- "byte range coalescing", and it's a new performance optimization shipping in Zarr-Python 3.3.0. Unlike the `FusedCodecPipeline`, this one is on by default with base settings we think are good, so most users won't need to tune anything. + +Two knobs control it, both documented in the [runtime configuration guide](../../user-guide/config.md). Nearby byte ranges in the same shard are merged into a single request when the gap between them is no larger than `array.sharding_coalesce_max_gap_bytes` (default 1 MiB) and the merged read stays within `array.sharding_coalesce_max_bytes` (default 16 MiB). The gap threshold is what trades wasted bytes against saved requests: raising it reads more data you didn't ask for, in exchange for fewer requests. + +For a runnable demonstration -- counting the store requests saved and timing them against a store with simulated latency -- see the [sharded read coalescing example](../../user-guide/examples/sharding_coalescing.md). + +You can set them globally, or per array by passing `config={...}` to [`zarr.create_array`][]: + +```python exec="true" session="blog-330" source="above" result="ansi" +import zarr +from zarr.storage import MemoryStore + +arr = zarr.create_array( + store=MemoryStore(), + shape=(1000, 1000), + chunks=(100, 100), + shards=(1000, 1000), + dtype="float32", + config={ + "sharding_coalesce_max_gap_bytes": 4 * 1024**2, # 4 MiB + "sharding_coalesce_max_bytes": 64 * 1024**2, # 64 MiB + }, +) +print(arr.shape) +``` + +## Tell us what you think + +We hope these new features are helpful, and we would appreciate any feedback that helps us improve them, or any other aspect of Zarr-Python. + +## Going faster + +The updates in this release are just the first step of a larger performance-oriented direction for Zarr-Python. Landing these two enhancements taught us a *lot* about the performance-sensitive areas of the library. We can and will invest more time in performance tuning, e.g. by adding or changing abstractions, writing code for special cases, etc. + +We plan to consider including compiled code that should enable significant performance improvements. The [`zarrs`](https://zarrs.dev/) project is an ecosystem of Zarr tools written in Rust, with [extremely high performance](https://book.zarrs.dev/#-zarrs-is-fast-). Is there a `zarrs` binding in Zarr-Python's future? I hope so! We are keenly observing development of [`zarrista`](https://developmentseed.org/zarrista/latest/) as a proof-of-concept for what a Python-`zarrs` binding layer might look like. Stay tuned! diff --git a/docs/release-notes.md b/docs/release-notes.md index 3fd8a5f360..e2332b21f9 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -14,7 +14,7 @@ concurrently. ([#3004](https://github.com/zarr-developers/zarr-python/issues/3004)) - Added a `subchunk_write_order` option to `ShardingCodec` to control the physical order of subchunks within a shard. Supported values are `morton`, `unordered`, `lexicographic`, and `colexicographic`. `unordered` makes no guarantee about subchunk layout. This setting affects only on-disk layout, not the data read back, and is not persisted in array metadata: it applies per codec instance and is not recovered when reopening a sharded array. ([#3826](https://github.com/zarr-developers/zarr-python/issues/3826)) - Added `SyncByteGetter` and `SyncByteSetter` runtime-checkable protocols and a `get_ranges_sync` method on the `Store` ABC. These let custom byte getters/setters opt into the synchronous codec pipeline's fast path for in-memory IO, which the sharding codec uses for its inner chunks. ([#3885](https://github.com/zarr-developers/zarr-python/issues/3885)) -- Added `FusedCodecPipeline`, an opt-in codec pipeline that runs codec compute synchronously and in bulk (avoiding the per-chunk async scheduling overhead of the default `BatchedCodecPipeline`), giving large speedups for sharded arrays (up to ~24x writes / ~14x reads on many-chunks-per-shard layouts, more with compression) and no regressions on compute-bound workloads. The default `BatchedCodecPipeline` is unchanged for standard configurations, so existing code keeps working unless you opt in; enable the new pipeline with `zarr.config.set({"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"})`. ([#3885](https://github.com/zarr-developers/zarr-python/issues/3885)) +- Added `FusedCodecPipeline`, an opt-in codec pipeline that runs codec compute synchronously and in bulk (avoiding the per-chunk async scheduling overhead of the default `BatchedCodecPipeline`), giving large speedups for sharded arrays. The default `BatchedCodecPipeline` is unchanged for standard configurations, so existing code keeps working unless you opt in; enable the new pipeline with `zarr.config.set({"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"})`. ([#3885](https://github.com/zarr-developers/zarr-python/issues/3885)) - Add `zarr.abc.store.Store.get_ranges` for concurrent, coalesced multi-range reads from a single key. The method is defined on the `Store` ABC with a default implementation built on `Store.get`, so every store inherits a working version; stores with native multi-range backends (e.g. `FsspecStore`) can override for efficiency. Coalescing knobs (`max_concurrency`, `max_gap_bytes`, `max_coalesced_bytes`) are passed as keyword arguments to `get_ranges`. Failures from underlying fetches surface as a `BaseExceptionGroup` (PEP 654); callers should use `except*` to filter for specific exception types such as `FileNotFoundError`. ([#3925](https://github.com/zarr-developers/zarr-python/issues/3925)) - Two new fields on `ArrayConfig` control how the sharding codec coalesces partial-shard reads: `sharding_coalesce_max_gap_bytes` (default 1 MiB) and `sharding_coalesce_max_bytes` (default 16 MiB). When reading multiple chunks from the same shard, nearby byte ranges are merged into a single request to the store if separated by no more than `sharding_coalesce_max_gap_bytes` and the merged read stays within `sharding_coalesce_max_bytes`. Defaults are seeded from the matching `array.sharding_coalesce_max_gap_bytes` / `array.sharding_coalesce_max_bytes` keys in [`zarr.config`][] at array-creation time, and can be overridden per array by passing `config={...}` to [`zarr.create_array`][]. ([#3987](https://github.com/zarr-developers/zarr-python/issues/3987)) diff --git a/docs/user-guide/examples/codec_pipeline_performance.md b/docs/user-guide/examples/codec_pipeline_performance.md new file mode 100644 index 0000000000..f21e31636e --- /dev/null +++ b/docs/user-guide/examples/codec_pipeline_performance.md @@ -0,0 +1,7 @@ +--8<-- "examples/codec_pipeline_performance/README.md" + +## Source Code + +```python exec="false" reason="pymdownx snippet include directive, not python source" +--8<-- "examples/codec_pipeline_performance/codec_pipeline_performance.py" +``` diff --git a/docs/user-guide/examples/sharding_coalescing.md b/docs/user-guide/examples/sharding_coalescing.md new file mode 100644 index 0000000000..8b2e054af5 --- /dev/null +++ b/docs/user-guide/examples/sharding_coalescing.md @@ -0,0 +1,7 @@ +--8<-- "examples/sharding_coalescing/README.md" + +## Source Code + +```python exec="false" reason="pymdownx snippet include directive, not python source" +--8<-- "examples/sharding_coalescing/sharding_coalescing.py" +``` diff --git a/docs/user-guide/groups.md b/docs/user-guide/groups.md index 337ad39554..7429a03847 100644 --- a/docs/user-guide/groups.md +++ b/docs/user-guide/groups.md @@ -51,6 +51,29 @@ print(root['foo/bar']) print(root['foo/bar/baz']) ``` +Accessing a member with `[]` returns either an [`zarr.Array`][] or a [`zarr.Group`][], depending on +what is stored at the given path. When you expect a node of a particular kind, use +[`zarr.Group.get_array`][] or [`zarr.Group.get_group`][] instead. These methods accept the same +paths as `[]`, but they have precise return types and raise an error if no node exists at the +given path, or if the node is not of the expected kind: + +```python exec="true" session="groups" source="above" result="ansi" +print(root.get_group('foo')) +``` + +```python exec="true" session="groups" source="above" result="ansi" +print(root.get_array('foo/bar/baz')) +``` + +```python exec="true" session="groups" source="above" result="ansi" +from zarr.errors import ContainsGroupError + +try: + root.get_array('foo') +except ContainsGroupError as e: + print(e) +``` + The [`zarr.Group.tree`][] method can be used to print a tree representation of the hierarchy, e.g.: diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index 7e0154b2a0..0ba6202c76 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -124,6 +124,19 @@ array = zarr.create_array(store=store, shape=(2,), dtype='float64') print(array) ``` +In place of a path, `ZipStore` also accepts an open binary file object (for +example a file opened with `fsspec`, or an `obstore` reader), enabling zip +archives on remote storage. The file must stay open for as long as the store +is in use: + +```python exec="true" session="storage" source="above" result="ansi" +store.close() +f = open('data.zip', mode='rb') # must stay open while the store is used +array = zarr.open_array(store=zarr.storage.ZipStore(f), mode='r') +print(array[:]) +f.close() +``` + ### Remote Store The [`zarr.storage.FsspecStore`][] stores the contents of a Zarr hierarchy following the same diff --git a/examples/codec_pipeline_performance/README.md b/examples/codec_pipeline_performance/README.md new file mode 100644 index 0000000000..5d85412c29 --- /dev/null +++ b/examples/codec_pipeline_performance/README.md @@ -0,0 +1,59 @@ +# Codec Pipeline Performance + +This example compares the default `BatchedCodecPipeline` against the opt-in +`FusedCodecPipeline` on a sharded array, across two stores (memory and local) +and two codec regimes (uncompressed and gzip), at one worker and at `cpu_count`. + +A *codec pipeline* turns chunks of array data into stored bytes and back, running +the configured codecs and performing the storage IO. The default +`BatchedCodecPipeline` schedules both asynchronously -- roughly one coroutine per +chunk operation. That model pays off for high-latency stores, where there is +useful work to do while waiting on IO. For low-latency stores (in-process memory, +the local filesystem) the IO completes too quickly for the overlap to be worth +its cost, and the async scheduling becomes pure overhead. + +`FusedCodecPipeline` runs codec compute and synchronous IO synchronously, +removing that overhead. It is +[experimental](https://zarr.readthedocs.io/en/stable/user-guide/experimental/) +and opt-in; the default pipeline is unchanged. + +## What it shows + +- How to select a pipeline with `zarr.config.set`, and why the array must be + *created* inside the config block: the pipeline class is resolved at array + construction time and then travels with the array. +- That the benefit depends strongly on layout and on whether compression is in + play. Some configurations are slower under the fused pipeline -- the script + reports speedups below 1.00x rather than hiding them. +- That `codec_pipeline.max_workers` is read only by `FusedCodecPipeline`; the + default pipeline ignores it entirely. + +## Running + +```bash +uv run codec_pipeline_performance.py +``` + +The script has no arguments and writes only to an in-memory store. + +## Interpreting the output + +The numbers are specific to your CPU, your Python build, and the workload chosen +here. They are a measurement of your machine, not a published benchmark -- treat +a single run as indicative and re-measure against your own data and store before +switching pipelines in production. + +Two effects are worth watching for: + +- **The two codec regimes tell opposite stories about `max_workers`.** + Uncompressed IO is dominated by per-chunk *scheduling*, so `Fused (1 worker)` + is already fastest and a thread pool only adds overhead. gzip is genuinely + CPU-bound: a single worker compresses chunk after chunk sequentially and can + be *slower than the default*, while a thread pool spreads that compression + over cores and reclaims the win. That flip is why the fused pipeline is + threaded by default, and why pinning `max_workers=1` is worth it for + memory-backed uncompressed data. +- **Chunk size decides whether threading can help at all.** The 64×64 inner + chunks here are small enough that per-chunk scheduling dominates uncompressed + IO, yet large enough that per-chunk gzip is real work to parallelize. Much + coarser chunks leave the pool with too few items to spread. diff --git a/examples/codec_pipeline_performance/codec_pipeline_performance.py b/examples/codec_pipeline_performance/codec_pipeline_performance.py new file mode 100644 index 0000000000..b821f3e6a7 --- /dev/null +++ b/examples/codec_pipeline_performance/codec_pipeline_performance.py @@ -0,0 +1,214 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "zarr @ git+https://github.com/zarr-developers/zarr-python.git@main", +# "numpy", +# ] +# /// + +""" +Compare the `BatchedCodecPipeline` and the `FusedCodecPipeline`. + +The default `BatchedCodecPipeline` schedules storage IO and codec compute +asynchronously -- roughly one coroutine per chunk operation. For a *sharded* +array that means one coroutine per inner chunk inside every shard. That is the +right model for high-latency stores, where there is useful work to do while +waiting for IO. For low-latency stores (in-process memory, the local +filesystem) the IO completes too quickly for the overlap to pay for itself, and +the scheduling becomes pure overhead. + +The `FusedCodecPipeline` runs codec compute and synchronous IO synchronously, +removing that overhead. Whether it wins, and whether its thread pool helps, +depends on which resource is actually scarce: + + * Uncompressed IO is dominated by per-chunk *scheduling*, not compute. There + is nothing for a thread pool to parallelize, so a single worker is already + fastest and extra workers only add overhead. + * gzip is genuinely CPU-bound. A single worker compresses every chunk + sequentially and can be *slower than the default*, while a thread pool + spreads that compression across cores and reclaims the win. This is when + `max_workers > 1` earns its keep. + +Run it with: + + uv run codec_pipeline_performance.py + +Numbers are hardware-, layout-, and codec-dependent. Treat the output as a +measurement of *your* machine, not as a published benchmark. +""" + +from __future__ import annotations + +import operator +import os +import statistics +import tempfile +import timeit +from pathlib import Path +from typing import TYPE_CHECKING + +import numpy as np + +import zarr +from zarr.storage import LocalStore, MemoryStore + +if TYPE_CHECKING: + from collections.abc import Callable + + from zarr.abc.store import Store + +BATCHED = "zarr.core.codec_pipeline.BatchedCodecPipeline" +FUSED = "zarr.core.codec_pipeline.FusedCodecPipeline" + +# gzip is CPU-bound to encode, which is exactly the regime where the thread +# pool matters. Level 6 is gzip's own default. +GZIP = {"name": "gzip", "configuration": {"level": 6}} + +# 4096x4096 int32 = 64 MiB, split into 16 shards of 1024x1024, each holding +# 16x16 = 256 inner chunks of 64x64 -> 4096 inner chunks in total. The chunks +# are small enough that per-chunk coroutine scheduling dominates uncompressed +# IO, yet large enough that per-chunk gzip is real work to spread over cores. +SHAPE = (4096, 4096) +SHARDS = (1024, 1024) +CHUNKS = (64, 64) +DTYPE = "int32" + +CONFIGS: tuple[tuple[str, dict[str, object]], ...] = ( + ("Batched (default)", {"codec_pipeline.path": BATCHED}), + ("Fused (1 worker)", {"codec_pipeline.path": FUSED, "codec_pipeline.max_workers": 1}), + ("Fused (cpu_count)", {"codec_pipeline.path": FUSED, "codec_pipeline.max_workers": None}), +) + + +def time_call(fn: Callable[[], object], repeat: int = 3) -> float: + """Median wall-clock seconds for one call to `fn`. + + `timeit.Timer` supplies `perf_counter` and disables the cyclic garbage + collector during each run, so a collection triggered by earlier work cannot + land inside a measurement. `number=1` because a single call here already + moves 64 MiB -- the per-call overhead `timeit` amortizes is irrelevant at + this scale. + """ + return statistics.median(timeit.Timer(fn).repeat(repeat=repeat, number=1)) + + +def measure( + settings: dict[str, object], + store: Store, + data: np.ndarray, + compressors: object, +) -> tuple[float, float]: + """Time one full write and one full read of `data` under `settings`. + + The whole operation runs inside `zarr.config.set`, not just the array + construction. The pipeline class is resolved when the array is built, but + `codec_pipeline.max_workers` is read *per operation*, so a timed call made + outside the config block would silently use whatever worker count was + globally in effect -- which makes every configuration look identical. + """ + everything = slice(None) + + def write_once() -> None: + with zarr.config.set(settings): + array = zarr.create_array( + store=store, + shape=SHAPE, + chunks=CHUNKS, + shards=SHARDS, + dtype=DTYPE, + compressors=compressors, + fill_value=0, + overwrite=True, + ) + operator.setitem(array, everything, data) + + write = time_call(write_once) + + # The bytes on disk are identical whichever pipeline wrote them, so reading + # back what we just wrote isolates read performance on the same data. + def read_once() -> object: + with zarr.config.set(settings): + return zarr.open_array(store=store, mode="r")[everything] + + read = time_call(read_once) + + if not np.array_equal(read_once(), data): + raise AssertionError("round trip mismatch") + return write, read + + +def make_store(kind: str, tmp: Path) -> Store: + if kind == "memory": + return MemoryStore() + return LocalStore(tmp / f"demo_{kind}_{os.getpid()}.zarr") + + +def main() -> None: + n_cpu = os.cpu_count() or 1 + + # Each regime gets the data that actually exercises it. `arange` is + # trivially compressible, which is fine when nothing compresses it, but it + # would make gzip finish almost instantly and hide the CPU-bound behavior + # this example is about. The noisy array keeps gzip genuinely busy. + n = int(np.prod(SHAPE)) + plain_data = np.arange(n, dtype=DTYPE).reshape(SHAPE) + noisy_data = np.random.default_rng(0).integers(0, 2**24, size=SHAPE, dtype=DTYPE) + + n_shards = int(np.prod([s // c for s, c in zip(SHAPE, SHARDS, strict=True)])) + per_shard = int(np.prod([s // c for s, c in zip(SHARDS, CHUNKS, strict=True)])) + print(f"zarr {zarr.__version__} | {n_cpu} CPUs") + print( + f"array {SHAPE} {DTYPE} = {plain_data.nbytes / 2**20:.0f} MiB | " + f"{n_shards} shards x {per_shard} inner chunks = {n_shards * per_shard} chunks\n" + ) + + with tempfile.TemporaryDirectory() as tmp: + for store_kind in ("memory", "local"): + for codec_label, compressors, data in ( + ("uncompressed", None, plain_data), + ("gzip-6 (CPU-bound)", GZIP, noisy_data), + ): + print(f"=== {store_kind} store / {codec_label} ===") + print( + f"{'pipeline':<22}{'write (s)':>11}{'vs base':>10}" + f"{'read (s)':>12}{'vs base':>10}" + ) + results: dict[str, tuple[float, float]] = {} + for label, settings in CONFIGS: + store = make_store(store_kind, Path(tmp)) + results[label] = measure(settings, store, data, compressors) + + base_write, base_read = results[CONFIGS[0][0]] + for label, (write, read) in results.items(): + print( + f"{label:<22}{write:>10.3f}{base_write / write:>9.1f}x" + f"{read:>11.3f}{base_read / read:>9.1f}x" + ) + + # The headline comparison: does the thread pool earn its keep? + single_write, single_read = results["Fused (1 worker)"] + pool_write, pool_read = results["Fused (cpu_count)"] + print( + f" workers (cpu_count vs 1 worker): " + f"write {single_write / pool_write:.1f}x " + f"read {single_read / pool_read:.1f}x" + ) + print() + + print( + "Reading it:\n" + " * Uncompressed IO is scheduling-bound, so Fused (1 worker) is already\n" + " fastest -- a thread pool has nothing to parallelize and only adds\n" + " overhead.\n" + " * gzip is CPU-bound, so Fused (1 worker) can be *slower* than the\n" + " default, while Fused (cpu_count) spreads compression across cores\n" + " and reclaims the win. That flip is why the fused pipeline is\n" + " threaded by default, and why pinning max_workers=1 is worth it for\n" + " memory-backed uncompressed data.\n" + " * `codec_pipeline.max_workers` is read only by the FusedCodecPipeline;\n" + " the default BatchedCodecPipeline ignores it." + ) + + +if __name__ == "__main__": + main() diff --git a/examples/sharding_coalescing/README.md b/examples/sharding_coalescing/README.md new file mode 100644 index 0000000000..29ba08c9ce --- /dev/null +++ b/examples/sharding_coalescing/README.md @@ -0,0 +1,63 @@ +# Sharded Read Coalescing + +This example demonstrates byte-range coalescing for partial reads of sharded +arrays, a performance optimization added in Zarr-Python 3.3.0 and enabled by +default. + +A shard is one stored object containing many inner chunks, each occupying its own +byte range. Reading N inner chunks could mean N separate byte-range requests. +Because byte ranges are intervals, nearby ranges can be merged: `[a, b)` and +`[b, c)` together cover `[a, c)`, so a single request can serve both. Merging +trades reading some bytes you did not ask for against issuing fewer requests -- +worthwhile whenever a request is expensive, as with object storage. + +## What it shows + +- Reading scattered inner chunks from one shard with coalescing **off** issues + one store request per inner chunk; with the **default** settings the same read + collapses to a single request. +- The resulting wall-clock difference against a store with simulated latency. +- That coalescing changes only *how* data is fetched, never *what* is returned -- + the script asserts both configurations produce identical arrays. +- A case where coalescing changes nothing: a contiguous selection already has + adjacent byte ranges, so it merges under any setting. + +## Running + +```bash +uv run sharding_coalescing.py +``` + +## How the comparison is set up + +Two details make the effect observable, and both are worth understanding if you +adapt this script: + +- **The selection must have gaps.** A contiguous read produces adjacent byte + ranges that merge regardless of configuration. The strided selections skip + inner chunks, creating the gaps that the `sharding_coalesce_max_gap_bytes` + budget decides whether to bridge. +- **Latency must be charged per merged fetch.** The example defines a small + `WrapperStore` subclass that sleeps in `get`. It deliberately does *not* keep + `WrapperStore.get_ranges`, which forwards straight to the wrapped store and + would bypass the latency entirely; inheriting the `Store` ABC's `get_ranges` + instead runs the coalescer over its own `get`, so each merged fetch pays once. + + `zarr.testing.store` ships a ready-made `LatencyStore`, but importing it pulls + in `pytest`. Defining the wrapper inline keeps the example runnable with only + `zarr` and `numpy` installed. + +## Configuration + +Two settings control the behavior, both settable globally via `zarr.config` or +per array via `config=` on `zarr.create_array` / `Array.with_config`: + +| Setting | Default | Meaning | +| --- | --- | --- | +| `sharding_coalesce_max_gap_bytes` | 1 MiB | Merge two ranges only if the gap between them is no larger than this | +| `sharding_coalesce_max_bytes` | 16 MiB | Never let a merged read exceed this size | + +Setting the gap to `0` merges only exactly-adjacent ranges, which approximates +the pre-3.3.0 behavior; that is how the example emulates the old path. Raising +the gap reads more unwanted bytes in exchange for fewer round trips -- the right +value depends on how expensive a request is against how fast your link is. diff --git a/examples/sharding_coalescing/sharding_coalescing.py b/examples/sharding_coalescing/sharding_coalescing.py new file mode 100644 index 0000000000..5da62ed806 --- /dev/null +++ b/examples/sharding_coalescing/sharding_coalescing.py @@ -0,0 +1,226 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "zarr @ git+https://github.com/zarr-developers/zarr-python.git@main", +# "numpy", +# ] +# /// + +""" +Demonstrate byte-range coalescing for partial reads of sharded arrays. + +A shard is a single stored object holding many inner chunks, each occupying +its own byte range. Reading N inner chunks could mean issuing N separate +byte-range requests to the store. Because byte ranges are intervals, a reader +can instead merge nearby ranges: `[a, b)` and `[b, c)` together cover +`[a, c)`, so one request can serve both. Merging trades reading some bytes you +did not ask for against issuing fewer requests -- a good trade whenever a +request is expensive, which is the normal case for object storage. + +Zarr-Python 3.3.0 does this automatically. Two settings control it: + + * `sharding_coalesce_max_gap_bytes` (default 1 MiB) -- merge two ranges only + if the gap between them is no larger than this. + * `sharding_coalesce_max_bytes` (default 16 MiB) -- never let a merged read + exceed this size. + +Setting the gap to 0 disables merging of non-adjacent ranges, which +approximates the pre-3.3.0 behavior. This script compares the two, counting +store requests and measuring wall-clock time against a store with simulated +latency. + +Run it with: + + uv run sharding_coalescing.py +""" + +from __future__ import annotations + +import asyncio +import operator +import statistics +import timeit +from contextlib import contextmanager +from functools import partial +from typing import TYPE_CHECKING + +import numpy as np + +import zarr +import zarr.core._coalesce as coalesce_module +from zarr.abc.store import ByteRequest, RangeByteRequest, Store +from zarr.storage import MemoryStore, WrapperStore + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator, Sequence + + from zarr.core.buffer import Buffer, BufferPrototype + +# Emulates the pre-3.3.0 behavior: with a zero gap budget, only ranges that are +# exactly adjacent get merged, so scattered inner chunks are fetched one by one. +NO_COALESCING = {"sharding_coalesce_max_gap_bytes": 0} + +# The shipped defaults. Spelled out here so the comparison is explicit rather +# than relying on whatever the global config happens to be. +DEFAULT_COALESCING = { + "sharding_coalesce_max_gap_bytes": 1 << 20, # 1 MiB + "sharding_coalesce_max_bytes": 16 << 20, # 16 MiB +} + +GET_LATENCY_S = 0.005 # 5 ms per request, a modest stand-in for object storage + + +class PerRequestLatencyStore(WrapperStore[Store]): + """Wraps a store, charging a fixed latency per byte-range fetch. + + `zarr.testing.store` ships a `LatencyStore`, but importing it pulls in + `pytest`; defining the wrapper here keeps this example runnable with only + zarr and numpy installed. + + Two details matter for the measurement: + + * The latency is applied in `get`, which is what an individual fetch costs. + * `get_ranges` is explicitly *not* overridden to forward to the wrapped + store. `WrapperStore.get_ranges` does forward, which would skip this + class's `get` entirely and make every configuration look identical. + Inheriting the `Store` ABC's implementation instead runs the coalescer + over `self.get`, so each *merged* fetch pays the latency once -- which is + exactly the cost coalescing exists to reduce. + """ + + get_ranges = Store.get_ranges + + def __init__(self, store: Store, *, get_latency: float) -> None: + super().__init__(store) + self.get_latency = get_latency + + def _with_store(self, store: Store) -> PerRequestLatencyStore: + # `WrapperStore` rebuilds the wrapper when opening read-only, so the + # latency setting has to be carried across. + return type(self)(store, get_latency=self.get_latency) + + async def get( + self, + key: str, + prototype: BufferPrototype, + byte_range: ByteRequest | None = None, + ) -> Buffer | None: + await asyncio.sleep(self.get_latency) + return await self._store.get(key, prototype, byte_range) + + +@contextmanager +def counting_requests() -> Iterator[Callable[[], int]]: + """Count the store fetches issued inside the block. + + Wraps the coalescing planner rather than the store: every merged group it + returns, plus every range it declined to merge, becomes exactly one fetch. + Counting here rather than at the store means the number reported is the + planner's decision, which is precisely what the settings control. + """ + original = coalesce_module.coalesce_ranges + total = 0 + + def counting_coalesce_ranges( + byte_ranges: Sequence[ByteRequest | None], + *, + max_gap_bytes: int, + max_coalesced_bytes: int, + ) -> tuple[ + list[list[tuple[int, RangeByteRequest]]], + list[tuple[int, ByteRequest | None]], + ]: + nonlocal total + groups, uncoalescable = original( + byte_ranges, + max_gap_bytes=max_gap_bytes, + max_coalesced_bytes=max_coalesced_bytes, + ) + total += len(groups) + len(uncoalescable) + return groups, uncoalescable + + coalesce_module.coalesce_ranges = counting_coalesce_ranges + try: + yield lambda: total + finally: + coalesce_module.coalesce_ranges = original + + +def measure_read(array: zarr.Array, selection: slice) -> tuple[int, float]: + """Return (store fetches, median seconds) for reading `selection`.""" + read = partial(operator.getitem, array, selection) + + with counting_requests() as fetches: + result = read() + requests = fetches() + + # `timeit.Timer` supplies the loop, `perf_counter`, and GC handling. The + # median of several runs keeps one unlucky run from dominating. + elapsed = statistics.median(timeit.Timer(read).repeat(repeat=5, number=1)) + + assert result.size > 0 # a read that returned nothing would time as "fast" + return requests, elapsed + + +def main() -> None: + n = 8192 + chunk = 64 + inner_chunks = n // chunk + + base = MemoryStore() + source = (np.arange(n, dtype="uint64") % 251).astype("uint8") + + # One shard holding every inner chunk, uncompressed so inner-chunk byte + # offsets stay predictable and the demonstration is easy to reason about. + writable = zarr.create_array( + store=base, shape=(n,), chunks=(chunk,), shards=(n,), dtype="uint8", compressors=None + ) + writable[:] = source + + store = PerRequestLatencyStore(base, get_latency=GET_LATENCY_S) + + print(f"zarr {zarr.__version__}") + print(f"array: {n} uint8 values, {inner_chunks} inner chunks of {chunk} in a single shard") + print(f"store: MemoryStore wrapped with {GET_LATENCY_S * 1000:.0f} ms of latency per request\n") + + # A strided selection touches inner chunks with unread chunks in between, + # so there are real gaps for the coalescer to bridge. A contiguous + # selection would merge under any setting, since its ranges are adjacent. + selections = { + "every 2nd inner chunk": slice(None, None, chunk * 2), + "every 4th inner chunk": slice(None, None, chunk * 4), + "contiguous quarter": slice(0, n // 4), + } + + header = f"{'selection':<24} {'coalescing':<12} {'requests':>9} {'time':>10}" + print(header) + print("-" * len(header)) + + for label, selection in selections.items(): + results = {} + for mode, config in (("off", NO_COALESCING), ("default", DEFAULT_COALESCING)): + array = zarr.open_array(store=store, mode="r").with_config(config) + requests, elapsed = measure_read(array, selection) + results[mode] = (requests, elapsed) + print(f"{label:<24} {mode:<12} {requests:>9} {elapsed * 1000:>9.1f}ms") + + off_requests, off_time = results["off"] + on_requests, on_time = results["default"] + if on_requests < off_requests: + print( + f"{'':<24} {'->':<12} " + f"{off_requests // on_requests:>8}x fewer {off_time / on_time:>9.1f}x faster" + ) + else: + print(f"{'':<24} {'->':<12} {'no change (ranges already adjacent)':>30}") + print() + + # Correctness is the point: coalescing must not change what you read back. + for mode, config in (("off", NO_COALESCING), ("default", DEFAULT_COALESCING)): + array = zarr.open_array(store=store, mode="r").with_config(config) + assert np.array_equal(array[::128], source[::128]), mode + print("Both configurations return identical data; coalescing only changes how it is fetched.") + + +if __name__ == "__main__": + main() diff --git a/mkdocs.yml b/mkdocs.yml index 46bfc1764c..647bc40a2a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -31,6 +31,8 @@ nav: - Examples: - user-guide/examples/custom_dtype.md - user-guide/examples/rectilinear_chunks.md + - user-guide/examples/codec_pipeline_performance.md + - user-guide/examples/sharding_coalescing.md - API Reference: - api/zarr/index.md - ' zarr.abc': @@ -94,6 +96,8 @@ nav: - ' zarr.zeros_like': api/zarr/functions/zeros_like.md - release-notes.md - contributing.md + - Blog: + - blog/index.md hooks: - mkdocs_hooks.py @@ -152,6 +156,14 @@ extra_css: plugins: - autorefs + - blog: + blog_dir: blog + post_dir: "{blog}/posts" + post_url_format: "{slug}" + # The blog is a simple reverse-chronological list of posts; the archive + # and category indexes add navigation we don't have the volume to justify. + archive: false + categories: false - search - markdown-exec - mkdocstrings: diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index a842e07886..69b80d7332 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -1,51 +1,86 @@ # zarr-metadata -Python type definitions for Zarr v2 and v3 metadata. +Python types, models, and validators for Zarr v2 and v3 metadata. ## What this is -A typed-data package: `TypedDict` definitions and `Literal` aliases for the -JSON shapes specified by the [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html) -and [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) -specifications, plus types for [`zarr-extensions`](https://github.com/zarr-developers/zarr-extensions/) -and a few widely-used-but-unspecified entities (e.g. consolidated metadata). +Two layers and an optional integration: + +- **Typed JSON shapes**: `TypedDict` definitions and `Literal` aliases for the + JSON documents specified by the [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html) + and [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) + specifications, plus types for [`zarr-extensions`](https://github.com/zarr-developers/zarr-extensions/) + and a few widely-used-but-unspecified entities (e.g. consolidated metadata). +- **Document models** (`zarr_metadata.model`): canonical frozen-dataclass + models of whole metadata documents, with structural validators, loc-aware + parsers, and store-key (de)serialization. A document produced by `to_json` + shares no mutable state with the model that produced it. +- **Optional Pydantic integration** (`zarr_metadata.pydantic`, requires + Pydantic 2.13 or newer): each model as a Pydantic field type that validates + raw documents through the same strict parser. ## What this is for -These types describe the JSON shape of Zarr metadata. They are -intended for libraries that **read, write, validate, or transform** -Zarr metadata. Pair them with a runtime validator like -[pydantic](https://docs.pydantic.dev/) to check JSON loaded from disk: +The public `TypedDict` definitions describe the static JSON shape of Zarr +metadata. For strict, loc-aware validation of JSON loaded from disk, use the +model parser: ```python import json -from pydantic import TypeAdapter -from zarr_metadata.v3.array import ArrayMetadataV3 +from zarr_metadata.model import ZarrV3ArrayMetadata with open("zarr.json", "rb") as f: raw = json.load(f) -metadata = TypeAdapter(ArrayMetadataV3).validate_python(raw) +metadata = ZarrV3ArrayMetadata.from_json(raw) +``` + +The optional Pydantic integration delegates raw input to the same strict +parser and returns the same normalized model class: + +```python +from pydantic import TypeAdapter +import zarr_metadata.pydantic as zmp + +metadata = TypeAdapter(zmp.ZarrV3ArrayMetadata).validate_python(raw) +encoded = metadata.to_key_value()["zarr.json"] ``` -## What this is *not* +A bare `TypeAdapter` over a public document `TypedDict` is a coercive shape +adapter, not a Zarr conformance validator; it may coerce values or discard +members that the strict model parser rejects. + +## Validation boundary -- Not a parser or builder. There are no `make_array_metadata(...)` factories — - that surface belongs to consumer libraries. -- Not a runtime validator on its own. Pair with `pydantic`, `msgspec`, or - similar to enforce shapes at decode time. +The model validators enforce the declared document structure and a small set +of context-free consistency rules, including fixed format literals, finite +JSON numbers, non-negative dimensions, non-empty v3 codec pipelines, and one +`dimension_names` entry per array dimension. They do not interpret extension +names or configurations, resolve codec pipelines, or decide whether a data +type, chunk grid, codec, or storage transformer is supported. Those decisions +belong to consumer implementations. -Even with a runtime validator, these types only describe **structural** -shape — they will not flag *semantically* invalid metadata, like a 3D v3 -array whose `dimension_names` has 4 entries instead of 3. That's a job -for downstream validator routines. +The Pydantic integration's generated JSON Schemas express independently +checkable document structure and field constraints, but they are not a +replacement for runtime model validation. Standard JSON Schema treats a +mathematically integral number such as `1.0` as an integer, while the runtime +boundary requires Python `int` values, and it cannot express arbitrary +same-length relations such as `dimension_names` versus `shape` or v2 `chunks` +versus `shape`. Consumers should run the model parser after schema validation. ## Scope At minimum, this library supports what Zarr-Python needs: the complete Zarr v2 and v3 specs, consolidated metadata, and a subset of the metadata defined in `zarr-extensions`. We are generally open to contributions that -add types for Zarr metadata with a published spec. +add types, models, or structural validation for Zarr metadata with a +published spec. + +Runtime array behavior is out of scope: nothing here encodes or decodes +chunks, resolves codec or data type names to implementations, or performs +store I/O. The models begin and end at the metadata documents themselves — +`from_key_value` / `to_key_value` map documents to store keys and bytes, +and everything past that belongs to consumer libraries. ## Releasing diff --git a/packages/zarr-metadata/changes/4119.feature.md b/packages/zarr-metadata/changes/4119.feature.md new file mode 100644 index 0000000000..b9d0bb508c --- /dev/null +++ b/packages/zarr-metadata/changes/4119.feature.md @@ -0,0 +1,92 @@ +Added `zarr_metadata.model`: frozen-dataclass models (`ZarrV2ArrayMetadata`, +`ZarrV3ArrayMetadata`, `ZarrV2GroupMetadata`, `ZarrV3GroupMetadata`, +`ZarrV2ConsolidatedMetadata`, `ZarrV3ConsolidatedMetadata`, `ZarrV3NamedConfig`) +that are canonical, semantically lossless representations of Zarr metadata +documents, plus structural validators (`validate_*` / `is_*` / `parse_*`). +Every v3 extension point (data type, chunk grid, chunk key encoding, codecs, +storage transformers) is held as `ZarrV3NamedConfig`: a name, configuration, +and `must_understand` obligation; nothing is interpreted. On the wire, an +empty configuration with the default obligation uses the spec's plain-string +shorthand. Model fields are annotated with the role alias +`ZarrV3MetadataField` (today exactly `ZarrV3NamedConfig`), so annotations +convey the logical meaning and stay put if the spec adds another field form. + +Validation is strict about what the types declare: v2 `dtype` / `order` / +`compressor` / `filters` / `dimension_separator` shapes and the fixed +`zarr_format` / `node_type` literals are all enforced. Every +`ValidationProblem` carries a machine-readable `kind` +(`missing_key` / `invalid_type` / `invalid_value` / `invalid_json`) so +consumers can dispatch on the failure mode without matching message strings, +and every ingestion failure — including missing store keys and undecodable +bytes in `from_key_value` — surfaces as `MetadataValidationError`. An +adversarial review added further structural checks: JSON booleans are not +accepted as dimension lengths, dimensions are non-negative, +`dimension_names` must have one entry per dimension of `shape`, `attributes` +and `configuration` values are JSON-checked recursively (like `fill_value`), +non-finite floats and non-standard JSON constants are rejected, abstract +mappings and sequences normalize to encoder-safe canonical containers, +v2 `shape` and `chunks` must have the same rank, non-null v2 filter pipelines +contain at least one filter, document `TypeIs` guards only narrow values that +already use the declared canonical containers, +and the inline consolidated-metadata envelope and entries are deep-validated +so the group validator's verdict always agrees with the model constructor. + +The v3 models expose `must_understand_fields`: the subset of `extra_fields` +not explicitly waived with `must_understand: false` (fields are implicitly +must-understand per the spec). Readers discharge the spec's fail-to-open +duty by subtracting the extension names they recognize; the model only +partitions by obligation, since recognition is reader-specific. + +Optional pydantic integration ships as `zarr_metadata.pydantic` (importing it +requires pydantic 2.13 or newer; the core package does not depend on it): one +`Annotated` +field type per model, validating raw documents through `from_json`, passing +core-model instances through unchanged, serializing via `to_json`, and +publishing JSON Schemas derived from private constrained document types that +mirror the independently expressible runtime rules. Cross-field cardinality +relations still require runtime validation. The instances are the core model +classes, so values interoperate freely with non-pydantic code. + +`create_default` keeps its output self-consistent: overriding `shape` without +a chunk grid derives one regular chunk covering the array (v3 +`chunk_shape == shape`; v2 `chunks == shape`) instead of silently keeping the +scalar default's 0-d grid. + +A v2 `.zarray` that omits `dimension_separator` is interpreted with the v2 +convention's default `"."` (the model previously normalized absence to `"/"`, +which would misaddress the chunks of real-world default-separator arrays). +The value is never null: absent, `"."`, or `"/"` are the only spellings. + +Optional document keys use `UNSET` — a PEP 661 sentinel +(`typing_extensions.Sentinel`), usable directly in type expressions — never +`None`: in a model, `None` always corresponds to a JSON `null` in the +document (a v2 `compressor`, an unnamed dimension inside `dimension_names`), +and `UNSET` always means the key is absent. Checker note: ty types the +sentinel exactly; pyright needs `<= 1.1.404` until microsoft/pyright#11115 +is fixed (this package's CI pins it); mypy users need a `cast` or +`type: ignore` at narrowing sites until python/mypy#21647 merges. This keeps semantically distinct spellings +distinct — an absent `dimension_names` ("there are no dimension names") and +an explicit `[null, null]` ("every dimension has a name, which is null") are +different documents and round-trip as such. The `consolidated_metadata: null` +written by a historical zarr-python bug is the one deliberate exception to +faithful round-tripping: those stores remain readable, but the bug spelling +is repaired to absence on read and never written back. + +The v2 models treat the `.zattrs` file's presence as part of the store: +`attributes` is `UNSET` when no `.zattrs` file exists (and `to_key_value` +emits none), while an explicit empty `.zattrs` is `{}` and round-trips as a +file. Previously `to_key_value` always emitted `.zattrs`, silently adding a +file to stores that never had one. + +The store-key `Literal` aliases (`ZarrV2ArrayMetadataStoreKey`, +`ZarrV2AttributesStoreKey`, ...) are exported from `zarr_metadata.model` +alongside their constants, and each `to_key_value` return type is keyed by +them, so the set of store keys a model can emit is visible in its signature. +`from_key_value` deliberately keeps `Mapping[str, bytes]` input: it accepts +any string-keyed store mapping and ignores unrelated keys. + +`to_json` returns a document that shares no mutable state with the model: +every value that can hold a mutable container (attributes, configurations, +extra fields, v2 codec configurations, fill values, consolidated entries) is +deep-copied on the way out, so editing a serialized document can never +silently mutate the frozen model that produced it. diff --git a/packages/zarr-metadata/changes/4119.removal.md b/packages/zarr-metadata/changes/4119.removal.md new file mode 100644 index 0000000000..2a9a6f84c4 --- /dev/null +++ b/packages/zarr-metadata/changes/4119.removal.md @@ -0,0 +1,41 @@ +The document (TypedDict) types are renamed to put the format version at the +front of the name and to mark the JSON-document form with a `JSON` suffix, +so a format version can never be misread as a class revision and the bare +entity names are reserved for the `zarr_metadata.model` dataclasses: + +- `ArrayMetadataV2` → `ZarrV2ArrayMetadataJSON` (and `...Partial` accordingly) +- `ArrayMetadataV3` → `ZarrV3ArrayMetadataJSON` (and `...Partial` accordingly) +- `GroupMetadataV2` → `ZarrV2GroupMetadataJSON` (and `...Partial` accordingly) +- `GroupMetadataV3` → `ZarrV3GroupMetadataJSON` (and `...Partial` accordingly) +- `ConsolidatedMetadataV2` → `ZarrV2ConsolidatedMetadataJSON` +- `ConsolidatedMetadataV3` → `ZarrV3ConsolidatedMetadataJSON` +- `NamedConfigV3` → `ZarrV3NamedConfigJSON` +- `MetadataV3` → `ZarrV3MetadataFieldJSON` (the union of the bare-name and + named-configuration spellings of one metadata field) +- `ExtensionFieldV3` → `ZarrV3ExtensionField` +- `CodecMetadataV2` → `ZarrV2CodecMetadata` +- `DataTypeMetadataV2` → `ZarrV2DataTypeMetadata` +- `ArrayOrderV2` → `ZarrV2ArrayOrder` +- `ArrayDimensionSeparatorV2` → `ZarrV2ArrayDimensionSeparator` +- `ZArrayMetadata` → `ZarrV2ZArrayJSON` (the strict on-disk `.zarray` document) +- `ZGroupMetadata` → `ZarrV2ZGroupJSON` (the strict on-disk `.zgroup` document) +- `ZAttrsMetadata` → `ZarrV2ZAttrsJSON` (the `.zattrs` document) + +The old names are removed, not aliased. The `zarr_metadata.pydantic` field +types take the bare entity names (`ZarrV3ArrayMetadata`, ...), matching the +model classes they validate into. + +The conventions, stated once for future additions: CamelCase type names put +the format version first (`ZarrV2ArrayMetadataJSON`, +`ZarrV3ArrayMetadataStoreKey`), while SCREAMING_SNAKE constants and +snake_case functions put it last (`ARRAY_METADATA_STORE_KEY_V2`, +`validate_array_metadata_v3`). The `JSON` suffix marks a raw-document type +whose bare name is taken by (or reserved for) a `zarr_metadata.model` +dataclass; raw field-level types the models hold verbatim +(`ZarrV2CodecMetadata`, `ZarrV3ExtensionField`) keep their bare names. +Extension-entity types put the registered entity name first and end in +exactly one role suffix (`BloscCodecMetadata`, `Uint8DataTypeName`) — the +`V2` in `V2ChunkKeyEncodingMetadata` is that encoding's entity name, not a +format version, which is always spelled `ZarrV2`/`ZarrV3`. Every public +type name is checked against this grammar by +`tests/test_public_api.py::test_public_type_names_comply_with_naming_grammar`. diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 05667d59e3..edc4b696a6 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "zarr-metadata" dynamic = ["version"] -description = "Spec-defined metadata types for Zarr v2 and v3." +description = "Spec-defined metadata types, models, and validators for Zarr v2 and v3." readme = "README.md" requires-python = ">=3.11" license = "MIT" @@ -32,7 +32,11 @@ classifiers = [ ] keywords = ["zarr"] dependencies = [ - "typing_extensions>=4.13", + # >=4.16: first release where `Sentinel` pickles by reference + # (`__reduce__` returns the sentinel's name), so `UNSET` — and any model + # holding it — can cross process boundaries and be deep-copied with its + # singleton identity intact. 4.15 and earlier refuse to pickle sentinels. + "typing_extensions>=4.16", ] [project.urls] @@ -43,7 +47,7 @@ Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/z Documentation = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-metadata/README.md" [dependency-groups] -test = ["pytest", "pydantic>=2"] +test = ["pytest", "pydantic>=2.13", "jsonschema"] [tool.hatch.version] source = "vcs" @@ -67,9 +71,9 @@ xfail_strict = true addopts = ["-ra", "--strict-config", "--strict-markers"] filterwarnings = [ "error", - # pydantic warns about ReadOnly TypedDict items not being enforced at runtime. - # That's expected here — we rely on type-checker enforcement, not pydantic mutation guards. - "ignore::UserWarning:pydantic._internal._generate_schema", + # Pydantic validates these public immutable-shape TypedDicts correctly but + # cannot enforce the type checker's ReadOnly mutation restriction. + "ignore:Items? .* using the `ReadOnly` qualifier.*:UserWarning:pydantic._internal._generate_schema", ] [tool.numpydoc_validation] @@ -82,6 +86,9 @@ checks = [ "PR06", ] +# CI pins pyright==1.1.404: later versions regress PEP 661 sentinel typing in +# class attributes (microsoft/pyright#11115), which zarr_metadata.model._sentinel +# relies on. Use the same pin locally; unpin when the fix lands. [tool.pyright] include = ["src"] enableExperimentalFeatures = true diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index 46949570a2..b5e52e976d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -1,22 +1,48 @@ from importlib.metadata import version -from zarr_metadata._common import JSONValue, NamedConfigV3 +from zarr_metadata._common import JSONValue, ZarrV3NamedConfigJSON +from zarr_metadata.model import ( + UNSET, + MetadataValidationError, + ProblemKind, + ValidationProblem, + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadataPartial, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV2GroupMetadataPartial, + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadataPartial, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, + ZarrV3GroupMetadataPartial, + ZarrV3MetadataField, + ZarrV3NamedConfig, +) from zarr_metadata.v2.array import ( ARRAY_DIMENSION_SEPARATOR_V2, ARRAY_ORDER_V2, - ArrayDimensionSeparatorV2, - ArrayMetadataV2, - ArrayMetadataV2Partial, - ArrayOrderV2, - DataTypeMetadataV2, - ZArrayMetadata, -) -from zarr_metadata.v2.attributes import ZAttrsMetadata -from zarr_metadata.v2.codec import CodecMetadataV2 -from zarr_metadata.v2.consolidated import ConsolidatedMetadataV2 -from zarr_metadata.v2.group import GroupMetadataV2, GroupMetadataV2Partial, ZGroupMetadata -from zarr_metadata.v3._common import MetadataV3 -from zarr_metadata.v3.array import ArrayMetadataV3, ArrayMetadataV3Partial, ExtensionFieldV3 + ZarrV2ArrayDimensionSeparator, + ZarrV2ArrayMetadataJSON, + ZarrV2ArrayMetadataJSONPartial, + ZarrV2ArrayOrder, + ZarrV2DataTypeMetadata, + ZarrV2ZArrayJSON, +) +from zarr_metadata.v2.attributes import ZarrV2ZAttrsJSON +from zarr_metadata.v2.codec import ZarrV2CodecMetadata +from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON +from zarr_metadata.v2.group import ( + ZarrV2GroupMetadataJSON, + ZarrV2GroupMetadataJSONPartial, + ZarrV2ZGroupJSON, +) +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +from zarr_metadata.v3.array import ( + ZarrV3ArrayMetadataJSON, + ZarrV3ArrayMetadataJSONPartial, + ZarrV3ExtensionField, +) from zarr_metadata.v3.chunk_grid.rectilinear import ( RECTILINEAR_CHUNK_GRID_NAME, RectilinearChunkGridMetadata, @@ -86,7 +112,7 @@ TransposeCodecName, ) from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME, ZstdCodecMetadata, ZstdCodecName -from zarr_metadata.v3.consolidated import ConsolidatedMetadataV3 +from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON from zarr_metadata.v3.data_type.bool import ( BOOL_DATA_TYPE_NAME, BoolDataTypeName, @@ -185,7 +211,7 @@ Uint64DataTypeName, Uint64FillValue, ) -from zarr_metadata.v3.group import GroupMetadataV3, GroupMetadataV3Partial +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON, ZarrV3GroupMetadataJSONPartial __version__ = version("zarr-metadata") @@ -231,15 +257,10 @@ "UINT16_DATA_TYPE_NAME", "UINT32_DATA_TYPE_NAME", "UINT64_DATA_TYPE_NAME", + "UNSET", "V2_CHUNK_KEY_ENCODING_NAME", "V2_CHUNK_KEY_ENCODING_SEPARATOR", "ZSTD_CODEC_NAME", - "ArrayDimensionSeparatorV2", - "ArrayMetadataV2", - "ArrayMetadataV2Partial", - "ArrayMetadataV3", - "ArrayMetadataV3Partial", - "ArrayOrderV2", "BloscCName", "BloscCodecMetadata", "BloscCodecName", @@ -254,31 +275,22 @@ "CastRoundingMode", "CastValueCodecMetadata", "CastValueCodecName", - "CodecMetadataV2", "Complex64DataTypeName", "Complex64FillValue", "Complex128DataTypeName", "Complex128FillValue", - "ConsolidatedMetadataV2", - "ConsolidatedMetadataV3", "Crc32cCodecMetadata", "Crc32cCodecName", - "DataTypeMetadataV2", "DefaultChunkKeyEncodingMetadata", "DefaultChunkKeyEncodingName", "DefaultChunkKeyEncodingSeparator", "Endianness", - "ExtensionFieldV3", "Float16DataTypeName", "Float16FillValue", "Float32DataTypeName", "Float32FillValue", "Float64DataTypeName", "Float64FillValue", - "GroupMetadataV2", - "GroupMetadataV2Partial", - "GroupMetadataV3", - "GroupMetadataV3Partial", "GzipCodecMetadata", "GzipCodecName", "Int8DataTypeName", @@ -290,13 +302,13 @@ "Int64DataTypeName", "Int64FillValue", "JSONValue", - "MetadataV3", - "NamedConfigV3", + "MetadataValidationError", "NumpyDatetime64DataTypeName", "NumpyDatetime64FillValue", "NumpyTimeUnit", "NumpyTimedelta64DataTypeName", "NumpyTimedelta64FillValue", + "ProblemKind", "RawBytesDataTypeName", "RawBytesFillValue", "RectilinearChunkGridMetadata", @@ -325,9 +337,39 @@ "V2ChunkKeyEncodingMetadata", "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingSeparator", - "ZArrayMetadata", - "ZAttrsMetadata", - "ZGroupMetadata", + "ValidationProblem", + "ZarrV2ArrayDimensionSeparator", + "ZarrV2ArrayMetadata", + "ZarrV2ArrayMetadataJSON", + "ZarrV2ArrayMetadataJSONPartial", + "ZarrV2ArrayMetadataPartial", + "ZarrV2ArrayOrder", + "ZarrV2CodecMetadata", + "ZarrV2ConsolidatedMetadata", + "ZarrV2ConsolidatedMetadataJSON", + "ZarrV2DataTypeMetadata", + "ZarrV2GroupMetadata", + "ZarrV2GroupMetadataJSON", + "ZarrV2GroupMetadataJSONPartial", + "ZarrV2GroupMetadataPartial", + "ZarrV2ZArrayJSON", + "ZarrV2ZAttrsJSON", + "ZarrV2ZGroupJSON", + "ZarrV3ArrayMetadata", + "ZarrV3ArrayMetadataJSON", + "ZarrV3ArrayMetadataJSONPartial", + "ZarrV3ArrayMetadataPartial", + "ZarrV3ConsolidatedMetadata", + "ZarrV3ConsolidatedMetadataJSON", + "ZarrV3ExtensionField", + "ZarrV3GroupMetadata", + "ZarrV3GroupMetadataJSON", + "ZarrV3GroupMetadataJSONPartial", + "ZarrV3GroupMetadataPartial", + "ZarrV3MetadataField", + "ZarrV3MetadataFieldJSON", + "ZarrV3NamedConfig", + "ZarrV3NamedConfigJSON", "ZstdCodecMetadata", "ZstdCodecName", "__version__", diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index 598a12e80c..f3259f7b73 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -13,7 +13,14 @@ JSONValue = TypeAliasType( "JSONValue", - "int | float | bool | None | str | list[JSONValue] | tuple[JSONValue, ...] | Mapping[str, JSONValue]", # type: ignore[reportInvalidTypeForm] + int + | float + | bool + | str + | list["JSONValue"] + | tuple["JSONValue", ...] + | Mapping[str, "JSONValue"] + | None, ) """A recursive type alias for JSON-encodable values. @@ -24,13 +31,14 @@ """ -class NamedConfigV3(TypedDict): +class ZarrV3NamedConfigJSON(TypedDict): """ Externally-tagged union member for a metadata field. - The `configuration` mapping holds arbitrary JSON-encodable values; - it is typed as `Mapping[str, JSONValue]`. + The optional `configuration` mapping holds arbitrary JSON-encodable + values. `must_understand` is implicitly true when absent. """ name: str configuration: NotRequired[Mapping[str, JSONValue]] + must_understand: NotRequired[bool] diff --git a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py new file mode 100644 index 0000000000..e9792d6931 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py @@ -0,0 +1,114 @@ +"""Private input types used only to generate accurate Pydantic JSON schemas.""" + +from __future__ import annotations + +from collections.abc import Mapping # noqa: TC003 # resolved by Pydantic at runtime +from typing import Annotated, Literal, NotRequired + +from pydantic import Field +from typing_extensions import TypedDict + +from zarr_metadata._common import JSONValue +from zarr_metadata.v2.array import ( # noqa: TC001 # resolved by Pydantic at runtime + ZarrV2DataTypeMetadata, +) +from zarr_metadata.v2.codec import ( # resolved by Pydantic at runtime + ZarrV2CodecMetadata, +) + +NonNegativeInt = Annotated[int, Field(ge=0)] + + +class ZarrV3NamedConfigJSON(TypedDict, closed=True): + """Closed v3 named configuration accepted at optional extension points.""" + + name: str + configuration: NotRequired[Mapping[str, JSONValue]] + must_understand: NotRequired[bool] + + +class ZarrV3MandatoryNamedConfigJSON(TypedDict, closed=True): + """Closed named configuration accepted where understanding is mandatory.""" + + name: str + configuration: NotRequired[Mapping[str, JSONValue]] + must_understand: NotRequired[Literal[True]] + + +ZarrV3MetadataFieldJSON = str | ZarrV3NamedConfigJSON +ZarrV3MandatoryMetadataFieldJSON = str | ZarrV3MandatoryNamedConfigJSON +ZarrV3CodecPipelineJSON = Annotated[tuple[ZarrV3MetadataFieldJSON, ...], Field(min_length=1)] +ZarrV2FilterPipelineJSON = Annotated[tuple[ZarrV2CodecMetadata, ...], Field(min_length=1)] + + +class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=JSONValue): + """Schema input for a v3 array document, including arbitrary extensions.""" + + zarr_format: Literal[3] + node_type: Literal["array"] + data_type: ZarrV3MandatoryMetadataFieldJSON + shape: tuple[NonNegativeInt, ...] + chunk_grid: ZarrV3MandatoryMetadataFieldJSON + chunk_key_encoding: ZarrV3MandatoryMetadataFieldJSON + fill_value: JSONValue + codecs: ZarrV3CodecPipelineJSON + attributes: NotRequired[Mapping[str, JSONValue]] + storage_transformers: NotRequired[tuple[ZarrV3MetadataFieldJSON, ...]] + dimension_names: NotRequired[tuple[str | None, ...]] + + +class ZarrV3ConsolidatedMetadataJSON(TypedDict, closed=True): + """Schema input for the closed inline consolidated-metadata envelope.""" + + kind: Literal["inline"] + must_understand: Literal[False] + metadata: Mapping[str, ZarrV3ArrayMetadataJSON | ZarrV3GroupMetadataJSON] + + +class ZarrV3GroupMetadataJSON(TypedDict, extra_items=JSONValue): + """Schema input for a v3 group document, including arbitrary extensions.""" + + zarr_format: Literal[3] + node_type: Literal["group"] + attributes: NotRequired[Mapping[str, JSONValue]] + consolidated_metadata: NotRequired[ZarrV3ConsolidatedMetadataJSON | None] + + +class ZarrV2ArrayMetadataJSON(TypedDict, closed=True): + """Schema input for the closed, merged v2 array representation.""" + + zarr_format: Literal[2] + shape: tuple[NonNegativeInt, ...] + chunks: tuple[NonNegativeInt, ...] + dtype: ZarrV2DataTypeMetadata + compressor: ZarrV2CodecMetadata | None + fill_value: JSONValue + order: Literal["C", "F"] + filters: ZarrV2FilterPipelineJSON | None + dimension_separator: NotRequired[Literal[".", "/"]] + attributes: NotRequired[Mapping[str, JSONValue]] + + +class ZarrV2GroupMetadataJSON(TypedDict, closed=True): + """Schema input for the closed, merged v2 group representation.""" + + zarr_format: Literal[2] + attributes: NotRequired[Mapping[str, JSONValue]] + + +class ZarrV2ConsolidatedMetadataJSON(TypedDict, closed=True): + """Schema input matching the v2 consolidated model's structural parser.""" + + zarr_consolidated_format: Literal[1] + metadata: Mapping[str, JSONValue] + + +__all__ = [ + "ZarrV2ArrayMetadataJSON", + "ZarrV2ConsolidatedMetadataJSON", + "ZarrV2GroupMetadataJSON", + "ZarrV3ArrayMetadataJSON", + "ZarrV3ConsolidatedMetadataJSON", + "ZarrV3GroupMetadataJSON", + "ZarrV3MetadataFieldJSON", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py new file mode 100644 index 0000000000..e726c54d3e --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -0,0 +1,132 @@ +"""In-memory models for Zarr metadata documents. + +Models are frozen dataclasses that hold a canonical, semantically lossless +representation of the JSON documents; they never interpret extension points +(codecs, chunk grids, data types). Validators check JSON structure, not domain validity. +Each document concept gets a `validate_*` function returning every problem +found (a `list[ValidationProblem]`, each with a machine-readable `kind`), an +`is_*` type guard, and a `parse_*` function that narrows or raises +`MetadataValidationError`. Model `from_json` / `from_key_value` constructors +raise `MetadataValidationError` for every ingestion failure, including +missing store keys and undecodable bytes. +""" + +from zarr_metadata.model._array import ( + ARRAY_METADATA_STORE_KEY_V2, + ARRAY_METADATA_STORE_KEY_V3, + ATTRIBUTES_STORE_KEY_V2, + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadataPartial, + ZarrV2ArrayMetadataStoreKey, + ZarrV2AttributesStoreKey, + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadataPartial, + ZarrV3ArrayMetadataStoreKey, + ZarrV3MetadataField, + ZarrV3NamedConfig, +) +from zarr_metadata.model._group import ( + CONSOLIDATED_METADATA_KEY_V3, + CONSOLIDATED_METADATA_STORE_KEY_V2, + GROUP_METADATA_STORE_KEY_V2, + GROUP_METADATA_STORE_KEY_V3, + ZarrV2ConsolidatedMetadata, + ZarrV2ConsolidatedMetadataStoreKey, + ZarrV2GroupMetadata, + ZarrV2GroupMetadataPartial, + ZarrV2GroupMetadataStoreKey, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, + ZarrV3GroupMetadataPartial, + ZarrV3GroupMetadataStoreKey, +) +from zarr_metadata.model._sentinel import UNSET +from zarr_metadata.model._validation import ( + ARRAY_METADATA_OPTIONAL_KEYS_V3, + ARRAY_METADATA_REQUIRED_KEYS_V2, + ARRAY_METADATA_REQUIRED_KEYS_V3, + ARRAY_METADATA_STANDARD_KEYS_V3, + GROUP_METADATA_OPTIONAL_KEYS_V3, + GROUP_METADATA_REQUIRED_KEYS_V2, + GROUP_METADATA_REQUIRED_KEYS_V3, + GROUP_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ProblemKind, + ValidationProblem, + is_array_metadata_v2, + is_array_metadata_v3, + is_group_metadata_v2, + is_group_metadata_v3, + is_json, + is_metadata_field_v3, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_group_metadata_v2, + parse_group_metadata_v3, + parse_json, + parse_metadata_field_v3, + validate_array_metadata_v2, + validate_array_metadata_v3, + validate_group_metadata_v2, + validate_group_metadata_v3, + validate_json, + validate_metadata_field_v3, +) + +__all__ = [ + "ARRAY_METADATA_OPTIONAL_KEYS_V3", + "ARRAY_METADATA_REQUIRED_KEYS_V2", + "ARRAY_METADATA_REQUIRED_KEYS_V3", + "ARRAY_METADATA_STANDARD_KEYS_V3", + "ARRAY_METADATA_STORE_KEY_V2", + "ARRAY_METADATA_STORE_KEY_V3", + "ATTRIBUTES_STORE_KEY_V2", + "CONSOLIDATED_METADATA_KEY_V3", + "CONSOLIDATED_METADATA_STORE_KEY_V2", + "GROUP_METADATA_OPTIONAL_KEYS_V3", + "GROUP_METADATA_REQUIRED_KEYS_V2", + "GROUP_METADATA_REQUIRED_KEYS_V3", + "GROUP_METADATA_STANDARD_KEYS_V3", + "GROUP_METADATA_STORE_KEY_V2", + "GROUP_METADATA_STORE_KEY_V3", + "UNSET", + "MetadataValidationError", + "ProblemKind", + "ValidationProblem", + "ZarrV2ArrayMetadata", + "ZarrV2ArrayMetadataPartial", + "ZarrV2ArrayMetadataStoreKey", + "ZarrV2AttributesStoreKey", + "ZarrV2ConsolidatedMetadata", + "ZarrV2ConsolidatedMetadataStoreKey", + "ZarrV2GroupMetadata", + "ZarrV2GroupMetadataPartial", + "ZarrV2GroupMetadataStoreKey", + "ZarrV3ArrayMetadata", + "ZarrV3ArrayMetadataPartial", + "ZarrV3ArrayMetadataStoreKey", + "ZarrV3ConsolidatedMetadata", + "ZarrV3GroupMetadata", + "ZarrV3GroupMetadataPartial", + "ZarrV3GroupMetadataStoreKey", + "ZarrV3MetadataField", + "ZarrV3NamedConfig", + "is_array_metadata_v2", + "is_array_metadata_v3", + "is_group_metadata_v2", + "is_group_metadata_v3", + "is_json", + "is_metadata_field_v3", + "parse_array_metadata_v2", + "parse_array_metadata_v3", + "parse_group_metadata_v2", + "parse_group_metadata_v3", + "parse_json", + "parse_metadata_field_v3", + "validate_array_metadata_v2", + "validate_array_metadata_v3", + "validate_group_metadata_v2", + "validate_group_metadata_v3", + "validate_json", + "validate_metadata_field_v3", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py new file mode 100644 index 0000000000..c4c967f891 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -0,0 +1,498 @@ +"""In-memory models for Zarr array metadata documents.""" + +from __future__ import annotations + +import copy +import dataclasses +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Final, Literal, TypeAlias, cast + +from typing_extensions import TypedDict, Unpack + +from zarr_metadata.model._sentinel import UNSET +from zarr_metadata.model._validation import ( + ARRAY_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ValidationProblem, + arrays_to_tuples, + dump_store_json, + load_store_json, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_metadata_field_v3, +) + +if TYPE_CHECKING: + from zarr_metadata._common import JSONValue, ZarrV3NamedConfigJSON + from zarr_metadata.v2.array import ( + ZarrV2ArrayDimensionSeparator, + ZarrV2ArrayMetadataJSON, + ZarrV2ArrayOrder, + ZarrV2DataTypeMetadata, + ) + from zarr_metadata.v2.codec import ZarrV2CodecMetadata + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON, ZarrV3ExtensionField + +ZarrV3ArrayMetadataStoreKey = Literal["zarr.json"] +ARRAY_METADATA_STORE_KEY_V3: Final[ZarrV3ArrayMetadataStoreKey] = "zarr.json" + +ZarrV2ArrayMetadataStoreKey = Literal[".zarray"] +ARRAY_METADATA_STORE_KEY_V2: Final[ZarrV2ArrayMetadataStoreKey] = ".zarray" + +ZarrV2AttributesStoreKey = Literal[".zattrs"] +ATTRIBUTES_STORE_KEY_V2: Final[ZarrV2AttributesStoreKey] = ".zattrs" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ZarrV3NamedConfig: + """A normalized v3 metadata field with its reader obligation. + + Bare names and missing configurations normalize to an empty configuration. + Bare names and missing `must_understand` members normalize to the spec's + implicit `True` value. + """ + + name: str + configuration: dict[str, JSONValue] + must_understand: bool = True + + def to_json(self) -> ZarrV3MetadataFieldJSON: + if not self.configuration and self.must_understand: + return self.name + out: ZarrV3NamedConfigJSON = {"name": self.name} + if self.configuration: + # to_json output shares no mutable state with the model. + out["configuration"] = copy.deepcopy(self.configuration) + if not self.must_understand: + out["must_understand"] = False + return out + + @classmethod + def from_json(cls, data: object) -> ZarrV3NamedConfig: + field = parse_metadata_field_v3(data) + if isinstance(field, str): + return cls(name=field, configuration={}, must_understand=True) + # Sound cast: parse_metadata_field_v3 checked the configuration is a + # string-keyed mapping of JSON values; arrays_to_tuples only converts + # lists to tuples within that shape. + configuration = cast( + "dict[str, JSONValue]", arrays_to_tuples(dict(field.get("configuration", {}))) + ) + return cls( + name=field["name"], + configuration=configuration, + must_understand=field.get("must_understand", True), + ) + + +ZarrV3MetadataField: TypeAlias = ZarrV3NamedConfig +"""The in-memory model of one field of a v3 metadata document. + +This is the role-named alias for annotation positions: model fields and +consumer signatures should say `ZarrV3MetadataField` (the logical meaning) +rather than `ZarrV3NamedConfig` (the serialized form the field currently +takes). Today every metadata field normalizes to a named configuration plus +its reader obligation, so the alias is exactly `ZarrV3NamedConfig`; if a future +spec revision adds a field form that cannot be normalized to those values, +this alias widens to a union and annotation sites do not change. Mirrors the +raw-layer split between `ZarrV3NamedConfigJSON` (shape) and +`ZarrV3MetadataFieldJSON` (field union). +""" + + +def must_understand_subset( + extra_fields: Mapping[str, ZarrV3ExtensionField], +) -> dict[str, ZarrV3ExtensionField]: + """The subset of `extra_fields` the reader is obligated to understand. + + Per the v3 spec, an extension field is implicitly `must_understand: True` + unless it explicitly says otherwise, and an implementation MUST fail to + open a group or array carrying fields it does not recognize that are not + explicitly `must_understand: false`. A non-mapping field value cannot + carry the explicit waiver, so it always requires understanding (the + runtime isinstance check defends against values looser than the declared + `ZarrV3ExtensionField`). + """ + fields = cast("Mapping[str, object]", extra_fields) + return cast( + "dict[str, ZarrV3ExtensionField]", + { + name: value + for name, value in fields.items() + if not ( + isinstance(value, Mapping) + and cast("Mapping[str, object]", value).get("must_understand") is False + ) + }, + ) + + +class ZarrV3ArrayMetadataPartial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `ZarrV3ArrayMetadata`. + + Every key is optional and typed with the model's own (not serialized) + value types, so it describes valid keyword arguments to + `ZarrV3ArrayMetadata.update`. The `init=False` fields `zarr_format` and + `node_type` are intentionally excluded, since they cannot be passed to + `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_array.py::test_partial_keys_match_settable_model_fields`. + """ + + shape: tuple[int, ...] + fill_value: JSONValue + data_type: ZarrV3MetadataField + chunk_grid: ZarrV3MetadataField + codecs: tuple[ZarrV3MetadataField, ...] + chunk_key_encoding: ZarrV3MetadataField + dimension_names: tuple[str | None, ...] | UNSET + attributes: dict[str, JSONValue] + storage_transformers: tuple[ZarrV3MetadataField, ...] + extra_fields: dict[str, ZarrV3ExtensionField] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ZarrV3ArrayMetadata: + """In-memory model of a v3 array metadata document. + + A canonical, semantically lossless representation of the `zarr.json` + content for an array. Extension points (`data_type`, `chunk_grid`, + `chunk_key_encoding`, `codecs`, `storage_transformers`) are held as + `ZarrV3MetadataField` values (currently `ZarrV3NamedConfig` name, + configuration, and obligation records) and are never interpreted; + `fill_value` is held verbatim in its JSON form. Equivalent extension + spellings normalize to shorthand strings when configuration is empty and + understanding is required. + """ + + zarr_format: Literal[3] = field(default=3, init=False) + node_type: Literal["array"] = field(default="array", init=False) + shape: tuple[int, ...] + fill_value: JSONValue + data_type: ZarrV3MetadataField + chunk_grid: ZarrV3MetadataField + codecs: tuple[ZarrV3MetadataField, ...] + chunk_key_encoding: ZarrV3MetadataField + dimension_names: tuple[str | None, ...] | UNSET + attributes: dict[str, JSONValue] + storage_transformers: tuple[ZarrV3MetadataField, ...] + extra_fields: dict[str, ZarrV3ExtensionField] + + @classmethod + def create_default(cls, **overrides: Unpack[ZarrV3ArrayMetadataPartial]) -> ZarrV3ArrayMetadata: + """ + Create a default (empty) v3 array metadata model, with optional overrides. + + The default is a structurally-valid scalar `uint8` array — the array + analog of `list()` returning `[]`. Any field can be overridden by keyword + (the same fields accepted by `update`). Overriding `shape` without + `chunk_grid` derives a consistent default grid: one regular chunk + covering the array (`chunk_shape` equal to `shape`). + + The derivation is deliberately one-way. A user-supplied `chunk_grid` + is an extension point and is taken verbatim — deriving `shape` from + it would require interpreting the grid's configuration, which this + layer never does (and cannot do for unrecognized grid names). So + overriding `chunk_grid` without `shape` keeps the scalar default + `shape=()`, and consistency between the two is the caller's + responsibility. + """ + if "shape" in overrides and "chunk_grid" not in overrides: + overrides["chunk_grid"] = ZarrV3NamedConfig( + name="regular", configuration={"chunk_shape": tuple(overrides["shape"])} + ) + default = cls( + shape=(), + fill_value=0, + data_type=ZarrV3NamedConfig(name="uint8", configuration={}), + chunk_grid=ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": ()}), + codecs=(ZarrV3NamedConfig(name="bytes", configuration={}),), + chunk_key_encoding=ZarrV3NamedConfig(name="default", configuration={}), + dimension_names=UNSET, + attributes={}, + storage_transformers=(), + extra_fields={}, + ) + return default.update(**overrides) + + def update(self, **kwargs: Unpack[ZarrV3ArrayMetadataPartial]) -> ZarrV3ArrayMetadata: + """ + Return a new `ZarrV3ArrayMetadata` with the given fields updated. + + Only the constructor-settable fields listed in + `ZarrV3ArrayMetadataPartial` can be updated; any attempt to update + other fields (including the fixed `zarr_format` / `node_type`) is + rejected at the type level. Each given field fully replaces its + previous value, including `extra_fields`. + + This is useful for test fixtures that want to override a few fields of a + base template without having to re-specify the entire document. + + No re-validation is performed (`update` is `dataclasses.replace`), so + a repair or edit can produce an invalid document; validity is checked + on `from_json`, not on field replacement. + """ + return dataclasses.replace(self, **kwargs) + + def __post_init__(self) -> None: + overlap = set(self.extra_fields.keys()).intersection(ARRAY_METADATA_STANDARD_KEYS_V3) + if overlap: + raise MetadataValidationError( + [ + ValidationProblem( + ("extra_fields",), + "Extra fields cannot overlap with standard Zarr V3 array metadata fields", + "invalid_value", + ) + ] + ) + + def to_json(self) -> ZarrV3ArrayMetadataJSON: + # to_json output shares no mutable state with the model: every value + # that can hold a mutable container is deep-copied. + out: ZarrV3ArrayMetadataJSON = { + "zarr_format": self.zarr_format, + "node_type": self.node_type, + "shape": self.shape, + "fill_value": copy.deepcopy(self.fill_value), + "data_type": self.data_type.to_json(), + "chunk_grid": self.chunk_grid.to_json(), + "codecs": tuple(codec.to_json() for codec in self.codecs), + "chunk_key_encoding": self.chunk_key_encoding.to_json(), + } + if self.dimension_names is not UNSET: + out["dimension_names"] = self.dimension_names + if len(self.attributes) > 0: + out["attributes"] = copy.deepcopy(self.attributes) + if len(self.storage_transformers) > 0: + out["storage_transformers"] = tuple( + transformer.to_json() for transformer in self.storage_transformers + ) + # Extra fields are the TypedDict's `extra_items` (PEP 728). Assign them + # by key rather than `out.update(**...)`: type checkers understand the + # indexed-write path against `extra_items`, but not the `update(**...)` + # overload. + for key, value in self.extra_fields.items(): + out[key] = copy.deepcopy(value) + return out + + @classmethod + def from_json(cls, data: object) -> ZarrV3ArrayMetadata: + parsed = parse_array_metadata_v3(arrays_to_tuples(data)) + # Sound cast: the TypedDict types all non-standard keys as its + # `extra_items` (`ZarrV3ExtensionField`); the comprehension's inferred value + # type is the union over ALL keys because the key filter cannot narrow it. + extra_fields = cast( + "dict[str, ZarrV3ExtensionField]", + {k: v for k, v in parsed.items() if k not in ARRAY_METADATA_STANDARD_KEYS_V3}, + ) + return cls( + shape=parsed["shape"], + fill_value=parsed["fill_value"], + data_type=ZarrV3NamedConfig.from_json(parsed["data_type"]), + chunk_grid=ZarrV3NamedConfig.from_json(parsed["chunk_grid"]), + codecs=tuple(ZarrV3NamedConfig.from_json(c) for c in parsed["codecs"]), + chunk_key_encoding=ZarrV3NamedConfig.from_json(parsed["chunk_key_encoding"]), + dimension_names=parsed.get("dimension_names", UNSET), + attributes=dict(parsed.get("attributes", {})), + storage_transformers=tuple( + ZarrV3NamedConfig.from_json(t) for t in parsed.get("storage_transformers", ()) + ), + extra_fields=extra_fields, + ) + + @property + def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: + """Extra fields the reader is obligated to understand. + + Everything in `extra_fields` not explicitly waived with + `must_understand: false` (the spec's implicit-true rule). A compliant + reader MUST fail to open the array if this contains any field it does + not recognize; the model layer only partitions by obligation, since + recognition is reader-specific. + """ + return must_understand_subset(self.extra_fields) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3ArrayMetadata: + return cls.from_json(load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V3)) + + def to_key_value( + self, *, indent: int | str | None = None + ) -> Mapping[ZarrV3ArrayMetadataStoreKey, bytes]: + return {ARRAY_METADATA_STORE_KEY_V3: dump_store_json(self.to_json(), indent=indent)} + + +class ZarrV2ArrayMetadataPartial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `ZarrV2ArrayMetadata`. + + Every key is optional and typed with the model's own value types, so it + describes valid keyword arguments to `ZarrV2ArrayMetadata.update` and + `create_default`. The `init=False` field `zarr_format` is intentionally + excluded, since it cannot be passed to `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_array.py::test_v2_partial_keys_match_settable_model_fields`. + """ + + shape: tuple[int, ...] + dtype: ZarrV2DataTypeMetadata + chunks: tuple[int, ...] + fill_value: JSONValue + order: ZarrV2ArrayOrder + compressor: ZarrV2CodecMetadata | None + filters: tuple[ZarrV2CodecMetadata, ...] | None + dimension_separator: ZarrV2ArrayDimensionSeparator + attributes: dict[str, JSONValue] | UNSET + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ZarrV2ArrayMetadata: + """In-memory model of a v2 array metadata document. + + A canonical, lossless representation of the `.zarray` content plus the + sibling `.zattrs` attributes. `dtype`, `compressor`, and `filters` are + held in their raw JSON forms and are never interpreted; `fill_value` is + held verbatim in its JSON form. `attributes` is `UNSET` when no + `.zattrs` file (or merged `attributes` key) exists — distinct from an + explicit empty `.zattrs`, which is `{}` and round-trips as a file. One + spelling normalization: a `.zarray` that omits `dimension_separator` + means `"."` by the v2 convention, and the model holds and re-emits that + value explicitly. + """ + + zarr_format: Literal[2] = field(default=2, init=False) + shape: tuple[int, ...] + dtype: ZarrV2DataTypeMetadata + chunks: tuple[int, ...] + fill_value: JSONValue + order: ZarrV2ArrayOrder + compressor: ZarrV2CodecMetadata | None + filters: tuple[ZarrV2CodecMetadata, ...] | None + # "." is the v2 convention's default for an ABSENT dimension_separator key; + # from_json normalizes absence to it (a semantics-preserving spelling + # normalization, like the v3 bare-string metadata-field form). The value + # is never None: the document grammar has no null spelling for this field. + dimension_separator: ZarrV2ArrayDimensionSeparator = field(default=".") + attributes: dict[str, JSONValue] | UNSET + + def update(self, **kwargs: Unpack[ZarrV2ArrayMetadataPartial]) -> ZarrV2ArrayMetadata: + """ + Return a new `ZarrV2ArrayMetadata` with the given fields updated. + + Only the constructor-settable fields listed in + `ZarrV2ArrayMetadataPartial` can be updated; the fixed `zarr_format` is + rejected at the type level. Each given field fully replaces its previous + value. + """ + return dataclasses.replace(self, **kwargs) + + @classmethod + def create_default(cls, **overrides: Unpack[ZarrV2ArrayMetadataPartial]) -> ZarrV2ArrayMetadata: + """ + Create a default (empty) v2 array metadata model, with optional overrides. + + The default is a structurally-valid scalar `uint8` (`"|u1"`) array — the + array analog of `list()` returning `[]`. Any field can be overridden by + keyword (the same fields accepted by `update`). Overriding `shape` + without `chunks` derives `chunks` equal to `shape` (one chunk covering + the array). + + The derivation is deliberately one-way, matching the v3 model: + overriding `chunks` without `shape` keeps the scalar default + `shape=()`, and consistency between the two is the caller's + responsibility. + """ + if "shape" in overrides and "chunks" not in overrides: + overrides["chunks"] = tuple(overrides["shape"]) + default = cls( + shape=(), + dtype="|u1", + chunks=(), + fill_value=0, + order="C", + compressor=None, + filters=None, + attributes=UNSET, + ) + return default.update(**overrides) + + def to_json(self) -> ZarrV2ArrayMetadataJSON: + """Return the merged in-memory document form. + + `attributes` is included when set (even empty). This is not the + on-disk `.zarray` content: a conforming `.zarray` must exclude + `attributes` (they live in the sibling `.zattrs` file). Use + `to_key_value` to produce the spec-conforming split for storage. + """ + # to_json output shares no mutable state with the model: every value + # that can hold a mutable container is deep-copied. + out: ZarrV2ArrayMetadataJSON = { + "zarr_format": self.zarr_format, + "shape": self.shape, + "dtype": self.dtype, + "order": self.order, + "chunks": self.chunks, + "fill_value": copy.deepcopy(self.fill_value), + "dimension_separator": self.dimension_separator, + "compressor": copy.deepcopy(self.compressor), + "filters": copy.deepcopy(self.filters), + } + if self.attributes is not UNSET: + out["attributes"] = copy.deepcopy(self.attributes) + return out + + @classmethod + def from_json(cls, data: object) -> ZarrV2ArrayMetadata: + parsed = parse_array_metadata_v2(arrays_to_tuples(data)) + return cls( + shape=parsed["shape"], + dtype=parsed["dtype"], + chunks=parsed["chunks"], + fill_value=parsed["fill_value"], + order=parsed["order"], + compressor=parsed["compressor"], + filters=parsed["filters"], + dimension_separator=parsed.get("dimension_separator", "."), + attributes=(dict(parsed["attributes"]) if "attributes" in parsed else UNSET), + ) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ArrayMetadata: + zarray_raw = cast("object", load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V2)) + if not isinstance(zarray_raw, Mapping): + return cls.from_json(zarray_raw) + zarray = cast("Mapping[str, object]", zarray_raw) + if "attributes" in zarray: + raise MetadataValidationError( + [ + ValidationProblem( + ("attributes",), + "unexpected document member", + "invalid_value", + ) + ] + ) + if ATTRIBUTES_STORE_KEY_V2 in mapping: + zattrs = cast("object", load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2)) + return cls.from_json({**zarray, "attributes": zattrs}) + return cls.from_json(zarray) + + def to_key_value( + self, *, indent: int | str | None = None + ) -> Mapping[ZarrV2ArrayMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]: + # Attributes live only in the sibling `.zattrs` file; the `.zarray` + # document must exclude them. The `.zattrs` key is present exactly + # when attributes are set (even empty) — UNSET emits no file. + zarray = {k: v for k, v in self.to_json().items() if k != "attributes"} + out: dict[ZarrV2ArrayMetadataStoreKey | ZarrV2AttributesStoreKey, bytes] = { + ARRAY_METADATA_STORE_KEY_V2: dump_store_json(zarray, indent=indent) + } + if self.attributes is not UNSET: + out[ATTRIBUTES_STORE_KEY_V2] = dump_store_json(self.attributes, indent=indent) + return out diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py new file mode 100644 index 0000000000..d576833c26 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -0,0 +1,442 @@ +"""In-memory models for Zarr group and consolidated metadata documents.""" + +from __future__ import annotations + +import copy +import dataclasses +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Final, Literal, cast + +from typing_extensions import TypedDict, Unpack + +from zarr_metadata.model._array import ( + ATTRIBUTES_STORE_KEY_V2, + ZarrV3ArrayMetadata, + must_understand_subset, +) +from zarr_metadata.model._sentinel import UNSET +from zarr_metadata.model._validation import ( + GROUP_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ValidationProblem, + arrays_to_tuples, + dump_store_json, + load_store_json, + parse_group_metadata_v2, + parse_group_metadata_v3, + validate_consolidated_metadata_v3, + validate_json, +) + +if TYPE_CHECKING: + from zarr_metadata._common import JSONValue + from zarr_metadata.model._array import ZarrV2AttributesStoreKey + from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON + from zarr_metadata.v3.array import ZarrV3ExtensionField + from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON + from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON + +ZarrV3GroupMetadataStoreKey = Literal["zarr.json"] +GROUP_METADATA_STORE_KEY_V3: Final[ZarrV3GroupMetadataStoreKey] = "zarr.json" + +ZarrV2GroupMetadataStoreKey = Literal[".zgroup"] +GROUP_METADATA_STORE_KEY_V2: Final[ZarrV2GroupMetadataStoreKey] = ".zgroup" + +ZarrV2ConsolidatedMetadataStoreKey = Literal[".zmetadata"] +CONSOLIDATED_METADATA_STORE_KEY_V2: Final[ZarrV2ConsolidatedMetadataStoreKey] = ".zmetadata" + +# The key under which consolidated metadata is embedded in a v3 group document. +# This is a reference-implementation convention (not a spec artifact), stored +# as an extension field on the group's `zarr.json`. +CONSOLIDATED_METADATA_KEY_V3: Final = "consolidated_metadata" + + +class ZarrV3GroupMetadataPartial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `ZarrV3GroupMetadata`. + + Every key is optional and typed with the model's own value types, so it + describes valid keyword arguments to `ZarrV3GroupMetadata.update` and + `create_default`. The `init=False` fields `zarr_format` and `node_type` + are intentionally excluded, since they cannot be passed to + `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_group.py::test_group_partial_keys_match_settable_model_fields`. + """ + + attributes: dict[str, JSONValue] + consolidated_metadata: ZarrV3ConsolidatedMetadata | UNSET + extra_fields: dict[str, ZarrV3ExtensionField] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ZarrV3GroupMetadata: + """In-memory model of a v3 group metadata document. + + A canonical, semantically lossless representation of the `zarr.json` + content for a group. The `consolidated_metadata` reference-implementation + convention is modeled as a typed field holding thin child models; every + other unknown top-level key lands in `extra_fields` verbatim. + """ + + zarr_format: Literal[3] = field(default=3, init=False) + node_type: Literal["group"] = field(default="group", init=False) + attributes: dict[str, JSONValue] + consolidated_metadata: ZarrV3ConsolidatedMetadata | UNSET + extra_fields: dict[str, ZarrV3ExtensionField] + + def __post_init__(self) -> None: + reserved = GROUP_METADATA_STANDARD_KEYS_V3 | {CONSOLIDATED_METADATA_KEY_V3} + if set(self.extra_fields.keys()).intersection(reserved): + raise MetadataValidationError( + [ + ValidationProblem( + ("extra_fields",), + "Extra fields cannot overlap with standard Zarr V3 group metadata fields", + "invalid_value", + ) + ] + ) + + @classmethod + def create_default(cls, **overrides: Unpack[ZarrV3GroupMetadataPartial]) -> ZarrV3GroupMetadata: + """ + Create a default (empty) v3 group metadata model, with optional overrides. + + The default is a structurally-valid group with no attributes — the group + analog of `list()` returning `[]`. Any field can be overridden by keyword + (the same fields accepted by `update`). + """ + default = cls(attributes={}, consolidated_metadata=UNSET, extra_fields={}) + return default.update(**overrides) + + def update(self, **kwargs: Unpack[ZarrV3GroupMetadataPartial]) -> ZarrV3GroupMetadata: + """ + Return a new `ZarrV3GroupMetadata` with the given fields updated. + + Only the constructor-settable fields listed in + `ZarrV3GroupMetadataPartial` can be updated; the fixed `zarr_format` / + `node_type` are rejected at the type level. Each given field fully + replaces its previous value, including `extra_fields`. + """ + return dataclasses.replace(self, **kwargs) + + def to_json(self) -> ZarrV3GroupMetadataJSON: + # to_json output shares no mutable state with the model: every value + # that can hold a mutable container is deep-copied. + out: ZarrV3GroupMetadataJSON = { + "zarr_format": self.zarr_format, + "node_type": self.node_type, + } + if len(self.attributes) > 0: + out["attributes"] = copy.deepcopy(self.attributes) + if self.consolidated_metadata is not UNSET: + # Consolidated metadata is a known non-core top-level JSON field. + out[CONSOLIDATED_METADATA_KEY_V3] = cast( + "ZarrV3ExtensionField", self.consolidated_metadata.to_json() + ) + for key, value in self.extra_fields.items(): + out[key] = copy.deepcopy(value) + return out + + @classmethod + def from_json(cls, data: object) -> ZarrV3GroupMetadata: + parsed = parse_group_metadata_v3(arrays_to_tuples(data)) + # Cast for narrowing across standard and arbitrary extra TypedDict items. + consolidated_raw = cast("object", parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET)) + consolidated: ZarrV3ConsolidatedMetadata | UNSET + if consolidated_raw is UNSET or consolidated_raw is None: + # consolidated_metadata: null was written by a historical + # zarr-python bug; it gets no model representation. It is read as + # absence and never written back — repaired, not preserved. + consolidated = UNSET + else: + consolidated = ZarrV3ConsolidatedMetadata.from_json(consolidated_raw) + # Sound cast: the TypedDict types all non-standard keys as its + # `extra_items` (`ZarrV3ExtensionField`); the comprehension's inferred value + # type is the union over ALL keys because the key filter cannot narrow it. + extra_fields = cast( + "dict[str, ZarrV3ExtensionField]", + { + k: v + for k, v in parsed.items() + if k not in GROUP_METADATA_STANDARD_KEYS_V3 and k != CONSOLIDATED_METADATA_KEY_V3 + }, + ) + return cls( + attributes=dict(parsed.get("attributes", {})), + consolidated_metadata=consolidated, + extra_fields=extra_fields, + ) + + @property + def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: + """Extra fields the reader is obligated to understand. + + Everything in `extra_fields` not explicitly waived with + `must_understand: false` (the spec's implicit-true rule). A compliant + reader MUST fail to open the group if this contains any field it does + not recognize; the model layer only partitions by obligation, since + recognition is reader-specific. + """ + return must_understand_subset(self.extra_fields) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3GroupMetadata: + return cls.from_json(load_store_json(mapping, GROUP_METADATA_STORE_KEY_V3)) + + def to_key_value( + self, *, indent: int | str | None = None + ) -> Mapping[ZarrV3GroupMetadataStoreKey, bytes]: + return {GROUP_METADATA_STORE_KEY_V3: dump_store_json(self.to_json(), indent=indent)} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ZarrV3ConsolidatedMetadata: + """In-memory model of v3 inline consolidated metadata. + + Models the reference-implementation convention where consolidated metadata + is embedded as an extension field on a group's `zarr.json`. Each entry in + `metadata` is a complete child document, held as a thin array or group + model. `must_understand` is typed permissively as `bool` to mirror the + document shape, but only `False` is valid; this is enforced at runtime. + """ + + kind: Literal["inline"] = field(default="inline", init=False) + must_understand: bool = False + metadata: dict[str, ZarrV3ArrayMetadata | ZarrV3GroupMetadata] + + def __post_init__(self) -> None: + if self.must_understand is not False: + raise MetadataValidationError( + [ + ValidationProblem( + ("must_understand",), + f"Invalid value for 'must_understand'. Expected False. " + f"Got {self.must_understand!r}.", + "invalid_value", + ) + ] + ) + + def to_json(self) -> ZarrV3ConsolidatedMetadataJSON: + # `must_understand` is emitted as the literal False: the field is typed + # permissively as `bool`, but `__post_init__` guarantees the value. + return { + "kind": self.kind, + "must_understand": False, + "metadata": {key: node.to_json() for key, node in self.metadata.items()}, + } + + @classmethod + def from_json(cls, data: object) -> ZarrV3ConsolidatedMetadata: + normalized = arrays_to_tuples(data) + problems = validate_consolidated_metadata_v3(normalized) + if problems: + raise MetadataValidationError(problems) + env = cast("Mapping[str, object]", normalized) + entries: dict[str, ZarrV3ArrayMetadata | ZarrV3GroupMetadata] = {} + for key, entry in cast("Mapping[str, object]", env["metadata"]).items(): + node_type = cast("Mapping[str, object]", entry).get("node_type") + if node_type == "array": + entries[key] = ZarrV3ArrayMetadata.from_json(entry) + else: + entries[key] = ZarrV3GroupMetadata.from_json(entry) + return cls(metadata=entries) + + +class ZarrV2GroupMetadataPartial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `ZarrV2GroupMetadata`. + + Every key is optional and typed with the model's own value types, so it + describes valid keyword arguments to `ZarrV2GroupMetadata.update` and + `create_default`. The `init=False` field `zarr_format` is intentionally + excluded, since it cannot be passed to `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_group.py::test_group_partial_keys_match_settable_model_fields`. + """ + + attributes: dict[str, JSONValue] | UNSET + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ZarrV2GroupMetadata: + """In-memory model of a v2 group metadata document. + + A canonical, lossless representation of the `.zgroup` content plus the + sibling `.zattrs` attributes, folded into a single in-memory value + (mirroring the merged `ZarrV2GroupMetadataJSON` document form). `attributes` is + `UNSET` when no `.zattrs` file (or merged `attributes` key) exists — + distinct from an explicit empty `.zattrs`, which is `{}` and round-trips + as a file. + """ + + zarr_format: Literal[2] = field(default=2, init=False) + attributes: dict[str, JSONValue] | UNSET + + @classmethod + def create_default(cls, **overrides: Unpack[ZarrV2GroupMetadataPartial]) -> ZarrV2GroupMetadata: + """ + Create a default (empty) v2 group metadata model, with optional overrides. + + The default is a structurally-valid group with no attributes — the group + analog of `list()` returning `[]`. Any field can be overridden by keyword + (the same fields accepted by `update`). + """ + default = cls(attributes=UNSET) + return default.update(**overrides) + + def update(self, **kwargs: Unpack[ZarrV2GroupMetadataPartial]) -> ZarrV2GroupMetadata: + """ + Return a new `ZarrV2GroupMetadata` with the given fields updated. + + Only the constructor-settable fields listed in + `ZarrV2GroupMetadataPartial` can be updated; the fixed `zarr_format` + is rejected at the type level. Each given field fully replaces its + previous value. + """ + return dataclasses.replace(self, **kwargs) + + def to_json(self) -> ZarrV2GroupMetadataJSON: + """Return the merged in-memory document form. + + `attributes` is included when set (even empty). This is not the + on-disk `.zgroup` content: a conforming `.zgroup` must exclude + `attributes` (they live in the sibling `.zattrs` file). Use + `to_key_value` to produce the spec-conforming split for storage. + """ + # to_json output shares no mutable state with the model. + out: ZarrV2GroupMetadataJSON = {"zarr_format": self.zarr_format} + if self.attributes is not UNSET: + out["attributes"] = copy.deepcopy(self.attributes) + return out + + @classmethod + def from_json(cls, data: object) -> ZarrV2GroupMetadata: + parsed = parse_group_metadata_v2(arrays_to_tuples(data)) + return cls(attributes=(dict(parsed["attributes"]) if "attributes" in parsed else UNSET)) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2GroupMetadata: + zgroup_raw = cast("object", load_store_json(mapping, GROUP_METADATA_STORE_KEY_V2)) + if not isinstance(zgroup_raw, Mapping): + return cls.from_json(zgroup_raw) + zgroup = cast("Mapping[str, object]", zgroup_raw) + if "attributes" in zgroup: + raise MetadataValidationError( + [ + ValidationProblem( + ("attributes",), + "unexpected document member", + "invalid_value", + ) + ] + ) + if ATTRIBUTES_STORE_KEY_V2 in mapping: + zattrs = cast("object", load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2)) + return cls.from_json({**zgroup, "attributes": zattrs}) + return cls.from_json(zgroup) + + def to_key_value( + self, *, indent: int | str | None = None + ) -> Mapping[ZarrV2GroupMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]: + # Attributes live only in the sibling `.zattrs` file; the `.zgroup` + # document must exclude them. The `.zattrs` key is present exactly + # when attributes are set (even empty) — UNSET emits no file. + zgroup = {k: v for k, v in self.to_json().items() if k != "attributes"} + out: dict[ZarrV2GroupMetadataStoreKey | ZarrV2AttributesStoreKey, bytes] = { + GROUP_METADATA_STORE_KEY_V2: dump_store_json(zgroup, indent=indent) + } + if self.attributes is not UNSET: + out[ATTRIBUTES_STORE_KEY_V2] = dump_store_json(self.attributes, indent=indent) + return out + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ZarrV2ConsolidatedMetadata: + """In-memory model of a v2 `.zmetadata` document. + + The `metadata` map holds the flat file-keyed entries (`"path/.zarray"`, + `"path/.zattrs"`, ...) verbatim, preserving the normalized JSON tree. + Entries are deliberately NOT merged into per-node models: which nodes had + a `.zattrs` file at all is information the canonical representation must + keep. Interpreting entries into node models is consumer work. + """ + + zarr_consolidated_format: Literal[1] = field(default=1, init=False) + metadata: dict[str, JSONValue] + + def to_json(self) -> dict[str, JSONValue]: + # to_json output shares no mutable state with the model. + return { + "zarr_consolidated_format": self.zarr_consolidated_format, + "metadata": copy.deepcopy(self.metadata), + } + + @classmethod + def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata: + normalized = arrays_to_tuples(data) + if not isinstance(normalized, Mapping): + raise MetadataValidationError( + [ValidationProblem((), "expected a mapping", "invalid_type")] + ) + doc = cast("Mapping[str, object]", normalized) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key", "missing_key") + for key in ("zarr_consolidated_format", "metadata") + if key not in doc + ] + problems.extend( + ValidationProblem((key,), "unexpected document member", "invalid_value") + for key in doc.keys() - {"zarr_consolidated_format", "metadata"} + ) + if "zarr_consolidated_format" in doc and ( + not isinstance(doc["zarr_consolidated_format"], int) + or isinstance(doc["zarr_consolidated_format"], bool) + or doc["zarr_consolidated_format"] != 1 + ): + problems.append( + ValidationProblem( + ("zarr_consolidated_format",), + f"expected 1, got {doc['zarr_consolidated_format']!r}", + "invalid_value", + ) + ) + if "metadata" in doc: + entries = doc["metadata"] + if not isinstance(entries, Mapping) or not all( + isinstance(k, str) for k in cast("Mapping[object, object]", entries) + ): + problems.append( + ValidationProblem( + ("metadata",), "expected a mapping with string keys", "invalid_type" + ) + ) + else: + for key, value in cast("Mapping[str, object]", entries).items(): + problems.extend( + ValidationProblem( + ("metadata", key, *problem.loc), problem.message, problem.kind + ) + for problem in validate_json(value) + ) + if problems: + raise MetadataValidationError(problems) + entries_tupled = cast( + "dict[str, JSONValue]", + arrays_to_tuples(dict(cast("Mapping[str, object]", doc["metadata"]))), + ) + return cls(metadata=entries_tupled) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ConsolidatedMetadata: + return cls.from_json(load_store_json(mapping, CONSOLIDATED_METADATA_STORE_KEY_V2)) + + def to_key_value( + self, *, indent: int | str | None = None + ) -> Mapping[ZarrV2ConsolidatedMetadataStoreKey, bytes]: + return {CONSOLIDATED_METADATA_STORE_KEY_V2: dump_store_json(self.to_json(), indent=indent)} diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py new file mode 100644 index 0000000000..ad71e216fa --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -0,0 +1,37 @@ +"""The absence sentinel for optional metadata-document keys. + +The models observe one invariant: `None` in a model always corresponds to a +JSON `null` in the document (a v2 `compressor`/`filters` value, an unnamed +dimension inside `dimension_names`), and `UNSET` always means the document +key is absent. The two are never interchangeable, so a model value can never +leak into a document as a spelling the writer did not intend. + +Check with identity: `if model.dimension_names is UNSET: ...`. + +Because the contract is identity, the sentinel must never be reconstructed +from state: pickling and copying work by *reference* (typing_extensions >= +4.16 implements `Sentinel.__reduce__` as a lookup of the sentinel's name on +its defining module), so `pickle.loads(pickle.dumps(UNSET)) is UNSET` holds +across process boundaries, and models holding `UNSET` pickle and deep-copy +freely. Earlier typing_extensions releases refused to pickle sentinels +outright — hence the `>=4.16` floor in this package's dependencies. + +Checker support (PEP 661 is Final; stdlib `sentinel` arrives in Python +3.15): ty types this spelling exactly, including `is`/`is not` narrowing. +Pyright supports it but a regression (1.1.405+, tracked as +https://github.com/microsoft/pyright/issues/11115) degrades class-attribute +reads to `Unknown`, so this package pins pyright to the last good version +until the fix lands. Mypy support is in review +(https://github.com/python/mypy/pull/21647); until it merges, mypy-checked +consumers of these fields need a `cast` or `type: ignore` at narrowing +sites. This is a deliberate short-term cost: the sentinel is the standard, +and the checkers are converging on it. +""" + +from __future__ import annotations + +from typing_extensions import Sentinel + +UNSET = Sentinel("UNSET") +"""Marks a metadata-document key as absent (PEP 661 sentinel; usable directly +in type expressions, e.g. `tuple[str, ...] | UNSET`). Test with `is UNSET`.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py new file mode 100644 index 0000000000..a12e1911b1 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -0,0 +1,875 @@ +"""Structural validation for Zarr metadata documents. + +Validators check JSON structure (key presence, value shapes, and fixed +literals like `zarr_format`), not domain validity. Each concept gets a +`validate_*` function returning every problem found, an `is_*` type guard, +and a `parse_*` function that narrows or raises `MetadataValidationError`. + +Every `ValidationProblem` carries a machine-readable `kind` alongside its +human-readable `message`, so consumers can dispatch on the failure mode +(`missing_key`, `invalid_type`, `invalid_value`, `invalid_json`) without +string-matching messages. +""" + +from __future__ import annotations + +import json +import math +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any, Final, Literal, NoReturn, cast + +from typing_extensions import TypeIs + +from zarr_metadata._common import JSONValue +from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON + +ProblemKind = Literal["missing_key", "invalid_type", "invalid_value", "invalid_json"] +"""Machine-readable classification of a `ValidationProblem`. + +- `missing_key`: a required key (document key or store key) is absent. +- `invalid_type`: a value has the wrong structural type (e.g. a string where + a mapping is required, a non-JSON-serializable object). +- `invalid_value`: a value has an acceptable type but an invalid content + (e.g. `zarr_format: 2` in a v3 document, `order: "Q"`). +- `invalid_json`: bytes that do not decode as JSON. +""" + + +@dataclass(frozen=True, slots=True) +class ValidationProblem: + """A single structural problem found while validating a metadata document. + + `loc` is the path from the document root to the offending value, e.g. + `("codecs", 0, "name")`. An empty `loc` refers to the document as a whole. + `kind` classifies the failure mode for programmatic dispatch; `message` + is the human-readable description. + """ + + loc: tuple[str | int, ...] + message: str + kind: ProblemKind + + def __str__(self) -> str: + location = ".".join(str(part) for part in self.loc) if self.loc else "" + return f"{location}: {self.message}" + + +class MetadataValidationError(ValueError): + """Raised when a value fails structural metadata validation. + + Carries every problem found (not just the first) in `.problems`. + """ + + def __init__(self, problems: list[ValidationProblem]) -> None: + self.problems = problems + super().__init__("\n".join(str(problem) for problem in problems)) + + +def _prefix(loc_head: str | int, problems: list[ValidationProblem]) -> list[ValidationProblem]: + """Prepend `loc_head` to the `loc` of every problem (for nested validators).""" + return [ValidationProblem((loc_head, *p.loc), p.message, p.kind) for p in problems] + + +def validate_json(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not JSON-serializable (recursively).""" + if isinstance(value, float): + if math.isfinite(value): + return [] + return [ValidationProblem((), f"non-finite float {value!r} is not JSON", "invalid_value")] + if isinstance(value, (str, int, bool)) or value is None: + return [] + problems: list[ValidationProblem] = [] + if isinstance(value, Mapping): + for key, item in cast("Mapping[object, object]", value).items(): + if not isinstance(key, str): + problems.append( + ValidationProblem((), f"non-string key {key!r} in JSON object", "invalid_type") + ) + continue + problems.extend(_prefix(key, validate_json(item))) + return problems + if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): + for index, item in enumerate(cast("Sequence[object]", value)): + problems.extend(_prefix(index, validate_json(item))) + return problems + return [ValidationProblem((), f"not a JSON-serializable value: {value!r}", "invalid_type")] + + +def _is_canonical_json(value: object) -> TypeIs[JSONValue]: + """Whether `value` already uses the concrete containers in `JSONValue`.""" + if isinstance(value, float): + return math.isfinite(value) + if isinstance(value, (str, int, bool)) or value is None: + return True + if isinstance(value, (list, tuple)): + sequence = cast("list[object] | tuple[object, ...]", value) + return all(_is_canonical_json(item) for item in sequence) + if isinstance(value, dict): + mapping = cast("dict[object, object]", value) + return all( + isinstance(key, str) and _is_canonical_json(item) for key, item in mapping.items() + ) + return False + + +def is_json(value: object) -> TypeIs[JSONValue]: + """Whether `value` is a canonical JSON structure (recursively).""" + return _is_canonical_json(value) + + +def parse_json(value: object) -> JSONValue: + """Return a canonical `JSONValue`, or raise `MetadataValidationError`.""" + normalized = arrays_to_tuples(value) + problems = validate_json(normalized) + if problems: + raise MetadataValidationError(problems) + return cast(JSONValue, normalized) + + +# The standard top-level keys of a v3 array metadata document. Anything outside +# this set is an extension field. Built from the TypedDict's required/optional +# key sets (which resolve inherited keys, unlike `__annotations__`). +ARRAY_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( + ZarrV3ArrayMetadataJSON.__required_keys__ +) +ARRAY_METADATA_OPTIONAL_KEYS_V3: Final[frozenset[str]] = frozenset( + ZarrV3ArrayMetadataJSON.__optional_keys__ +) +ARRAY_METADATA_STANDARD_KEYS_V3: Final[frozenset[str]] = ( + ARRAY_METADATA_REQUIRED_KEYS_V3 | ARRAY_METADATA_OPTIONAL_KEYS_V3 +) + +ARRAY_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( + ZarrV2ArrayMetadataJSON.__required_keys__ +) +ARRAY_METADATA_OPTIONAL_KEYS_V2: Final[frozenset[str]] = frozenset( + ZarrV2ArrayMetadataJSON.__optional_keys__ +) +ARRAY_METADATA_STANDARD_KEYS_V2: Final[frozenset[str]] = ( + ARRAY_METADATA_REQUIRED_KEYS_V2 | ARRAY_METADATA_OPTIONAL_KEYS_V2 +) + +# The standard top-level keys of a v3 group metadata document. Anything outside +# this set is an extension field. +GROUP_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( + ZarrV3GroupMetadataJSON.__required_keys__ +) +GROUP_METADATA_OPTIONAL_KEYS_V3: Final[frozenset[str]] = frozenset( + ZarrV3GroupMetadataJSON.__optional_keys__ +) +GROUP_METADATA_STANDARD_KEYS_V3: Final[frozenset[str]] = ( + GROUP_METADATA_REQUIRED_KEYS_V3 | GROUP_METADATA_OPTIONAL_KEYS_V3 +) + +GROUP_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( + ZarrV2GroupMetadataJSON.__required_keys__ +) +GROUP_METADATA_OPTIONAL_KEYS_V2: Final[frozenset[str]] = frozenset( + ZarrV2GroupMetadataJSON.__optional_keys__ +) +GROUP_METADATA_STANDARD_KEYS_V2: Final[frozenset[str]] = ( + GROUP_METADATA_REQUIRED_KEYS_V2 | GROUP_METADATA_OPTIONAL_KEYS_V2 +) + + +def _missing_keys(required: frozenset[str], doc: Mapping[str, object]) -> list[ValidationProblem]: + """One `missing_key` problem per required key absent from `doc`.""" + return [ + ValidationProblem((key,), "missing required key", "missing_key") + for key in sorted(required - doc.keys()) + ] + + +def _unexpected_keys( + allowed: frozenset[str], doc: Mapping[object, object] +) -> list[ValidationProblem]: + """One problem per member outside a closed document's declared shape.""" + problems: list[ValidationProblem] = [] + for key in doc: + if not isinstance(key, str): + problems.append( + ValidationProblem((), f"non-string document key {key!r}", "invalid_type") + ) + elif key not in allowed: + problems.append( + ValidationProblem((key,), "unexpected document member", "invalid_value") + ) + return problems + + +def _check_literal( + doc: Mapping[str, object], key: str, expected: object +) -> list[ValidationProblem]: + """One `invalid_value` problem if `doc[key]` is present but not `expected`.""" + if key in doc and (type(doc[key]) is not type(expected) or doc[key] != expected): + return [ + ValidationProblem((key,), f"expected {expected!r}, got {doc[key]!r}", "invalid_value") + ] + return [] + + +def _validate_extension_fields_v3( + doc: Mapping[object, object], + standard_keys: frozenset[str], + *, + additional_reserved_keys: frozenset[str] = frozenset(), +) -> list[ValidationProblem]: + """Validate v3 top-level key types and unknown-field JSON payloads.""" + problems: list[ValidationProblem] = [] + reserved_keys = standard_keys | additional_reserved_keys + for key, value in doc.items(): + if not isinstance(key, str): + problems.append( + ValidationProblem((), f"non-string top-level key {key!r}", "invalid_type") + ) + continue + if key in reserved_keys: + continue + problems.extend(_prefix(key, validate_json(value))) + return problems + + +def validate_metadata_field_v3( + value: object, *, allow_must_understand_false: bool = True +) -> list[ValidationProblem]: + """Return every reason `value` is not a v3 metadata field. + + A metadata field is a bare name string or a mapping containing `name` and + optional `configuration` and `must_understand` members. + """ + if isinstance(value, str): + return [] + if not isinstance(value, Mapping): + return [ + ValidationProblem( + (), + "expected a metadata field (string or extension object)", + "invalid_type", + ) + ] + field = cast("Mapping[object, object]", value) + problems: list[ValidationProblem] = [] + allowed_keys = frozenset({"name", "configuration", "must_understand"}) + for key in field: + if not isinstance(key, str): + problems.append( + ValidationProblem((), f"non-string metadata field key {key!r}", "invalid_type") + ) + elif key not in allowed_keys: + problems.append( + ValidationProblem((key,), "unexpected metadata field member", "invalid_value") + ) + if not isinstance(field.get("name"), str): + problems.append(ValidationProblem(("name",), "expected a string name", "invalid_type")) + if "configuration" in field: + configuration = field["configuration"] + if not isinstance(configuration, Mapping): + problems.append( + ValidationProblem(("configuration",), "expected a mapping", "invalid_type") + ) + elif not all(isinstance(k, str) for k in cast("Mapping[object, object]", configuration)): + problems.append( + ValidationProblem(("configuration",), "expected string keys", "invalid_type") + ) + else: + for key, item in cast("Mapping[str, object]", configuration).items(): + problems.extend(_prefix("configuration", _prefix(key, validate_json(item)))) + if "must_understand" in field: + must_understand = field["must_understand"] + if not isinstance(must_understand, bool): + problems.append( + ValidationProblem(("must_understand",), "expected a boolean", "invalid_type") + ) + elif not allow_must_understand_false and not must_understand: + problems.append( + ValidationProblem( + ("must_understand",), + "false is not supported at this extension point", + "invalid_value", + ) + ) + return problems + + +def is_metadata_field_v3(value: object) -> TypeIs[ZarrV3MetadataFieldJSON]: + """Whether `value` is a v3 metadata field: a bare name or a named config.""" + if isinstance(value, str): + return True + if not isinstance(value, dict): + return False + field = cast("dict[object, object]", value) + return _is_canonical_json(field) and not validate_metadata_field_v3(field) + + +def parse_metadata_field_v3(value: object) -> ZarrV3MetadataFieldJSON: + """Return `value` narrowed to `ZarrV3MetadataFieldJSON`, or raise `MetadataValidationError`.""" + normalized = arrays_to_tuples(value) + problems = validate_metadata_field_v3(normalized) + if problems: + raise MetadataValidationError(problems) + return cast(ZarrV3MetadataFieldJSON, normalized) + + +def _is_int_sequence(value: object) -> bool: + """Whether `value` is a non-string sequence of integers. + + JSON booleans decode to `bool`, which is an `int` subclass in Python but + is not an integer in a metadata document, so booleans are excluded. + """ + return ( + not isinstance(value, (str, bytes, bytearray)) + and isinstance(value, Sequence) + and all( + isinstance(item, int) and not isinstance(item, bool) + for item in cast("Sequence[object]", value) + ) + ) + + +def _validate_dim_sequence(doc: Mapping[str, object], key: str) -> list[ValidationProblem]: + """Validate a dimension sequence (`shape` / `chunks`) if present in `doc`. + + Dimension lengths are non-negative integers. + """ + if key not in doc: + return [] + value = doc[key] + if not _is_int_sequence(value): + return [ValidationProblem((key,), "expected a sequence of int", "invalid_type")] + if any(item < 0 for item in cast("Sequence[int]", value)): + return [ValidationProblem((key,), "expected non-negative integers", "invalid_value")] + return [] + + +def _is_dtype_v2(value: object) -> bool: + """Whether `value` is shaped like a v2 dtype: a string or field records. + + A field record is a `(name, dtype)` or `(name, dtype, shape)` sequence, + where `dtype` is itself a string or nested field records and `shape` is a + sequence of int. The string content is NOT interpreted — whether the + string names a real dtype is domain validity, not structure. + """ + if isinstance(value, str): + return True + if not isinstance(value, Sequence): + return False + for record in cast("Sequence[object]", value): + if isinstance(record, str) or not isinstance(record, Sequence): + return False + fields = cast("Sequence[object]", record) + if len(fields) not in (2, 3): + return False + if not isinstance(fields[0], str): + return False + if not _is_dtype_v2(fields[1]): + return False + if len(fields) == 3 and not _is_int_sequence(fields[2]): + return False + return True + + +def _is_canonical_dtype_v2(value: object) -> bool: + """Whether a validated v2 dtype uses the tuple-backed public representation.""" + if isinstance(value, str): + return True + if not isinstance(value, tuple): + return False + for record in cast("tuple[object, ...]", value): + if not isinstance(record, tuple): + return False + fields = cast("tuple[object, ...]", record) + if not _is_canonical_dtype_v2(fields[1]): + return False + if len(fields) == 3 and not isinstance(fields[2], tuple): + return False + return True + + +def _is_canonical_metadata_field_v3(value: object) -> bool: + """Whether a validated v3 metadata field has its declared runtime container type.""" + return isinstance(value, (str, dict)) + + +def _is_canonical_array_metadata_v3(value: object) -> bool: + """Whether a validated v3 array document matches `ZarrV3ArrayMetadataJSON` at runtime.""" + if not isinstance(value, dict): + return False + doc = cast("dict[str, object]", value) + if not isinstance(doc["shape"], tuple) or not isinstance(doc["codecs"], tuple): + return False + if "storage_transformers" in doc and not isinstance(doc["storage_transformers"], tuple): + return False + if "dimension_names" in doc and not isinstance(doc["dimension_names"], tuple): + return False + if not all( + _is_canonical_metadata_field_v3(doc[key]) + for key in ("data_type", "chunk_grid", "chunk_key_encoding") + ): + return False + if not all( + _is_canonical_metadata_field_v3(item) for item in cast("tuple[object, ...]", doc["codecs"]) + ): + return False + return "storage_transformers" not in doc or all( + _is_canonical_metadata_field_v3(item) + for item in cast("tuple[object, ...]", doc["storage_transformers"]) + ) + + +def _is_canonical_array_metadata_v2(value: object) -> bool: + """Whether a validated v2 array document matches `ZarrV2ArrayMetadataJSON` at runtime.""" + if not isinstance(value, dict): + return False + doc = cast("dict[str, object]", value) + if not isinstance(doc["shape"], tuple) or not isinstance(doc["chunks"], tuple): + return False + if not _is_canonical_dtype_v2(doc["dtype"]): + return False + compressor = doc["compressor"] + if compressor is not None and not isinstance(compressor, dict): + return False + filters = doc["filters"] + return filters is None or ( + isinstance(filters, tuple) + and all(isinstance(item, dict) for item in cast("tuple[object, ...]", filters)) + ) + + +def _is_codec_v2(value: object) -> bool: + """Whether `value` is shaped like a v2 codec config: a mapping with a string `id`.""" + return isinstance(value, Mapping) and isinstance( + cast("Mapping[object, object]", value).get("id"), str + ) + + +def _validate_codec_v2(value: object) -> list[ValidationProblem]: + """Validate a v2 codec's required shape and JSON-valued configuration.""" + if not _is_codec_v2(value): + return [ + ValidationProblem( + (), "expected a codec configuration with a string 'id'", "invalid_type" + ) + ] + return validate_json(value) + + +def _validate_attributes(value: object) -> list[ValidationProblem]: + """Validate an `attributes` value: a mapping with string keys. + + Returns a problem at `("attributes",)` if it is not, else `[]`. Shared by the + v2 and v3 validators. Unlike the other `validate_*` functions (which + return value-relative locs for the caller to `_prefix`), this emits the + already-parent-relative `("attributes",)` loc, since it is only ever called + with a document's `attributes` value. + """ + if not isinstance(value, Mapping) or not all( + isinstance(k, str) for k in cast("Mapping[object, object]", value) + ): + return [ + ValidationProblem( + ("attributes",), "expected a mapping with string keys", "invalid_type" + ) + ] + problems: list[ValidationProblem] = [] + for key, item in cast("Mapping[str, object]", value).items(): + problems.extend(_prefix("attributes", _prefix(key, validate_json(item)))) + return problems + + +def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v3 array doc. + + Checks structure, not domain validity. Unknown top-level keys are allowed + (they map to `extra_fields`). + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping", "invalid_type")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V3, doc) + problems.extend( + _validate_extension_fields_v3( + cast("Mapping[object, object]", value), ARRAY_METADATA_STANDARD_KEYS_V3 + ) + ) + problems.extend(_check_literal(doc, "zarr_format", 3)) + problems.extend(_check_literal(doc, "node_type", "array")) + problems.extend(_validate_dim_sequence(doc, "shape")) + if "fill_value" in doc: + problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) + for key in ("data_type", "chunk_grid", "chunk_key_encoding"): + if key in doc: + problems.extend( + _prefix( + key, + validate_metadata_field_v3(doc[key], allow_must_understand_false=False), + ) + ) + for key in ("codecs", "storage_transformers"): + if key in doc: + entries = doc[key] + if isinstance(entries, str) or not isinstance(entries, Sequence): + problems.append(ValidationProblem((key,), "expected a sequence", "invalid_type")) + else: + if key == "codecs" and len(cast("Sequence[object]", entries)) == 0: + problems.append( + ValidationProblem( + ("codecs",), "expected at least one codec", "invalid_value" + ) + ) + for index, entry in enumerate(cast("Sequence[object]", entries)): + problems.extend(_prefix(key, _prefix(index, validate_metadata_field_v3(entry)))) + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + if "dimension_names" in doc: + # Simple typed sequences (dimension_names, shape, chunks) report a single + # field-level loc, not per-bad-item locs; per-index locs are reserved for + # the metadata-field lists (codecs, storage_transformers). + names = doc["dimension_names"] + if isinstance(names, str) or not isinstance(names, Sequence): + problems.append( + ValidationProblem(("dimension_names",), "expected a sequence", "invalid_type") + ) + elif not all( + item is None or isinstance(item, str) for item in cast("Sequence[object]", names) + ): + problems.append( + ValidationProblem( + ("dimension_names",), "expected items of str or None", "invalid_type" + ) + ) + elif _is_int_sequence(doc.get("shape")) and len(cast("Sequence[object]", names)) != len( + cast("Sequence[int]", doc["shape"]) + ): + problems.append( + ValidationProblem( + ("dimension_names",), + "expected one name per dimension of shape", + "invalid_value", + ) + ) + return problems + + +def is_array_metadata_v3(value: object) -> TypeIs[ZarrV3ArrayMetadataJSON]: + """Whether `value` is a structurally-valid v3 array metadata document.""" + return ( + _is_canonical_json(value) + and not validate_array_metadata_v3(value) + and _is_canonical_array_metadata_v3(value) + ) + + +def parse_array_metadata_v3(value: object) -> ZarrV3ArrayMetadataJSON: + """Return `value` as `ZarrV3ArrayMetadataJSON`, or raise `MetadataValidationError`.""" + normalized = arrays_to_tuples(value) + problems = validate_array_metadata_v3(normalized) + if problems: + raise MetadataValidationError(problems) + return cast("ZarrV3ArrayMetadataJSON", normalized) + + +def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v2 array doc. + + Checks structure, not domain validity: `dtype` must be a string or field + records, but the string content is not interpreted; `compressor` and + `filters` are required keys that may be `None`, and otherwise must be + codec configurations (mappings with a string `id`). + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping", "invalid_type")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V2, doc) + problems.extend( + _unexpected_keys(ARRAY_METADATA_STANDARD_KEYS_V2, cast("Mapping[object, object]", value)) + ) + problems.extend(_check_literal(doc, "zarr_format", 2)) + shape_problems = _validate_dim_sequence(doc, "shape") + chunks_problems = _validate_dim_sequence(doc, "chunks") + problems.extend(shape_problems) + problems.extend(chunks_problems) + if ( + not shape_problems + and not chunks_problems + and _is_int_sequence(doc.get("shape")) + and _is_int_sequence(doc.get("chunks")) + ): + shape = cast("Sequence[int]", doc["shape"]) + chunks = cast("Sequence[int]", doc["chunks"]) + if len(shape) != len(chunks): + problems.append( + ValidationProblem( + ("chunks",), + "expected the same number of dimensions as shape", + "invalid_value", + ) + ) + if "dtype" in doc and not _is_dtype_v2(doc["dtype"]): + problems.append( + ValidationProblem( + ("dtype",), + "expected a v2 dtype string or a sequence of field records", + "invalid_type", + ) + ) + if "order" in doc and doc["order"] not in ("C", "F"): + problems.append( + ValidationProblem( + ("order",), f"expected 'C' or 'F', got {doc['order']!r}", "invalid_value" + ) + ) + if "compressor" in doc: + compressor = doc["compressor"] + if compressor is not None: + problems.extend(_prefix("compressor", _validate_codec_v2(compressor))) + if "filters" in doc: + filters = doc["filters"] + if filters is not None and ( + isinstance(filters, str) + or not isinstance(filters, Sequence) + or not all(_is_codec_v2(item) for item in cast("Sequence[object]", filters)) + ): + problems.append( + ValidationProblem( + ("filters",), + "expected null or a sequence of codec configurations with string 'id's", + "invalid_type", + ) + ) + elif filters is not None: + if len(cast("Sequence[object]", filters)) == 0: + problems.append( + ValidationProblem(("filters",), "expected at least one filter", "invalid_value") + ) + for index, item in enumerate(cast("Sequence[object]", filters)): + problems.extend(_prefix("filters", _prefix(index, validate_json(item)))) + if "dimension_separator" in doc and doc["dimension_separator"] not in (".", "/"): + problems.append( + ValidationProblem( + ("dimension_separator",), + f"expected '.' or '/', got {doc['dimension_separator']!r}", + "invalid_value", + ) + ) + if "fill_value" in doc: + problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + return problems + + +def is_array_metadata_v2(value: object) -> TypeIs[ZarrV2ArrayMetadataJSON]: + """Whether `value` is a structurally-valid v2 array metadata document.""" + return ( + _is_canonical_json(value) + and not validate_array_metadata_v2(value) + and _is_canonical_array_metadata_v2(value) + ) + + +def parse_array_metadata_v2(value: object) -> ZarrV2ArrayMetadataJSON: + """Return `value` as `ZarrV2ArrayMetadataJSON`, or raise `MetadataValidationError`.""" + normalized = arrays_to_tuples(value) + problems = validate_array_metadata_v2(normalized) + if problems: + raise MetadataValidationError(problems) + return cast("ZarrV2ArrayMetadataJSON", normalized) + + +def validate_consolidated_metadata_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a valid inline consolidated envelope. + + Locs are value-relative (the caller prefixes with `consolidated_metadata` + where appropriate). Entries recurse into the array and group document + validators, so a validator verdict always agrees with what + `ZarrV3ConsolidatedMetadata.from_json` accepts. + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping", "invalid_type")] + env = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key", "missing_key") + for key in ("kind", "must_understand", "metadata") + if key not in env + ] + problems.extend( + _unexpected_keys( + frozenset({"kind", "must_understand", "metadata"}), + cast("Mapping[object, object]", value), + ) + ) + problems.extend(_check_literal(env, "kind", "inline")) + if "must_understand" in env and env["must_understand"] is not False: + problems.append(ValidationProblem(("must_understand",), "expected False", "invalid_value")) + if "metadata" in env: + entries = env["metadata"] + if not isinstance(entries, Mapping): + problems.append(ValidationProblem(("metadata",), "expected a mapping", "invalid_type")) + else: + for key, entry in cast("Mapping[object, object]", entries).items(): + if not isinstance(key, str): + problems.append( + ValidationProblem(("metadata",), f"non-string key {key!r}", "invalid_type") + ) + continue + entry_obj: object = entry + node_type: object = None + if isinstance(entry, Mapping): + node_type = cast("Mapping[str, object]", entry).get("node_type") + if node_type == "array": + problems.extend( + _prefix("metadata", _prefix(key, validate_array_metadata_v3(entry_obj))) + ) + elif node_type == "group": + problems.extend( + _prefix("metadata", _prefix(key, validate_group_metadata_v3(entry_obj))) + ) + else: + problems.append( + ValidationProblem( + ("metadata", key, "node_type"), + "expected 'array' or 'group'", + "invalid_value", + ) + ) + return problems + + +def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v3 group doc. + + Checks structure, not domain validity. Unknown top-level keys are allowed + (they map to `extra_fields`); a `consolidated_metadata` key, if present, + is deep-validated (envelope and entries) via + `validate_consolidated_metadata_v3`. + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping", "invalid_type")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = _missing_keys(GROUP_METADATA_REQUIRED_KEYS_V3, doc) + problems.extend( + _validate_extension_fields_v3( + cast("Mapping[object, object]", value), + GROUP_METADATA_STANDARD_KEYS_V3, + additional_reserved_keys=frozenset({"consolidated_metadata"}), + ) + ) + problems.extend(_check_literal(doc, "zarr_format", 3)) + problems.extend(_check_literal(doc, "node_type", "group")) + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + if "consolidated_metadata" in doc and doc["consolidated_metadata"] is not None: + # consolidated_metadata: null (a historical zarr-python bug) is + # structurally accepted so those stores remain readable, but the model + # repairs it to absence on read and never writes it back. + problems.extend( + _prefix( + "consolidated_metadata", + validate_consolidated_metadata_v3(doc["consolidated_metadata"]), + ) + ) + return problems + + +def is_group_metadata_v3(value: object) -> TypeIs[ZarrV3GroupMetadataJSON]: + """Whether `value` is a structurally-valid v3 group metadata document.""" + return _is_canonical_json(value) and not validate_group_metadata_v3(value) + + +def parse_group_metadata_v3(value: object) -> ZarrV3GroupMetadataJSON: + """Return `value` narrowed to `ZarrV3GroupMetadataJSON`, or raise `MetadataValidationError`.""" + normalized = arrays_to_tuples(value) + problems = validate_group_metadata_v3(normalized) + if problems: + raise MetadataValidationError(problems) + return cast(ZarrV3GroupMetadataJSON, normalized) + + +def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v2 group doc. + + Validates the in-memory merged form: the `.zgroup` fields plus an + optional `attributes` mapping folded in from `.zattrs`. + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping", "invalid_type")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = _missing_keys(GROUP_METADATA_REQUIRED_KEYS_V2, doc) + problems.extend( + _unexpected_keys(GROUP_METADATA_STANDARD_KEYS_V2, cast("Mapping[object, object]", value)) + ) + problems.extend(_check_literal(doc, "zarr_format", 2)) + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + return problems + + +def is_group_metadata_v2(value: object) -> TypeIs[ZarrV2GroupMetadataJSON]: + """Whether `value` is a structurally-valid v2 group metadata document.""" + return _is_canonical_json(value) and not validate_group_metadata_v2(value) + + +def parse_group_metadata_v2(value: object) -> ZarrV2GroupMetadataJSON: + """Return `value` narrowed to `ZarrV2GroupMetadataJSON`, or raise `MetadataValidationError`.""" + normalized = arrays_to_tuples(value) + problems = validate_group_metadata_v2(normalized) + if problems: + raise MetadataValidationError(problems) + return cast(ZarrV2GroupMetadataJSON, normalized) + + +def _reject_json_constant(constant: str) -> NoReturn: + """Reject the JavaScript constants accepted by Python's JSON decoder.""" + raise ValueError(f"non-standard JSON constant {constant!r}") + + +def load_store_json(mapping: Mapping[str, bytes], key: str) -> Any: + """Decode the JSON document stored at `key` in `mapping`. + + Every ingestion failure surfaces as `MetadataValidationError`: a missing + store key is a `missing_key` problem and undecodable bytes are an + `invalid_json` problem, rather than leaking `KeyError` / + `json.JSONDecodeError` to callers. + """ + if key not in mapping: + raise MetadataValidationError( + [ValidationProblem((key,), "missing store key", "missing_key")] + ) + try: + return json.loads(mapping[key], parse_constant=_reject_json_constant) + except (UnicodeDecodeError, ValueError) as exc: + raise MetadataValidationError( + [ValidationProblem((key,), f"invalid JSON: {exc}", "invalid_json")] + ) from exc + + +def dump_store_json(value: object, *, indent: int | str | None = None) -> bytes: + """Encode a metadata document as strict RFC 8259 JSON bytes.""" + return json.dumps(value, indent=indent, allow_nan=False).encode("utf-8") + + +def arrays_to_tuples(obj: object) -> object: + """Recursively materialize mappings and convert array-like values to tuples.""" + if isinstance(obj, Sequence) and not isinstance(obj, (str, bytes, bytearray)): + sequence = cast("Sequence[object]", obj) + converted_sequence = tuple(arrays_to_tuples(item) for item in sequence) + if isinstance(obj, tuple) and all( + converted is original + for converted, original in zip(converted_sequence, sequence, strict=True) + ): + return cast("tuple[object, ...]", obj) + return converted_sequence + if isinstance(obj, Mapping): + mapping = cast("Mapping[object, object]", obj) + converted: dict[object, object] = { + key: arrays_to_tuples(value) for key, value in mapping.items() + } + if isinstance(obj, dict) and all(converted[key] is value for key, value in mapping.items()): + return cast("object", obj) + return converted + return obj diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py new file mode 100644 index 0000000000..8584efa570 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -0,0 +1,176 @@ +"""Optional pydantic (v2) integration: field types over the core models. + +Importing this module requires pydantic; the core package deliberately does +not depend on it, so this module is never imported by `zarr_metadata` itself. + +Each exported name is an `Annotated` field type over the corresponding core +model class — the instances ARE the core classes, so values interoperate +freely with non-pydantic code (equality, isinstance, nesting). Validation +delegates to the library: a raw document routes through `from_json` (the +single source of truth for structural validation and normalization, so +pydantic's field-level coercion can never bypass it), an existing model +instance passes through unchanged, and serialization emits the canonical +document via `to_json`. `MetadataValidationError` subclasses `ValueError`, +so a failed parse surfaces as a pydantic `ValidationError` carrying the +loc-annotated problem messages. + +Usage: + + import zarr_metadata.pydantic as zmp + + class ArrayManifest(BaseModel): + path: str + metadata: zmp.ZarrV3ArrayMetadata + +Static type checkers see each field type as its core model class, so +`manifest.metadata` is a `zarr_metadata.model.ZarrV3ArrayMetadata`. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Annotated, TypeVar + +from pydantic import BeforeValidator, InstanceOf, PlainSerializer + +from zarr_metadata import model as _model +from zarr_metadata._pydantic_schema import ( + ZarrV2ArrayMetadataJSON as _ZarrV2ArrayMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV2ConsolidatedMetadataJSON as _ZarrV2ConsolidatedMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV2GroupMetadataJSON as _ZarrV2GroupMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV3ArrayMetadataJSON as _ZarrV3ArrayMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV3ConsolidatedMetadataJSON as _ZarrV3ConsolidatedMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV3GroupMetadataJSON as _ZarrV3GroupMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV3MetadataFieldJSON as _ZarrV3MetadataFieldSchema, +) +from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON as _ZarrV2ArrayMetadataJSON +from zarr_metadata.v2.consolidated import ( + ZarrV2ConsolidatedMetadataJSON as _ZarrV2ConsolidatedMetadataJSON, +) +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON as _ZarrV2GroupMetadataJSON +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON as _ZarrV3MetadataFieldJSON +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON as _ZarrV3ArrayMetadataJSON +from zarr_metadata.v3.consolidated import ( + ZarrV3ConsolidatedMetadataJSON as _ZarrV3ConsolidatedMetadataJSON, +) +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON as _ZarrV3GroupMetadataJSON + +if TYPE_CHECKING: + from collections.abc import Callable + +_M = TypeVar("_M") + + +def _coerce_to(cls: type[_M], parse: Callable[[object], _M]) -> Callable[[object], _M]: + """A validator that passes instances of `cls` through and parses anything else.""" + + def coerce(value: object) -> _M: + if isinstance(value, cls): + return value + return parse(value) + + return coerce + + +ZarrV3ArrayMetadata = Annotated[ + InstanceOf[_model.ZarrV3ArrayMetadata], + BeforeValidator( + _coerce_to(_model.ZarrV3ArrayMetadata, _model.ZarrV3ArrayMetadata.from_json), + json_schema_input_type=_ZarrV3ArrayMetadataSchema, + ), + PlainSerializer(_model.ZarrV3ArrayMetadata.to_json, return_type=_ZarrV3ArrayMetadataJSON), +] +"""Field type for a v3 array metadata document (`zarr.json` content).""" + +ZarrV2ArrayMetadata = Annotated[ + InstanceOf[_model.ZarrV2ArrayMetadata], + BeforeValidator( + _coerce_to(_model.ZarrV2ArrayMetadata, _model.ZarrV2ArrayMetadata.from_json), + json_schema_input_type=_ZarrV2ArrayMetadataSchema, + ), + PlainSerializer(_model.ZarrV2ArrayMetadata.to_json, return_type=_ZarrV2ArrayMetadataJSON), +] +"""Field type for a v2 array metadata document (merged `.zarray` + `.zattrs` form).""" + +ZarrV3GroupMetadata = Annotated[ + InstanceOf[_model.ZarrV3GroupMetadata], + BeforeValidator( + _coerce_to(_model.ZarrV3GroupMetadata, _model.ZarrV3GroupMetadata.from_json), + json_schema_input_type=_ZarrV3GroupMetadataSchema, + ), + PlainSerializer(_model.ZarrV3GroupMetadata.to_json, return_type=_ZarrV3GroupMetadataJSON), +] +"""Field type for a v3 group metadata document (`zarr.json` content).""" + +ZarrV2GroupMetadata = Annotated[ + InstanceOf[_model.ZarrV2GroupMetadata], + BeforeValidator( + _coerce_to(_model.ZarrV2GroupMetadata, _model.ZarrV2GroupMetadata.from_json), + json_schema_input_type=_ZarrV2GroupMetadataSchema, + ), + PlainSerializer(_model.ZarrV2GroupMetadata.to_json, return_type=_ZarrV2GroupMetadataJSON), +] +"""Field type for a v2 group metadata document (merged `.zgroup` + `.zattrs` form).""" + +ZarrV3ConsolidatedMetadata = Annotated[ + InstanceOf[_model.ZarrV3ConsolidatedMetadata], + BeforeValidator( + _coerce_to( + _model.ZarrV3ConsolidatedMetadata, + _model.ZarrV3ConsolidatedMetadata.from_json, + ), + json_schema_input_type=_ZarrV3ConsolidatedMetadataSchema, + ), + PlainSerializer( + _model.ZarrV3ConsolidatedMetadata.to_json, + return_type=_ZarrV3ConsolidatedMetadataJSON, + ), +] +"""Field type for v3 inline consolidated metadata.""" + +ZarrV2ConsolidatedMetadata = Annotated[ + InstanceOf[_model.ZarrV2ConsolidatedMetadata], + BeforeValidator( + _coerce_to( + _model.ZarrV2ConsolidatedMetadata, + _model.ZarrV2ConsolidatedMetadata.from_json, + ), + json_schema_input_type=_ZarrV2ConsolidatedMetadataSchema, + ), + PlainSerializer( + _model.ZarrV2ConsolidatedMetadata.to_json, + return_type=_ZarrV2ConsolidatedMetadataJSON, + ), +] +"""Field type for a v2 `.zmetadata` document.""" + +ZarrV3MetadataField = Annotated[ + InstanceOf[_model.ZarrV3NamedConfig], + BeforeValidator( + _coerce_to(_model.ZarrV3NamedConfig, _model.ZarrV3NamedConfig.from_json), + json_schema_input_type=_ZarrV3MetadataFieldSchema, + ), + PlainSerializer(_model.ZarrV3NamedConfig.to_json, return_type=_ZarrV3MetadataFieldJSON), +] +"""Field type for one normalized v3 metadata extension envelope.""" + +__all__ = [ + "ZarrV2ArrayMetadata", + "ZarrV2ConsolidatedMetadata", + "ZarrV2GroupMetadata", + "ZarrV3ArrayMetadata", + "ZarrV3ConsolidatedMetadata", + "ZarrV3GroupMetadata", + "ZarrV3MetadataField", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py index 4e9a76125b..b9001d168e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py @@ -1,26 +1,26 @@ """Zarr v2 metadata types.""" from zarr_metadata.v2.array import ( - ArrayDimensionSeparatorV2, - ArrayMetadataV2, - ArrayOrderV2, - DataTypeMetadataV2, - ZArrayMetadata, + ZarrV2ArrayDimensionSeparator, + ZarrV2ArrayMetadataJSON, + ZarrV2ArrayOrder, + ZarrV2DataTypeMetadata, + ZarrV2ZArrayJSON, ) -from zarr_metadata.v2.attributes import ZAttrsMetadata -from zarr_metadata.v2.codec import CodecMetadataV2 -from zarr_metadata.v2.consolidated import ConsolidatedMetadataV2 -from zarr_metadata.v2.group import GroupMetadataV2, ZGroupMetadata +from zarr_metadata.v2.attributes import ZarrV2ZAttrsJSON +from zarr_metadata.v2.codec import ZarrV2CodecMetadata +from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON, ZarrV2ZGroupJSON __all__ = [ - "ArrayDimensionSeparatorV2", - "ArrayMetadataV2", - "ArrayOrderV2", - "CodecMetadataV2", - "ConsolidatedMetadataV2", - "DataTypeMetadataV2", - "GroupMetadataV2", - "ZArrayMetadata", - "ZAttrsMetadata", - "ZGroupMetadata", + "ZarrV2ArrayDimensionSeparator", + "ZarrV2ArrayMetadataJSON", + "ZarrV2ArrayOrder", + "ZarrV2CodecMetadata", + "ZarrV2ConsolidatedMetadataJSON", + "ZarrV2DataTypeMetadata", + "ZarrV2GroupMetadataJSON", + "ZarrV2ZArrayJSON", + "ZarrV2ZAttrsJSON", + "ZarrV2ZGroupJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/array.py b/packages/zarr-metadata/src/zarr_metadata/v2/array.py index 999c341dc7..84b6446bcb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/array.py @@ -3,18 +3,27 @@ from collections.abc import Mapping from typing import Final, Literal, NotRequired -from typing_extensions import TypedDict +from typing_extensions import TypeAliasType, TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v2.codec import CodecMetadataV2 - -DataTypeMetadataV2 = str | tuple[tuple[str, str] | tuple[str, str, tuple[int, ...]], ...] +from zarr_metadata.v2.codec import ZarrV2CodecMetadata + +ZarrV2DataTypeMetadata = TypeAliasType( + "ZarrV2DataTypeMetadata", + str + | tuple[ + tuple[str, "ZarrV2DataTypeMetadata"] + | tuple[str, "ZarrV2DataTypeMetadata", tuple[int, ...]], + ..., + ], +) """The v2 dtype representation. Either a numpy-style dtype string (e.g. `"/.zarray` for a v2 array. User attributes live in a sibling `.zattrs` file and are - NOT part of this type; see `ZAttrsMetadata`. + NOT part of this type; see `ZarrV2ZAttrsJSON`. See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ @@ -60,15 +69,15 @@ class ZArrayMetadata(TypedDict): zarr_format: Literal[2] shape: tuple[int, ...] chunks: tuple[int, ...] - dtype: DataTypeMetadataV2 - compressor: CodecMetadataV2 | None + dtype: ZarrV2DataTypeMetadata + compressor: ZarrV2CodecMetadata | None fill_value: JSONValue - order: ArrayOrderV2 - filters: tuple[CodecMetadataV2, ...] | None - dimension_separator: NotRequired[ArrayDimensionSeparatorV2] + order: ZarrV2ArrayOrder + filters: tuple[ZarrV2CodecMetadata, ...] | None + dimension_separator: NotRequired[ZarrV2ArrayDimensionSeparator] -class ArrayMetadataV2(TypedDict): +class ZarrV2ArrayMetadataJSON(TypedDict): """ Zarr v2 array metadata document, in-memory merged form. @@ -78,7 +87,7 @@ class ArrayMetadataV2(TypedDict): `attributes` field so a single TypedDict represents the complete in-memory state of a v2 array node. Consumers that read or write a real `.zarray` file should split / merge `attributes` accordingly, - or use `ZArrayMetadata` (strict on-disk) plus `ZAttrsMetadata` directly. + or use `ZarrV2ZArrayJSON` (strict on-disk) plus `ZarrV2ZAttrsJSON` directly. See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ @@ -86,12 +95,12 @@ class ArrayMetadataV2(TypedDict): zarr_format: Literal[2] shape: tuple[int, ...] chunks: tuple[int, ...] - dtype: DataTypeMetadataV2 - compressor: CodecMetadataV2 | None + dtype: ZarrV2DataTypeMetadata + compressor: ZarrV2CodecMetadata | None fill_value: JSONValue - order: ArrayOrderV2 - filters: tuple[CodecMetadataV2, ...] | None - dimension_separator: NotRequired[ArrayDimensionSeparatorV2] + order: ZarrV2ArrayOrder + filters: tuple[ZarrV2CodecMetadata, ...] | None + dimension_separator: NotRequired[ZarrV2ArrayDimensionSeparator] attributes: NotRequired[Mapping[str, JSONValue]] """User attributes from the sibling `.zattrs` file (not part of `.zarray`). @@ -99,11 +108,11 @@ class ArrayMetadataV2(TypedDict): """ -class ArrayMetadataV2Partial(TypedDict, total=False): +class ZarrV2ArrayMetadataJSONPartial(TypedDict, total=False): """ - Partial form of `ArrayMetadataV2`: every field is `NotRequired`. + Partial form of `ZarrV2ArrayMetadataJSON`: every field is `NotRequired`. - Field annotations mirror `ArrayMetadataV2` exactly. The only difference is + Field annotations mirror `ZarrV2ArrayMetadataJSON` exactly. The only difference is `total=False`, which makes every key optional at the type level. Use this when typing dicts that intentionally hold a subset of a complete @@ -113,26 +122,26 @@ class ArrayMetadataV2Partial(TypedDict, total=False): The `NotRequired[...]` wrappers on `dimension_separator` and `attributes` are intentional: keeping them preserves byte-identical `__annotations__` - with `ArrayMetadataV2` so the `==` check in + with `ZarrV2ArrayMetadataJSON` so the `==` check in `tests/test_partial_equivalence.py` passes without special-casing those fields (PEP 655 explicitly permits `NotRequired` inside `total=False`). Note: v2 array metadata has no `extra_items` setting (the v2 spec has no extension-field concept), so this partial inherits the same closed shape. - Drift between this type and `ArrayMetadataV2` is prevented by + Drift between this type and `ZarrV2ArrayMetadataJSON` is prevented by `tests/test_partial_equivalence.py`. """ zarr_format: Literal[2] shape: tuple[int, ...] chunks: tuple[int, ...] - dtype: DataTypeMetadataV2 - compressor: CodecMetadataV2 | None + dtype: ZarrV2DataTypeMetadata + compressor: ZarrV2CodecMetadata | None fill_value: JSONValue - order: ArrayOrderV2 - filters: tuple[CodecMetadataV2, ...] | None - dimension_separator: NotRequired[ArrayDimensionSeparatorV2] + order: ZarrV2ArrayOrder + filters: tuple[ZarrV2CodecMetadata, ...] | None + dimension_separator: NotRequired[ZarrV2ArrayDimensionSeparator] attributes: NotRequired[Mapping[str, JSONValue]] """User attributes from the sibling `.zattrs` file (not part of `.zarray`). @@ -143,10 +152,10 @@ class ArrayMetadataV2Partial(TypedDict, total=False): __all__ = [ "ARRAY_DIMENSION_SEPARATOR_V2", "ARRAY_ORDER_V2", - "ArrayDimensionSeparatorV2", - "ArrayMetadataV2", - "ArrayMetadataV2Partial", - "ArrayOrderV2", - "DataTypeMetadataV2", - "ZArrayMetadata", + "ZarrV2ArrayDimensionSeparator", + "ZarrV2ArrayMetadataJSON", + "ZarrV2ArrayMetadataJSONPartial", + "ZarrV2ArrayOrder", + "ZarrV2DataTypeMetadata", + "ZarrV2ZArrayJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py b/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py index 18b8ded9da..f7cc31babe 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py @@ -7,16 +7,16 @@ from zarr_metadata._common import JSONValue -ZAttrsMetadata = Mapping[str, JSONValue] +ZarrV2ZAttrsJSON = Mapping[str, JSONValue] """On-disk `.zattrs` file content. A JSON object holding user-defined attributes for a v2 array or group. Spec-defined keys for arrays / groups live in sibling `.zarray` / `.zgroup` -files (modeled by `ZArrayMetadata` / `ZGroupMetadata`). This type does not +files (modeled by `ZarrV2ZArrayJSON` / `ZarrV2ZGroupJSON`). This type does not constrain the keys or values of the attributes mapping. """ __all__ = [ - "ZAttrsMetadata", + "ZarrV2ZAttrsJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/codec.py b/packages/zarr-metadata/src/zarr_metadata/v2/codec.py index 6d194b7e29..69125544e6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/codec.py @@ -10,7 +10,7 @@ from zarr_metadata._common import JSONValue -class CodecMetadataV2(TypedDict, extra_items=JSONValue): # type: ignore[call-arg] +class ZarrV2CodecMetadata(TypedDict, extra_items=JSONValue): """ A numcodecs configuration dict, used as a v2 compressor or filter. @@ -25,5 +25,5 @@ class CodecMetadataV2(TypedDict, extra_items=JSONValue): # type: ignore[call-ar __all__ = [ - "CodecMetadataV2", + "ZarrV2CodecMetadata", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py index 61a5527085..6b586bb92e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py @@ -10,12 +10,12 @@ from typing_extensions import TypedDict -from zarr_metadata.v2.array import ZArrayMetadata -from zarr_metadata.v2.attributes import ZAttrsMetadata -from zarr_metadata.v2.group import ZGroupMetadata +from zarr_metadata.v2.array import ZarrV2ZArrayJSON +from zarr_metadata.v2.attributes import ZarrV2ZAttrsJSON +from zarr_metadata.v2.group import ZarrV2ZGroupJSON -class ConsolidatedMetadataV2(TypedDict): +class ZarrV2ConsolidatedMetadataJSON(TypedDict): """ `.zmetadata` file contents. @@ -24,9 +24,9 @@ class ConsolidatedMetadataV2(TypedDict): that path. The keys include the filename suffix, not just the node path; the value's shape is determined by which file the key points at: - - `/.zarray` -> `ZArrayMetadata` - - `/.zgroup` -> `ZGroupMetadata` - - `/.zattrs` -> `ZAttrsMetadata` + - `/.zarray` -> `ZarrV2ZArrayJSON` + - `/.zgroup` -> `ZarrV2ZGroupJSON` + - `/.zattrs` -> `ZarrV2ZAttrsJSON` The TypedDict cannot discriminate the value shape on the key suffix at the type level; consumers should narrow at runtime by inspecting @@ -34,9 +34,9 @@ class ConsolidatedMetadataV2(TypedDict): """ zarr_consolidated_format: int - metadata: Mapping[str, ZArrayMetadata | ZGroupMetadata | ZAttrsMetadata] + metadata: Mapping[str, ZarrV2ZArrayJSON | ZarrV2ZGroupJSON | ZarrV2ZAttrsJSON] __all__ = [ - "ConsolidatedMetadataV2", + "ZarrV2ConsolidatedMetadataJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/group.py b/packages/zarr-metadata/src/zarr_metadata/v2/group.py index 5f456fe8d3..50f2482e6f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/group.py @@ -11,14 +11,14 @@ from zarr_metadata._common import JSONValue -class ZGroupMetadata(TypedDict): +class ZarrV2ZGroupJSON(TypedDict): """ On-disk `.zgroup` file content. Strict shape of the JSON document persisted at `/.zgroup` for a v2 group. The spec defines exactly one field. User attributes live in a sibling `.zattrs` file and are NOT part of this type; see - `ZAttrsMetadata`. + `ZarrV2ZAttrsJSON`. See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ @@ -26,7 +26,7 @@ class ZGroupMetadata(TypedDict): zarr_format: Literal[2] -class GroupMetadataV2(TypedDict): +class ZarrV2GroupMetadataJSON(TypedDict): """ Zarr v2 group metadata document, in-memory merged form. @@ -34,8 +34,8 @@ class GroupMetadataV2(TypedDict): and `.zattrs` (user attributes). On disk these are persisted as two separate files; this type folds them so a single TypedDict represents the complete in-memory state of a v2 group node. Consumers that read - or write the real on-disk files should use `ZGroupMetadata` (strict - `.zgroup`) plus `ZAttrsMetadata` directly. + or write the real on-disk files should use `ZarrV2ZGroupJSON` (strict + `.zgroup`) plus `ZarrV2ZAttrsJSON` directly. See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ @@ -44,11 +44,11 @@ class GroupMetadataV2(TypedDict): attributes: NotRequired[Mapping[str, JSONValue]] -class GroupMetadataV2Partial(TypedDict, total=False): +class ZarrV2GroupMetadataJSONPartial(TypedDict, total=False): """ - Partial form of `GroupMetadataV2`: every field is `NotRequired`. + Partial form of `ZarrV2GroupMetadataJSON`: every field is `NotRequired`. - Field annotations mirror `GroupMetadataV2` exactly. The only difference is + Field annotations mirror `ZarrV2GroupMetadataJSON` exactly. The only difference is `total=False`, which makes every key optional at the type level. Use this when typing dicts that intentionally hold a subset of a complete @@ -58,7 +58,7 @@ class GroupMetadataV2Partial(TypedDict, total=False): `*Partial` types; the practical effect is that `zarr_format` becomes optional. The `NotRequired[...]` wrapper on `attributes` is intentional: keeping it - preserves byte-identical `__annotations__` with `GroupMetadataV2` so the + preserves byte-identical `__annotations__` with `ZarrV2GroupMetadataJSON` so the `==` check in `tests/test_partial_equivalence.py` passes without special-casing that field (PEP 655 explicitly permits `NotRequired` inside `total=False`). @@ -66,7 +66,7 @@ class GroupMetadataV2Partial(TypedDict, total=False): Note: v2 group metadata has no `extra_items` setting (the v2 spec has no extension-field concept), so this partial inherits the same closed shape. - Drift between this type and `GroupMetadataV2` is prevented by + Drift between this type and `ZarrV2GroupMetadataJSON` is prevented by `tests/test_partial_equivalence.py`. """ @@ -75,7 +75,7 @@ class GroupMetadataV2Partial(TypedDict, total=False): __all__ = [ - "GroupMetadataV2", - "GroupMetadataV2Partial", - "ZGroupMetadata", + "ZarrV2GroupMetadataJSON", + "ZarrV2GroupMetadataJSONPartial", + "ZarrV2ZGroupJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v3/__init__.py index c897f20d52..4e335f9573 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/__init__.py @@ -1,14 +1,14 @@ """Zarr v3 metadata types.""" -from zarr_metadata.v3._common import MetadataV3 -from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 -from zarr_metadata.v3.consolidated import ConsolidatedMetadataV3 -from zarr_metadata.v3.group import GroupMetadataV3 +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON, ZarrV3ExtensionField +from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON __all__ = [ - "ArrayMetadataV3", - "ConsolidatedMetadataV3", - "ExtensionFieldV3", - "GroupMetadataV3", - "MetadataV3", + "ZarrV3ArrayMetadataJSON", + "ZarrV3ConsolidatedMetadataJSON", + "ZarrV3ExtensionField", + "ZarrV3GroupMetadataJSON", + "ZarrV3MetadataFieldJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_common.py b/packages/zarr-metadata/src/zarr_metadata/v3/_common.py index 3424587a43..406b76b723 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_common.py @@ -2,14 +2,14 @@ This module is private (underscore-prefixed) and exists to avoid circular imports between leaf modules and sub-package `__init__.py` re-exports. -Public consumers should import `MetadataV3` from `zarr_metadata.v3`. +Public consumers should import `ZarrV3MetadataFieldJSON` from `zarr_metadata.v3`. """ -from zarr_metadata._common import NamedConfigV3 +from zarr_metadata._common import ZarrV3NamedConfigJSON -MetadataV3 = str | NamedConfigV3 +ZarrV3MetadataFieldJSON = str | ZarrV3NamedConfigJSON """The JSON shape of any v3 metadata extension-point entry: either a bare -short-hand name string or a `{name, configuration}` envelope. +short-hand name string or a `{name, configuration, must_understand}` envelope. Used for `data_type`, `chunk_grid`, `chunk_key_encoding`, individual codec entries, and `storage_transformers` in v3 array metadata, and for @@ -19,5 +19,5 @@ __all__ = [ - "MetadataV3", + "ZarrV3MetadataFieldJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/array.py b/packages/zarr-metadata/src/zarr_metadata/v3/array.py index a8b0fa3358..96341f73ca 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/array.py @@ -1,73 +1,49 @@ """Zarr v3 array metadata types.""" from collections.abc import Mapping -from typing import Literal, NotRequired +from typing import Literal, NotRequired, TypeAlias from typing_extensions import TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v3._common import MetadataV3 +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +ZarrV3ExtensionField: TypeAlias = JSONValue +"""The JSON value of an unknown top-level v3 metadata field. -class ExtensionFieldV3(TypedDict, extra_items=JSONValue): # type: ignore[call-arg] - """ - Required shape of any extension field on a v3 metadata document. - - The Zarr v3 spec permits extra keys on array and group metadata - documents, provided each value is an object with a `must_understand` - boolean key. This TypedDict captures that constraint and is used as - the `extra_items=` parameter on `ArrayMetadataV3` and `GroupMetadataV3`. - - `must_understand` is typed as `bool` rather than `Literal[False]` so - that applications which understand a particular extension can produce - or consume it with `must_understand: true` (signalling that readers - that don't recognize the extension MUST refuse to open the document). - The common case is still `false`, signalling that unknown readers may - safely ignore the field. - - Spec interpretation: this type follows the original Zarr v3.0 reading - of the spec, under which any object with a `must_understand` key is a - valid extension field. The v3.1 spec rewrite added language requiring - extension fields to also include a `name: str` key (the "Extension - definition" form). Under the strict v3.1 reading, real-world extension - fields written by zarr-python and zarrs (notably `consolidated_metadata`, - which has no `name` field) are out of spec. The community consensus at - the time of writing is that this is a regression to be reverted; this - package models the v3.0 / pre-revert interpretation. See - https://github.com/zarr-developers/zarr-specs/issues/371 for the - ongoing discussion. - """ - - must_understand: bool +An object carrying the literal member `must_understand: false` may be ignored. +Every other JSON shape implicitly requires understanding; recognition itself +belongs to the reader rather than this structural type. +""" -class ArrayMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=ZarrV3ExtensionField): """ Zarr v3 array metadata document (the `zarr.json` content for an array). - Extra keys are permitted if they conform to `ExtensionFieldV3`. + Extra keys may contain arbitrary JSON values. See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#array-metadata """ zarr_format: Literal[3] node_type: Literal["array"] - data_type: MetadataV3 + data_type: ZarrV3MetadataFieldJSON shape: tuple[int, ...] - chunk_grid: MetadataV3 - chunk_key_encoding: MetadataV3 + chunk_grid: ZarrV3MetadataFieldJSON + chunk_key_encoding: ZarrV3MetadataFieldJSON fill_value: JSONValue - codecs: tuple[MetadataV3, ...] + codecs: tuple[ZarrV3MetadataFieldJSON, ...] attributes: NotRequired[Mapping[str, JSONValue]] - storage_transformers: NotRequired[tuple[MetadataV3, ...]] + storage_transformers: NotRequired[tuple[ZarrV3MetadataFieldJSON, ...]] dimension_names: NotRequired[tuple[str | None, ...]] -class ArrayMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class ZarrV3ArrayMetadataJSONPartial(TypedDict, total=False, extra_items=ZarrV3ExtensionField): """ - Partial form of `ArrayMetadataV3`: every field is `NotRequired`. + Partial form of `ZarrV3ArrayMetadataJSON`: every field is `NotRequired`. - Field annotations and `extra_items=` mirror `ArrayMetadataV3` exactly. + Field annotations and `extra_items=` mirror `ZarrV3ArrayMetadataJSON` exactly. The only difference is `total=False`, which makes every key optional at the type level. @@ -78,29 +54,29 @@ class ArrayMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV The `NotRequired[...]` wrappers on `attributes`, `storage_transformers`, and `dimension_names` are intentional: keeping them preserves byte-identical - `__annotations__` with `ArrayMetadataV3` so the `==` check in + `__annotations__` with `ZarrV3ArrayMetadataJSON` so the `==` check in `tests/test_partial_equivalence.py` passes without special-casing those fields (PEP 655 explicitly permits `NotRequired` inside `total=False`). - Drift between this type and `ArrayMetadataV3` is prevented by + Drift between this type and `ZarrV3ArrayMetadataJSON` is prevented by `tests/test_partial_equivalence.py`. """ zarr_format: Literal[3] node_type: Literal["array"] - data_type: MetadataV3 + data_type: ZarrV3MetadataFieldJSON shape: tuple[int, ...] - chunk_grid: MetadataV3 - chunk_key_encoding: MetadataV3 + chunk_grid: ZarrV3MetadataFieldJSON + chunk_key_encoding: ZarrV3MetadataFieldJSON fill_value: JSONValue - codecs: tuple[MetadataV3, ...] + codecs: tuple[ZarrV3MetadataFieldJSON, ...] attributes: NotRequired[Mapping[str, JSONValue]] - storage_transformers: NotRequired[tuple[MetadataV3, ...]] + storage_transformers: NotRequired[tuple[ZarrV3MetadataFieldJSON, ...]] dimension_names: NotRequired[tuple[str | None, ...]] __all__ = [ - "ArrayMetadataV3", - "ArrayMetadataV3Partial", - "ExtensionFieldV3", + "ZarrV3ArrayMetadataJSON", + "ZarrV3ArrayMetadataJSONPartial", + "ZarrV3ExtensionField", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index fef5793626..e2783d296d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -4,6 +4,12 @@ Intended only to allow existing v2 arrays to be converted to v3 without having to rename chunks. Not recommended for new arrays. +Naming note: these are Zarr **v3** types. The leading `V2` in +`V2ChunkKeyEncodingMetadata` (and friends) is the encoding's registered +*entity name* (`"v2"`), not the format-version marker that `ZarrV2...` +names carry — this package's version-prefixed names always spell it +`ZarrV2` / `ZarrV3`. + See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-key-encoding """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py index b4f357117f..c8a9a150fc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py @@ -11,7 +11,7 @@ `CodecConfiguration`, etc., import directly from the leaf submodule. For the field-level "any codec entry" alias (used in array metadata's -`codecs` list and in sharding's inner pipelines), import `MetadataV3` +`codecs` list and in sharding's inner pipelines), import `ZarrV3MetadataFieldJSON` from `zarr_metadata.v3`. See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/index.html diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 7e9b071669..96c39e5916 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -9,7 +9,7 @@ from typing_extensions import TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v3._common import MetadataV3 +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON CAST_VALUE_CODEC_NAME: Final = "cast_value" """The `name` field value of the `cast_value` codec.""" @@ -71,7 +71,7 @@ class CastValueCodecConfiguration(TypedDict): bare-string primitive name or a `{name, configuration}` envelope. """ - data_type: MetadataV3 + data_type: ZarrV3MetadataFieldJSON rounding: NotRequired[CastRoundingMode] out_of_range: NotRequired[CastOutOfRangeMode] scalar_map: NotRequired[ScalarMap] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index ea35ae5f1d..6b9b46c43d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -18,7 +18,7 @@ """Literal type of the `name` field of the `crc32c` codec.""" -class Empty(TypedDict, closed=True): # type: ignore[call-arg] +class Empty(TypedDict, closed=True): """An empty mapping""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index a1488f7c30..a8c9247ec4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -8,7 +8,7 @@ from typing_extensions import TypedDict -from zarr_metadata.v3._common import MetadataV3 +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON SHARDING_INDEXED_CODEC_NAME: Final = "sharding_indexed" """The `name` field value of the `sharding_indexed` codec.""" @@ -40,8 +40,8 @@ class ShardingIndexedCodecConfiguration(TypedDict): """ chunk_shape: tuple[int, ...] - codecs: tuple[MetadataV3, ...] - index_codecs: tuple[MetadataV3, ...] + codecs: tuple[ZarrV3MetadataFieldJSON, ...] + index_codecs: tuple[ZarrV3MetadataFieldJSON, ...] index_location: NotRequired[ShardingIndexLocation] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py index 486a0897a5..bcbe675947 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py @@ -5,14 +5,10 @@ implementation (and zarrs), where consolidated metadata is embedded as an extension field on a group's `zarr.json`. -The shape modeled here (`{kind, must_understand, metadata}` with no `name` -field) reflects the original Zarr v3.0 reading of the extension-field -rules. Under the strict Zarr v3.1 reading, every extension field must -also include a `name: str` key, which would make this shape — and every -real-world consolidated metadata document in the wild — out of spec. -See `ExtensionFieldV3` and -https://github.com/zarr-developers/zarr-specs/issues/371 for the -ongoing discussion. +This is a known non-core interoperability extension. Its +`{kind, must_understand, metadata}` payload is an unknown top-level JSON value +to the core document model; implementations that recognize the convention may +interpret it through this dedicated type. """ from collections.abc import Mapping @@ -20,24 +16,24 @@ from typing_extensions import TypedDict -from zarr_metadata.v3.array import ArrayMetadataV3 -from zarr_metadata.v3.group import GroupMetadataV3 +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON -class ConsolidatedMetadataV3(TypedDict): +class ZarrV3ConsolidatedMetadataJSON(TypedDict): """ Inline consolidated metadata embedded in a v3 group. - The `metadata` map contains only v3 array and group entries - v2 - entries are excluded by design. Mixing v2 entries into a v3 - consolidated metadata document is invalid per spec. + The `metadata` map contains only v3 array and group entries. V2 entries + are excluded from this interoperability convention by design; the v3 core + specification does not define consolidated metadata. """ kind: Literal["inline"] must_understand: Literal[False] - metadata: Mapping[str, ArrayMetadataV3 | GroupMetadataV3] + metadata: Mapping[str, ZarrV3ArrayMetadataJSON | ZarrV3GroupMetadataJSON] __all__ = [ - "ConsolidatedMetadataV3", + "ZarrV3ConsolidatedMetadataJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 5291e5c309..b1b6b50308 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -10,7 +10,7 @@ from typing_extensions import ReadOnly, TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v3._common import MetadataV3 +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON STRUCT_DATA_TYPE_NAME: Final = "struct" """The `name` field value of the `struct` data type.""" @@ -33,7 +33,7 @@ class StructField(TypedDict): """ name: ReadOnly[str] - data_type: ReadOnly[MetadataV3] + data_type: ReadOnly[ZarrV3MetadataFieldJSON] class StructConfiguration(TypedDict): diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/group.py b/packages/zarr-metadata/src/zarr_metadata/v3/group.py index 27186b6059..033e91ff8c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/group.py @@ -9,14 +9,14 @@ from typing_extensions import TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v3.array import ExtensionFieldV3 +from zarr_metadata.v3.array import ZarrV3ExtensionField -class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class ZarrV3GroupMetadataJSON(TypedDict, extra_items=ZarrV3ExtensionField): """ Zarr v3 group metadata document (the `zarr.json` content for a group). - Extra keys are permitted if they conform to `ExtensionFieldV3`. + Extra keys may contain arbitrary JSON values. See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#group-metadata """ @@ -26,11 +26,11 @@ class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[ attributes: NotRequired[Mapping[str, JSONValue]] -class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class ZarrV3GroupMetadataJSONPartial(TypedDict, total=False, extra_items=ZarrV3ExtensionField): """ - Partial form of `GroupMetadataV3`: every field is `NotRequired`. + Partial form of `ZarrV3GroupMetadataJSON`: every field is `NotRequired`. - Field annotations and `extra_items=` mirror `GroupMetadataV3` exactly. + Field annotations and `extra_items=` mirror `ZarrV3GroupMetadataJSON` exactly. The only difference is `total=False`, which makes every key optional at the type level. @@ -40,12 +40,12 @@ class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV into a complete document elsewhere. The `NotRequired[...]` wrapper on `attributes` is intentional: keeping it - preserves byte-identical `__annotations__` with `GroupMetadataV3` so the + preserves byte-identical `__annotations__` with `ZarrV3GroupMetadataJSON` so the `==` check in `tests/test_partial_equivalence.py` passes without special-casing that field (PEP 655 explicitly permits `NotRequired` inside `total=False`). - Drift between this type and `GroupMetadataV3` is prevented by + Drift between this type and `ZarrV3GroupMetadataJSON` is prevented by `tests/test_partial_equivalence.py`. """ @@ -55,6 +55,6 @@ class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV __all__ = [ - "GroupMetadataV3", - "GroupMetadataV3Partial", + "ZarrV3GroupMetadataJSON", + "ZarrV3GroupMetadataJSONPartial", ] diff --git a/packages/zarr-metadata/tests/model/__init__.py b/packages/zarr-metadata/tests/model/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/zarr-metadata/tests/model/_cases.py b/packages/zarr-metadata/tests/model/_cases.py new file mode 100644 index 0000000000..15faa65539 --- /dev/null +++ b/packages/zarr-metadata/tests/model/_cases.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING, Generic, TypeVar + +import pytest + +if TYPE_CHECKING: + from contextlib import AbstractContextManager + +TIn = TypeVar("TIn") +TOut = TypeVar("TOut") + + +@dataclass(frozen=True) +class Expect(Generic[TIn, TOut]): + """A test case with explicit input, expected output, and a human-readable id.""" + + input: TIn + output: TOut + id: str + + +@dataclass(frozen=True) +class ExpectFail(Generic[TIn]): + """A test case that should raise an exception. + + `msg` is a regex matched against the exception text (pytest's native + `match=` semantics). Leave it `None` to assert only the exception type. Set + `escape=True` when `msg` is a literal that contains regex metacharacters + such as `(`, `[`, or `.`; `escape` has no effect when `msg` is `None`. + """ + + input: TIn + exception: type[Exception] + id: str + msg: str | None = None + escape: bool = False + + def raises(self) -> AbstractContextManager[pytest.ExceptionInfo[Exception]]: + if self.msg is None: + return pytest.raises(self.exception) + pattern = re.escape(self.msg) if self.escape else self.msg + return pytest.raises(self.exception, match=pattern) + + +def mutate_nested_containers(value: object) -> None: + """Recursively mutate every mutable container reachable inside `value`. + + Adds a marker key to every dict and appends a marker to every list, + descending through tuples. Used to prove a `to_json` document shares no + mutable state with the model that produced it. + """ + if isinstance(value, dict): + for item in value.values(): + mutate_nested_containers(item) + value["__mutated__"] = "__mutated__" + elif isinstance(value, list): + for item in value: + mutate_nested_containers(item) + value.append("__mutated__") + elif isinstance(value, tuple): + for item in value: + mutate_nested_containers(item) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py new file mode 100644 index 0000000000..467ef1e2ad --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -0,0 +1,1738 @@ +"""Tests for the metadata models in ``zarr_metadata.model``.""" + +import copy +import dataclasses +import json +from collections import UserDict +from collections.abc import Callable +from typing import TYPE_CHECKING, get_args + +import pytest +from typing_extensions import Unpack + +from tests.model._cases import Expect, ExpectFail, mutate_nested_containers +from zarr_metadata.model import ( + ARRAY_METADATA_OPTIONAL_KEYS_V3, + ARRAY_METADATA_REQUIRED_KEYS_V3, + ARRAY_METADATA_STANDARD_KEYS_V3, + UNSET, + MetadataValidationError, + ValidationProblem, + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadataPartial, + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadataPartial, + ZarrV3MetadataField, + ZarrV3NamedConfig, + is_array_metadata_v2, + is_array_metadata_v3, + is_json, + is_metadata_field_v3, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_json, + parse_metadata_field_v3, + validate_array_metadata_v2, + validate_array_metadata_v3, + validate_json, + validate_metadata_field_v3, +) +from zarr_metadata.model._validation import _prefix, arrays_to_tuples + +if TYPE_CHECKING: + from zarr_metadata._common import JSONValue + from zarr_metadata.v2 import ZarrV2CodecMetadata + +# --- public exports -------------------------------------------------------- + + +def test_guards_exported_from_package() -> None: + """The wire-type guard/parser functions are exported from the package.""" + import zarr_metadata.model + + for name in ( + "is_json", + "parse_json", + "is_metadata_field_v3", + "parse_metadata_field_v3", + "is_array_metadata_v3", + "parse_array_metadata_v3", + "is_array_metadata_v2", + "parse_array_metadata_v2", + ): + assert name in zarr_metadata.model.__all__ + assert hasattr(zarr_metadata.model, name) + + +def test_store_key_pairs_exported_from_package() -> None: + """Each store-key constant is exported together with its Literal type + alias, and the pair cannot drift apart.""" + import zarr_metadata.model as m + + pairs = [ + ("ARRAY_METADATA_STORE_KEY_V2", "ZarrV2ArrayMetadataStoreKey"), + ("ARRAY_METADATA_STORE_KEY_V3", "ZarrV3ArrayMetadataStoreKey"), + ("ATTRIBUTES_STORE_KEY_V2", "ZarrV2AttributesStoreKey"), + ("GROUP_METADATA_STORE_KEY_V2", "ZarrV2GroupMetadataStoreKey"), + ("GROUP_METADATA_STORE_KEY_V3", "ZarrV3GroupMetadataStoreKey"), + ("CONSOLIDATED_METADATA_STORE_KEY_V2", "ZarrV2ConsolidatedMetadataStoreKey"), + ] + for const_name, alias_name in pairs: + assert const_name in m.__all__ + assert alias_name in m.__all__ + assert (getattr(m, const_name),) == get_args(getattr(m, alias_name)) + + +def test_validation_diagnostics_exported_from_package() -> None: + """The validation-diagnostic types and validators are exported from the package.""" + import zarr_metadata.model + + for name in ( + "ValidationProblem", + "MetadataValidationError", + "validate_json", + "validate_metadata_field_v3", + "validate_array_metadata_v3", + "validate_array_metadata_v2", + ): + assert name in zarr_metadata.model.__all__ + assert hasattr(zarr_metadata.model, name) + + +def test_expect_expectfail_smoke() -> None: + """The Expect/ExpectFail test-case dataclasses behave as expected.""" + e = Expect(input=1, output=2, id="x") + assert (e.input, e.output, e.id) == (1, 2, "x") + f = ExpectFail(input=1, exception=ValueError, id="y", msg="boom") + with f.raises(): + raise ValueError("boom") + + +def test_v3_from_json_error_lists_all_problems() -> None: + """A malformed v3 document surfaces every problem via MetadataValidationError.problems.""" + doc: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) + del doc["shape"] + doc["data_type"] = 5 + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV3ArrayMetadata.from_json(doc) + locs = {p.loc for p in exc_info.value.problems} + assert ("shape",) in locs + assert ("data_type",) in locs + + +# --- JSON type / fill_value contract --------------------------------------- + + +def test_json_value_type_accepts_json_shapes() -> None: + # JSONValue is the package's public JSON type alias; assigning JSON-shaped + # values to it is valid. + """The JSONValue type alias accepts JSON-shaped values.""" + value: JSONValue = {"a": [1, 2.0, "x", True, None]} + assert value == {"a": [1, 2.0, "x", True, None]} + + +def test_string_nan_fill_value_roundtrips() -> None: + # Non-finite floats are represented as the spec strings ("NaN", "Infinity", + # "-Infinity") by the caller — the metadata layer does not interpret dtypes. + # The string form round-trips cleanly under default dataclass equality, + # unlike a raw float('nan') (which is an invalid fill_value the caller must + # not pass). + """A string 'NaN' fill_value round-trips cleanly (non-finite floats are the caller's responsibility).""" + m = ZarrV3ArrayMetadata.create_default(fill_value="NaN") + assert ZarrV3ArrayMetadata.from_json(m.to_json()) == m + assert ZarrV3ArrayMetadata.from_json(m.to_json()).fill_value == "NaN" + + +# --- ZarrV3NamedConfig.to_json ------------------------------------------------ + +ZARR_TO_JSON_CASES = [ + Expect( + ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": [1]}), + {"name": "regular", "configuration": {"chunk_shape": [1]}}, + id="with-configuration", + ), + Expect( + ZarrV3NamedConfig(name="bytes", configuration={}), + "bytes", + id="empty-configuration-shorthand", + ), +] + + +@pytest.mark.parametrize("case", ZARR_TO_JSON_CASES, ids=lambda c: c.id) +def test_zarr_metadata_v3_to_json(case: Expect[ZarrV3NamedConfig, object]) -> None: + """ZarrV3NamedConfig.to_json emits the canonical extension form.""" + assert case.input.to_json() == case.output + + +def test_zarr_metadata_v3_to_json_preserves_false_obligation() -> None: + """An empty optional extension stays an object so false is not lost.""" + model = ZarrV3NamedConfig(name="optional", configuration={}, must_understand=False) + assert model.to_json() == {"name": "optional", "must_understand": False} + + +# --- ZarrV3NamedConfig.from_json ----------------------------------------------- + +ZARR_FROM_JSON_CASES = [ + Expect("bytes", ZarrV3NamedConfig(name="bytes", configuration={}), id="bare-string"), + Expect( + {"name": "regular", "configuration": {"chunk_shape": [1]}}, + ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": (1,)}), + id="object-with-config", + ), + Expect( + {"name": "bytes"}, + ZarrV3NamedConfig(name="bytes", configuration={}), + id="object-without-config", + ), +] + + +@pytest.mark.parametrize("case", ZARR_FROM_JSON_CASES, ids=lambda c: c.id) +def test_zarr_metadata_v3_from_json(case: Expect[object, ZarrV3NamedConfig]) -> None: + """ZarrV3NamedConfig.from_json parses both the bare-string and object forms.""" + assert ZarrV3NamedConfig.from_json(case.input) == case.output + + +def test_zarr_metadata_v3_from_json_preserves_false_obligation() -> None: + """Explicit false is represented on the normalized model.""" + model = ZarrV3NamedConfig.from_json({"name": "optional", "must_understand": False}) + assert model.must_understand is False + + +# --- V3 baseline ----------------------------------------------------------- + + +def test_v3_to_json_emits_canonical_document() -> None: + """V3 to_json emits exactly the expected document (which covers every + spec-required key by construction).""" + out = ZarrV3ArrayMetadata.create_default( + shape=(10,), data_type=ZarrV3NamedConfig(name="int32", configuration={}) + ).to_json() + assert out == { + "zarr_format": 3, + "node_type": "array", + "shape": (10,), + "fill_value": 0, + "data_type": "int32", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (10,)}}, + "codecs": ("bytes",), + "chunk_key_encoding": "default", + } + + +def test_v3_dimension_names_included_when_present() -> None: + """V3 to_json includes dimension_names when they are set.""" + out: dict[str, object] = dict( + ZarrV3ArrayMetadata.create_default(dimension_names=("x",)).to_json() + ) + assert out["dimension_names"] == ("x",) + + +def test_v3_dimension_names_omitted_when_none() -> None: + """V3 to_json omits dimension_names when they are UNSET.""" + out = ZarrV3ArrayMetadata.create_default(dimension_names=UNSET).to_json() + assert "dimension_names" not in out + + +# --- BUG 1: attributes gated on dimension_names ---------------------------- + + +def test_v3_attributes_included_when_dimension_names_is_none() -> None: + """Attributes must be emitted regardless of dimension_names. + + Regression: attributes were gated on ``dimension_names is not None``, + so non-empty attributes were silently dropped when there were no + dimension names. + """ + out: dict[str, object] = dict( + ZarrV3ArrayMetadata.create_default( + dimension_names=UNSET, attributes={"foo": "bar"} + ).to_json() + ) + assert out["attributes"] == {"foo": "bar"} + + +# --- BUG 2: single storage transformer dropped ----------------------------- + + +def test_v3_single_storage_transformer_included() -> None: + """A single storage transformer must be emitted. + + Regression: the guard used ``> 1`` instead of ``> 0``, dropping a + lone storage transformer. + """ + st = ZarrV3NamedConfig(name="some_transformer", configuration={}) + out: dict[str, object] = dict( + ZarrV3ArrayMetadata.create_default(storage_transformers=(st,)).to_json() + ) + assert out["storage_transformers"] == ("some_transformer",) + + +def test_v3_no_storage_transformers_omitted() -> None: + """V3 to_json omits storage_transformers when empty.""" + out = ZarrV3ArrayMetadata.create_default(storage_transformers=()).to_json() + assert "storage_transformers" not in out + + +# --- V3 extra fields ------------------------------------------------------- + + +def test_v3_extra_fields_merged() -> None: + """V3 to_json merges extra_fields into the top-level document.""" + out = ZarrV3ArrayMetadata.create_default( + extra_fields={"my_ext": {"must_understand": False}} + ).to_json() + assert out["my_ext"] == {"must_understand": False} + + +def test_v3_extra_fields_overlapping_standard_field_rejected() -> None: + """Constructing a V3 model with an extra field that collides with a standard key is rejected.""" + with pytest.raises(ValueError): + ZarrV3ArrayMetadata.create_default(extra_fields={"shape": {"must_understand": False}}) + + +# --- V3 key/value ---------------------------------------------------------- + + +def test_v3_to_key_value_is_valid_json_under_zarr_json() -> None: + """V3 to_key_value produces valid JSON bytes under the zarr.json key.""" + kv = ZarrV3ArrayMetadata.create_default(attributes={"a": 1}).to_key_value() + assert set(kv) == {"zarr.json"} + parsed = json.loads(kv["zarr.json"].decode("utf-8")) + assert parsed["zarr_format"] == 3 + assert parsed["attributes"] == {"a": 1} + + +# --- V3 standard-key sets -------------------------------------------------- + + +def test_standard_keys_is_union_of_required_and_optional() -> None: + """The standard-key set is the union of the required and optional key sets.""" + assert ( + ARRAY_METADATA_STANDARD_KEYS_V3 + == ARRAY_METADATA_REQUIRED_KEYS_V3 | ARRAY_METADATA_OPTIONAL_KEYS_V3 + ) + + +def test_standard_keys_contains_known_fields_and_excludes_extensions() -> None: + """The standard-key set contains known fields and excludes extension keys.""" + assert { + "zarr_format", + "node_type", + "shape", + "codecs", + } <= ARRAY_METADATA_STANDARD_KEYS_V3 + assert "my_ext" not in ARRAY_METADATA_STANDARD_KEYS_V3 + + +# --- create_default -------------------------------------------------------- + + +def test_v3_create_default_is_valid_empty_array() -> None: + """V3 create_default builds a structurally valid empty array that round-trips.""" + m = ZarrV3ArrayMetadata.create_default() + assert m.shape == () + assert m.data_type == ZarrV3NamedConfig(name="uint8", configuration={}) + assert m.fill_value == 0 + assert m.attributes == {} + assert m.extra_fields == {} + # the default document is structurally valid and round-trips + assert validate_array_metadata_v3(m.to_json()) == [] + assert ZarrV3ArrayMetadata.from_json(m.to_json()) == m + + +def test_v3_create_default_applies_overrides() -> None: + """V3 create_default applies keyword overrides over the defaults.""" + m = ZarrV3ArrayMetadata.create_default(shape=(4, 4), attributes={"a": 1}) + assert m.shape == (4, 4) + assert m.attributes == {"a": 1} + # un-overridden fields keep their defaults + assert m.data_type == ZarrV3NamedConfig(name="uint8", configuration={}) + + +def test_v2_create_default_is_valid_empty_array() -> None: + """V2 create_default builds a structurally valid empty array that round-trips.""" + m = ZarrV2ArrayMetadata.create_default() + assert m.shape == () + assert m.chunks == () + assert m.fill_value == 0 + assert m.compressor is None + assert m.filters is None + assert m.attributes is UNSET + assert validate_array_metadata_v2(m.to_json()) == [] + assert ZarrV2ArrayMetadata.from_json(m.to_json()) == m + + +def test_v2_create_default_applies_overrides() -> None: + """V2 create_default applies keyword overrides over the defaults.""" + m = ZarrV2ArrayMetadata.create_default(shape=(8,), attributes={"k": "v"}) + assert m.shape == (8,) + assert m.attributes == {"k": "v"} + assert m.dtype == "|u1" # default dtype unchanged + + +# --- V3 update ------------------------------------------------------------- + +# Cluster 3: update same-shape pairs across versions — parametrized + +UPDATE_NEW_INSTANCE_PARAMS = [ + pytest.param(ZarrV3ArrayMetadata, id="v3"), + pytest.param(ZarrV2ArrayMetadata, id="v2"), +] + + +@pytest.mark.parametrize("model_cls", UPDATE_NEW_INSTANCE_PARAMS) +def test_update_returns_new_instance( + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], +) -> None: + """update returns a new instance with the field replaced, leaving the original unchanged.""" + base = model_cls.create_default(shape=(10,)) + updated = base.update(shape=(20,)) + assert updated.shape == (20,) + assert base.shape == (10,) # original unchanged + assert isinstance(updated, model_cls) + + +UPDATE_NO_ARGS_PARAMS = [ + pytest.param(ZarrV3ArrayMetadata, id="v3"), + pytest.param(ZarrV2ArrayMetadata, id="v2"), +] + + +@pytest.mark.parametrize("model_cls", UPDATE_NO_ARGS_PARAMS) +def test_update_no_args_returns_equal_model( + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], +) -> None: + """update with no arguments returns a model equal to the original.""" + base = model_cls.create_default() + assert base.update() == base + + +# V3-only update tests — kept direct (extra_fields is v3-specific) + + +def test_update_can_replace_extra_fields() -> None: + """update can replace the extra_fields mapping.""" + base = ZarrV3ArrayMetadata.create_default(extra_fields={}) + updated = base.update(extra_fields={"my_ext": {"must_understand": False}}) + assert updated.extra_fields == {"my_ext": {"must_understand": False}} + + +def test_update_replaces_extra_fields_rather_than_merging() -> None: + """update replaces extra_fields wholesale rather than merging.""" + base = ZarrV3ArrayMetadata.create_default(extra_fields={"a": {"must_understand": False}}) + updated = base.update(extra_fields={"b": {"must_understand": True}}) + assert updated.extra_fields == {"b": {"must_understand": True}} + + +def test_partial_keys_match_settable_model_fields() -> None: + """The partial TypedDict must list exactly the constructor-settable fields. + + Guards against drift: adding/removing a settable field on the model + without updating ``ZarrV3ArrayMetadataPartial`` fails here. + """ + settable = {f.name for f in dataclasses.fields(ZarrV3ArrayMetadata) if f.init} + assert set(ZarrV3ArrayMetadataPartial.__annotations__) == settable + + +# --- V2 model -------------------------------------------------------------- + + +def test_v2_partial_keys_match_settable_model_fields() -> None: + """The v2 partial TypedDict must list exactly the settable fields.""" + settable = {f.name for f in dataclasses.fields(ZarrV2ArrayMetadata) if f.init} + assert set(ZarrV2ArrayMetadataPartial.__annotations__) == settable + + +def test_v2_to_key_value_splits_zarray_and_zattrs() -> None: + """V2 to_key_value splits the document into .zarray and .zattrs.""" + kv = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_key_value() + assert set(kv) == {".zarray", ".zattrs"} + zarray = json.loads(kv[".zarray"].decode("utf-8")) + zattrs = json.loads(kv[".zattrs"].decode("utf-8")) + assert zarray["zarr_format"] == 2 + assert zattrs == {"a": 1} + + +def test_v2_zarray_excludes_attributes() -> None: + """The on-disk ``.zarray`` document must not contain user attributes. + + In v2, attributes live only in the sibling ``.zattrs`` file. The bundled + ``ZarrV2ArrayMetadataJSON`` / ``to_json()`` carry attributes for convenience, but + ``to_key_value()`` must split them out. + """ + kv = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_key_value() + zarray = json.loads(kv[".zarray"].decode("utf-8")) + assert "attributes" not in zarray + + +def test_v2_to_json_still_includes_attributes() -> None: + """``to_json()`` is the bundled in-memory form and keeps attributes.""" + out: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_json()) + assert out["attributes"] == {"a": 1} + + +# --- arrays_to_tuples helper ---------------------------------------------- + +ARRAYS_TO_TUPLES_CASES = [ + Expect([1, 2, 3], (1, 2, 3), id="top-level-list"), + Expect({"a": [1, [2, 3]], "b": "x"}, {"a": (1, (2, 3)), "b": "x"}, id="nested-in-dict"), + Expect(5, 5, id="scalar-int"), + Expect("s", "s", id="scalar-str"), + Expect(None, None, id="scalar-none"), + Expect( + {"name": "bytes", "configuration": {"nums": [1, 2]}}, + {"name": "bytes", "configuration": {"nums": (1, 2)}}, + id="dict-keys-preserved", + ), +] + + +@pytest.mark.parametrize("case", ARRAYS_TO_TUPLES_CASES, ids=lambda c: c.id) +def test_arrays_to_tuples(case: Expect[object, object]) -> None: + """arrays_to_tuples recursively converts JSON arrays to tuples.""" + assert arrays_to_tuples(case.input) == case.output + + +# --- ZarrV3ArrayMetadata.from_json ---------------------------------------- + + +def test_v3_from_json_reconstructs_required_fields() -> None: + """V3 from_json reconstructs the required fields from a document.""" + doc = ZarrV3ArrayMetadata.create_default( + shape=(7,), + attributes={"a": 1}, + data_type=ZarrV3NamedConfig(name="int32", configuration={}), + ).to_json() + model = ZarrV3ArrayMetadata.from_json(doc) + assert model.shape == (7,) + assert model.data_type == ZarrV3NamedConfig(name="int32", configuration={}) + assert model.attributes == {"a": 1} + + +def test_v3_from_json_defaults_for_omitted_optionals() -> None: + """V3 from_json supplies defaults for omitted optional fields.""" + doc = ZarrV3ArrayMetadata.create_default( + attributes={}, storage_transformers=(), dimension_names=UNSET + ).to_json() + # to_json omits these entirely; from_json must restore defaults + model = ZarrV3ArrayMetadata.from_json(doc) + assert model.attributes == {} + assert model.storage_transformers == () + assert model.dimension_names is UNSET + + +def test_v3_from_json_routes_unknown_keys_to_extra_fields() -> None: + """V3 from_json routes unknown top-level keys into extra_fields.""" + doc = ZarrV3ArrayMetadata.create_default( + extra_fields={"my_ext": {"must_understand": False}} + ).to_json() + model = ZarrV3ArrayMetadata.from_json(doc) + assert model.extra_fields == {"my_ext": {"must_understand": False}} + + +def test_v3_from_json_standard_keys_not_in_extra_fields() -> None: + """V3 from_json keeps standard keys out of extra_fields.""" + doc = ZarrV3ArrayMetadata.create_default( + shape=(10,), attributes={"a": 1}, dimension_names=("x",) + ).to_json() + model = ZarrV3ArrayMetadata.from_json(doc) + assert model.extra_fields == {} + + +def test_v3_from_json_nested_arrays_in_attributes_become_tuples() -> None: + """V3 from_json converts nested arrays in attributes into tuples.""" + doc = ZarrV3ArrayMetadata.create_default(attributes={"scale": [[1, 2], [3, 4]]}).to_json() + model = ZarrV3ArrayMetadata.from_json(doc) + assert model.attributes == {"scale": ((1, 2), (3, 4))} + + +# --- ZarrV3ArrayMetadata.from_key_value ---------------------------------- + + +def test_v3_from_key_value_parses_zarr_json() -> None: + """V3 from_key_value parses the zarr.json entry into a model.""" + kv = ZarrV3ArrayMetadata.create_default(shape=(3,)).to_key_value() + model = ZarrV3ArrayMetadata.from_key_value(kv) + assert model.shape == (3,) + + +# --- Cluster 2: from_key_value missing-key raises (parametrized) ----------- + +FROM_KEY_VALUE_MISSING_PARAMS = [ + pytest.param( + ZarrV3ArrayMetadata, + ExpectFail({}, MetadataValidationError, id="v3-missing-zarr-json", msg="missing store key"), + id="v3-missing-zarr-json", + ), + pytest.param( + ZarrV2ArrayMetadata, + ExpectFail({}, MetadataValidationError, id="v2-missing-zarray", msg="missing store key"), + id="v2-missing-zarray", + ), +] + + +@pytest.mark.parametrize(("model_cls", "case"), FROM_KEY_VALUE_MISSING_PARAMS) +def test_from_key_value_missing_key_raises( + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], + case: ExpectFail[dict[str, bytes]], +) -> None: + """from_key_value raises MetadataValidationError when the required store key is absent.""" + with case.raises(): + model_cls.from_key_value(case.input) + + +# --- Cluster 1: round-trips (model → json → model, parametrized) ----------- + +ROUNDTRIP_MODEL_JSON_PARAMS = [ + pytest.param( + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadata.create_default( + shape=(10,), + attributes={"a": 1}, + dimension_names=("x",), + storage_transformers=(ZarrV3NamedConfig(name="t", configuration={}),), + extra_fields={"ext": {"must_understand": False}}, + ), + id="v3-full", + ), + pytest.param( + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadata.create_default( + attributes={}, + dimension_names=UNSET, + storage_transformers=(), + extra_fields={}, + ), + id="v3-empty-optionals", + ), + pytest.param( + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadata.create_default(attributes={"a": 1}, filters=None, compressor=None), + id="v2-basic", + ), +] + + +@pytest.mark.parametrize(("model_cls", "model"), ROUNDTRIP_MODEL_JSON_PARAMS) +def test_roundtrip_model_json_model( + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], + model: ZarrV3ArrayMetadata | ZarrV2ArrayMetadata, +) -> None: + """A model round-trips through to_json/from_json back to an equal model.""" + assert model_cls.from_json(model.to_json()) == model + + +# --- Round-trips (model → key_value → model, parametrized) ----------------- + +ROUNDTRIP_KEY_VALUE_PARAMS = [ + pytest.param( + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadata.create_default(attributes={"a": 1}), + id="v3", + ), + pytest.param( + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadata.create_default(attributes={"a": 1}), + id="v2", + ), +] + + +@pytest.mark.parametrize(("model_cls", "model"), ROUNDTRIP_KEY_VALUE_PARAMS) +def test_roundtrip_via_key_value( + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], + model: ZarrV3ArrayMetadata | ZarrV2ArrayMetadata, +) -> None: + """A model round-trips through to_key_value/from_key_value back to an equal model.""" + assert model_cls.from_key_value(model.to_key_value()) == model + + +# --- Round-trips (json → model → json, direction distinct — kept direct) --- + + +def test_v3_roundtrip_json_model_json() -> None: + """A v3 document round-trips through from_json/to_json back to an equal document.""" + doc = ZarrV3ArrayMetadata.create_default( + shape=(10,), attributes={"a": 1}, dimension_names=("x",) + ).to_json() + assert ZarrV3ArrayMetadata.from_json(doc).to_json() == doc + + +def test_v2_roundtrip_json_model_json() -> None: + """A v2 document round-trips through from_json/to_json back to an equal document.""" + doc = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_json() + assert ZarrV2ArrayMetadata.from_json(doc).to_json() == doc + + +# --- to_json shares no mutable state with the model ------------------------ + +TO_JSON_NO_ALIASING_PARAMS = [ + pytest.param( + ZarrV3ArrayMetadata.create_default( + shape=(2,), + attributes={"a": {"b": [1]}}, + codecs=(ZarrV3NamedConfig(name="blosc", configuration={"opts": {"level": 1}}),), + extra_fields={"ext": {"must_understand": False, "cfg": {"x": [1]}}}, + ), + id="v3", + ), + pytest.param( + ZarrV2ArrayMetadata.create_default( + attributes={"a": {"b": [1]}}, + compressor={"id": "zstd", "opts": {"level": 1}}, + filters=({"id": "delta", "cfg": [1]},), + fill_value=[0, 0], + ), + id="v2", + ), +] + + +@pytest.mark.parametrize("model", TO_JSON_NO_ALIASING_PARAMS) +def test_to_json_shares_no_mutable_state_with_model( + model: ZarrV3ArrayMetadata | ZarrV2ArrayMetadata, +) -> None: + """Mutating a document returned by to_json leaves the model unchanged.""" + baseline = copy.deepcopy(model.to_json()) + mutate_nested_containers(model.to_json()) + assert model.to_json() == baseline + + +def test_v3_parser_accepts_bare_string_data_type() -> None: + """V3 from_json accepts a bare-string data_type and re-serializes it canonically.""" + doc = ZarrV3ArrayMetadata.create_default().to_json() + doc["data_type"] = "int32" + model = ZarrV3ArrayMetadata.from_json(doc) + assert model.data_type == ZarrV3NamedConfig(name="int32", configuration={}) + assert model.to_json()["data_type"] == "int32" + + +@pytest.mark.parametrize("name", ["bytes", "ANY string", "urn:example:codec"]) +def test_metadata_field_accepts_any_string_name(name: str) -> None: + """The structural layer checks the name type, not syntax or registration.""" + assert validate_metadata_field_v3({"name": name}) == [] + + +@pytest.mark.parametrize("value", [0, 1, "false", None]) +def test_metadata_field_must_understand_must_be_boolean(value: object) -> None: + """must_understand is a JSON boolean, not a truthy scalar.""" + problems = validate_metadata_field_v3({"name": "x", "must_understand": value}) + assert [(problem.loc, problem.kind) for problem in problems] == [ + (("must_understand",), "invalid_type") + ] + + +def test_metadata_field_rejects_unknown_envelope_member() -> None: + """Unknown envelope keys cannot be silently discarded during normalization.""" + problems = validate_metadata_field_v3({"name": "x", "typo": 1}) + assert [(problem.loc, problem.kind) for problem in problems] == [(("typo",), "invalid_value")] + + +@pytest.mark.parametrize("field", ["codecs", "storage_transformers"]) +def test_optional_extension_points_allow_must_understand_false(field: str) -> None: + """Codecs and storage transformers may be explicitly ignorable.""" + doc: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) + doc[field] = ({"name": "optional", "must_understand": False},) + assert validate_array_metadata_v3(doc) == [] + + +@pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) +def test_required_extension_points_reject_must_understand_false(field: str) -> None: + """Core extension points needed to locate or decode chunks cannot be ignored.""" + doc: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) + doc[field] = {"name": "optional", "must_understand": False} + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + ((field, "must_understand"), "invalid_value") + ] + + +def test_v3_codecs_cannot_be_empty() -> None: + """The core document requires at least one array-to-bytes codec.""" + doc: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) + doc["codecs"] = () + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + (("codecs",), "invalid_value") + ] + + +def test_v2_roundtrip_with_compressor_and_filters() -> None: + # Non-None compressor/filters must round-trip; extra assertion on .compressor. + """A v2 model with non-None compressor and filters round-trips.""" + compressor: ZarrV2CodecMetadata = {"id": "blosc", "clevel": 5} + filters: tuple[ZarrV2CodecMetadata, ...] = ({"id": "delta"},) + m = ZarrV2ArrayMetadata.create_default(compressor=compressor, filters=filters) + restored = ZarrV2ArrayMetadata.from_json(m.to_json()) + assert restored == m + assert restored.compressor == {"id": "blosc", "clevel": 5} + + +# --- ZarrV2ArrayMetadata.from_json ---------------------------------------- + + +def test_v2_from_json_reconstructs_fields() -> None: + """V2 from_json reconstructs the fields from a document.""" + doc = ZarrV2ArrayMetadata.create_default(shape=(4,), attributes={"a": 1}, dtype=" None: + """V2 from_json reads an absent attributes key as UNSET, distinct from an + explicit empty mapping.""" + absent = ZarrV2ArrayMetadata.from_json(ZarrV2ArrayMetadata.create_default().to_json()) + explicit = ZarrV2ArrayMetadata.from_json( + ZarrV2ArrayMetadata.create_default(attributes={}).to_json() + ) + assert absent.attributes is UNSET + assert explicit.attributes == {} + assert absent != explicit + + +# --- ZarrV2ArrayMetadata.from_key_value -------------------------------- + + +def test_v2_from_key_value_remerges_zattrs() -> None: + """V2 from_key_value re-merges .zattrs back into attributes.""" + kv = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}, shape=(10,)).to_key_value() + model = ZarrV2ArrayMetadata.from_key_value(kv) + assert model.attributes == {"a": 1} + assert model.shape == (10,) + + +@pytest.mark.parametrize("extra_key", ["attributes", "vendor_extension"]) +def test_v2_from_key_value_rejects_zarray_extra_members(extra_key: str) -> None: + """Raw `.zarray` documents reject every non-spec member.""" + doc: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default().to_json()) + doc.pop("attributes", None) + doc[extra_key] = {} + + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) + + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + ((extra_key,), "invalid_value") + ] + + +def test_v2_zattrs_presence_round_trips() -> None: + """The .zattrs file's presence is part of the store: an absent file reads + as UNSET and emits no .zattrs; an explicit empty file reads as {} and + emits .zattrs — the two stores stay distinct through a round-trip.""" + explicit_kv = dict(ZarrV2ArrayMetadata.create_default(attributes={}).to_key_value()) + assert ".zattrs" in explicit_kv + absent_kv = dict(explicit_kv) + del absent_kv[".zattrs"] + + absent = ZarrV2ArrayMetadata.from_key_value(absent_kv) + explicit = ZarrV2ArrayMetadata.from_key_value(explicit_kv) + assert absent.attributes is UNSET + assert explicit.attributes == {} + assert ".zattrs" not in absent.to_key_value() + assert ".zattrs" in explicit.to_key_value() + + +def test_v2_from_json_nested_arrays_in_attributes_become_tuples() -> None: + """V2 from_json converts nested arrays in attributes into tuples.""" + doc = ZarrV2ArrayMetadata.create_default(attributes={"axes": [[0, 1], [2, 3]]}).to_json() + model = ZarrV2ArrayMetadata.from_json(doc) + assert model.attributes == {"axes": ((0, 1), (2, 3))} + + +# --- scalar wire-type guards (is_/validate_/parse_) ------------------------ +# +# Each value is modelled once as Expect[object, frozenset[tuple[str | int, ...]]] +# where `output` is the set of expected problem locs validate_* must report — +# frozenset() means VALID. Valid iff output == frozenset(). + +JSON_VALIDATE_CASES: list[Expect[object, frozenset[tuple[str | int, ...]]]] = [ + Expect("s", frozenset(), id="str"), + Expect(1, frozenset(), id="int"), + Expect(1.5, frozenset(), id="float"), + Expect(True, frozenset(), id="bool"), + Expect(None, frozenset(), id="none"), + Expect({"a": [1, {"b": None}], "c": "x"}, frozenset(), id="nested-containers"), + Expect((1, 2, 3), frozenset(), id="tuple-array"), + Expect(float("nan"), frozenset({()}), id="nan"), + Expect(float("inf"), frozenset({()}), id="inf"), + Expect(float("-inf"), frozenset({()}), id="negative-inf"), + Expect(object(), frozenset({()}), id="object"), + Expect(b"abc", frozenset({()}), id="bytes"), + Expect(bytearray(b"abc"), frozenset({()}), id="bytearray"), + Expect({1: "x"}, frozenset({()}), id="non-str-key"), + Expect([1, object()], frozenset({(1,)}), id="non-json-list-item"), + Expect({"ok": object()}, frozenset({("ok",)}), id="non-json-value"), +] + + +@pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) +def test_is_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """is_json reports whether a value is JSON-serializable.""" + assert is_json(case.input) is (case.output == frozenset()) + + +@pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) +def test_validate_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """validate_json reports the problems (and their locs) for a value.""" + problems = validate_json(case.input) + assert (problems == []) is (case.output == frozenset()) + assert {p.loc for p in problems} >= case.output + + +@pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) +def test_parse_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """parse_json returns valid JSON values and raises on invalid ones.""" + if case.output == frozenset(): + parsed = parse_json(case.input) + assert arrays_to_tuples(parsed) == arrays_to_tuples(case.input) + else: + with pytest.raises(MetadataValidationError): + parse_json(case.input) + + +def test_parse_json_materializes_abstract_containers() -> None: + """Accepted Mapping and Sequence values normalize to JSON encoder containers.""" + value = UserDict({"values": range(3)}) + + parsed = parse_json(value) + + assert parsed == {"values": (0, 1, 2)} + assert type(parsed) is dict + assert type(parsed["values"]) is tuple + json.dumps(parsed, allow_nan=False) + + +def test_json_type_guard_rejects_abstract_sequence() -> None: + """A guard cannot narrow an abstract sequence that only the parser materializes.""" + assert not is_json(range(3)) + assert parse_json(range(3)) == (0, 1, 2) + + +def test_parse_metadata_field_materializes_abstract_containers() -> None: + """Named-config parsing produces canonical containers at every nesting level.""" + value = UserDict({"name": "example", "configuration": UserDict({"values": range(2)})}) + + parsed = parse_metadata_field_v3(value) + + assert isinstance(parsed, dict) + assert parsed == {"name": "example", "configuration": {"values": (0, 1)}} + assert type(parsed["configuration"]) is dict + + +def test_metadata_field_type_guard_rejects_abstract_mapping() -> None: + """A metadata-field guard only narrows concrete TypedDict-shaped objects.""" + value = UserDict({"name": "bytes"}) + + assert not is_metadata_field_v3(value) + assert parse_metadata_field_v3(value) == {"name": "bytes"} + + +def test_validate_json_reports_json_in_message() -> None: + """validate_json's message for a non-JSON value mentions JSON.""" + problems = validate_json(object()) + assert problems[0].loc == () + assert "JSON" in problems[0].message + + +METADATA_FIELD_VALIDATE_CASES: list[Expect[object, frozenset[tuple[str | int, ...]]]] = [ + Expect("bytes", frozenset(), id="bare-string"), + Expect({"name": "x", "configuration": {"a": 1}}, frozenset(), id="named-config"), + Expect({"name": "bytes"}, frozenset(), id="name-only"), + Expect(5, frozenset({()}), id="not-str-or-mapping"), + Expect({"configuration": {}}, frozenset({("name",)}), id="missing-name"), + Expect({"name": 3}, frozenset({("name",)}), id="non-str-name"), + Expect( + {"name": "x", "configuration": [1]}, + frozenset({("configuration",)}), + id="config-not-mapping", + ), + Expect( + {"name": "x", "configuration": {1: "y"}}, + frozenset({("configuration",)}), + id="config-non-str-key", + ), +] + + +@pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) +def test_is_metadata_field_v3(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """is_metadata_field_v3 reports whether a value is a v3 metadata field.""" + assert is_metadata_field_v3(case.input) is (case.output == frozenset()) + + +@pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) +def test_validate_metadata_field_v3( + case: Expect[object, frozenset[tuple[str | int, ...]]], +) -> None: + """validate_metadata_field_v3 reports the problems for a metadata-field value.""" + problems = validate_metadata_field_v3(case.input) + assert (problems == []) is (case.output == frozenset()) + assert {p.loc for p in problems} >= case.output + + +@pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) +def test_parse_metadata_field_v3( + case: Expect[object, frozenset[tuple[str | int, ...]]], +) -> None: + """parse_metadata_field_v3 returns valid fields and raises on invalid ones.""" + if case.output == frozenset(): + assert parse_metadata_field_v3(case.input) is case.input + else: + with pytest.raises(MetadataValidationError): + parse_metadata_field_v3(case.input) + + +# --- array-document wire-type guards (is_/validate_/parse_) ---------------- +# +# Each case starts from a valid document (built by `make`) and applies a +# mutation. `expected_locs` are loc paths `validate_*` must report for the +# invalid cases (a subset check, so accumulation of OTHER problems is allowed). + + +def _build_v3(**overrides: Unpack[ZarrV3ArrayMetadataPartial]) -> dict[str, object]: + return dict(ZarrV3ArrayMetadata.create_default(**overrides).to_json()) + + +def _build_v2(**overrides: Unpack[ZarrV2ArrayMetadataPartial]) -> dict[str, object]: + return dict(ZarrV2ArrayMetadata.create_default(**overrides).to_json()) + + +def _mutate(build: Callable[[], dict], mutate: Callable[[dict], object]) -> Callable[[], dict]: + def _factory() -> dict: + doc = build() + mutate(doc) + return doc + + return _factory + + +def _del(key: str) -> Callable[[dict], object]: + return lambda doc: doc.pop(key) + + +def _set(key: str, value: object) -> Callable[[dict], object]: + return lambda doc: doc.__setitem__(key, value) + + +V3_DOC_CASES: list[Expect[Callable[[], object], frozenset[tuple[str | int, ...]]]] = [ + Expect(_build_v3, frozenset(), id="valid"), + Expect( + lambda: _build_v3(shape=(10,), attributes={"a": 1}, dimension_names=("x",)), + frozenset(), + id="valid-with-attributes-and-dim-names", + ), + Expect( + lambda: _build_v3(extra_fields={"my_ext": {"must_understand": False}}), + frozenset(), + id="valid-with-extra-fields", + ), + Expect(_mutate(_build_v3, _del("shape")), frozenset({("shape",)}), id="missing-shape"), + Expect( + _mutate(_build_v3, _set("data_type", 5)), + frozenset({("data_type",)}), + id="bad-data-type", + ), + Expect( + _mutate(_build_v3, _set("shape", "not-a-shape")), + frozenset({("shape",)}), + id="shape-not-sequence", + ), + Expect( + _mutate(_build_v3, _set("shape", [1, "x"])), + frozenset({("shape",)}), + id="shape-non-int-item", + ), + Expect( + _mutate(_build_v3, _set("codecs", (5,))), + frozenset({("codecs", 0)}), + id="bad-codec-entry", + ), + Expect(lambda: [1, 2, 3], frozenset({()}), id="non-mapping-list"), + Expect(lambda: "nope", frozenset({()}), id="non-mapping-str"), + Expect( + _mutate(_mutate(_build_v3, _del("shape")), _set("data_type", 5)), + frozenset({("shape",), ("data_type",)}), + id="missing-shape-and-bad-data-type", + ), +] + +V2_DOC_CASES: list[Expect[Callable[[], object], frozenset[tuple[str | int, ...]]]] = [ + Expect(_build_v2, frozenset(), id="valid"), + Expect(lambda: _build_v2(attributes={"a": 1}), frozenset(), id="valid-with-attributes"), + Expect( + lambda: _build_v2(compressor=None, filters=None), + frozenset(), + id="valid-none-compressor-filters", + ), + Expect( + _mutate(_build_v2, _del("chunks")), + frozenset({("chunks",)}), + id="missing-chunks", + ), + Expect( + _mutate(_build_v2, _set("shape", [1, "x"])), + frozenset({("shape",)}), + id="bad-shape", + ), + Expect( + _mutate(_mutate(_build_v2, _del("chunks")), _set("shape", [1, "x"])), + frozenset({("chunks",), ("shape",)}), + id="missing-chunks-and-bad-shape", + ), +] + +ALL_DOC_CASES = [ + *( + pytest.param( + is_array_metadata_v3, + validate_array_metadata_v3, + parse_array_metadata_v3, + c, + id=f"v3-{c.id}", + ) + for c in V3_DOC_CASES + ), + *( + pytest.param( + is_array_metadata_v2, + validate_array_metadata_v2, + parse_array_metadata_v2, + c, + id=f"v2-{c.id}", + ) + for c in V2_DOC_CASES + ), +] + + +@pytest.mark.parametrize(("is_fn", "validate_fn", "parse_fn", "case"), ALL_DOC_CASES) +def test_array_metadata_guards( + is_fn: Callable[[object], bool], + validate_fn: Callable[[object], list[ValidationProblem]], + parse_fn: Callable[[object], object], + case: Expect[Callable[[], object], frozenset[tuple[str | int, ...]]], +) -> None: + """is_/validate_/parse_ array-metadata guards agree on validity and locs for each case.""" + doc = case.input() + valid = case.output == frozenset() + assert is_fn(doc) is valid + problems = validate_fn(doc) + assert (problems == []) is valid + assert {p.loc for p in problems} >= case.output + if valid: + assert parse_fn(doc) is doc + else: + with pytest.raises(MetadataValidationError): + parse_fn(doc) + + +# --- strict from_json validation ------------------------------------------- + + +FROM_JSON_REJECT_PARAMS = [ + pytest.param( + ZarrV3ArrayMetadata, + ExpectFail(lambda: {"zarr_format": 3}, MetadataValidationError, id="x"), + id="v3-missing-required", + ), + pytest.param( + ZarrV3ArrayMetadata, + ExpectFail(_mutate(_build_v3, _set("data_type", 5)), MetadataValidationError, id="x"), + id="v3-bad-field-type", + ), + pytest.param( + ZarrV2ArrayMetadata, + ExpectFail(lambda: {"zarr_format": 2}, MetadataValidationError, id="x"), + id="v2-missing-required", + ), + pytest.param( + ZarrV3NamedConfig, + ExpectFail(lambda: 5, MetadataValidationError, id="x"), + id="zarr-metadata-bad-input", + ), +] + + +@pytest.mark.parametrize(("model", "case"), FROM_JSON_REJECT_PARAMS) +def test_from_json_rejects_malformed( + model: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata | ZarrV3NamedConfig], + case: ExpectFail[Callable[[], object]], +) -> None: + """from_json raises MetadataValidationError on a malformed document.""" + with case.raises(): + model.from_json(case.input()) + + +# --- ValidationProblem / MetadataValidationError / _prefix ----------------- +# Small structural tests — not "parametrize over inputs" shaped, kept direct. + + +def test_validation_problem_str_with_loc() -> None: + """ValidationProblem.__str__ renders a non-empty loc as a dotted path.""" + p = ValidationProblem(loc=("codecs", 0, "name"), message="expected str", kind="invalid_type") + assert str(p) == "codecs.0.name: expected str" + + +def test_validation_problem_str_empty_loc() -> None: + """ValidationProblem.__str__ renders an empty loc as .""" + p = ValidationProblem(loc=(), message="not a mapping", kind="invalid_type") + assert str(p) == ": not a mapping" + + +def test_validation_problem_is_frozen() -> None: + """ValidationProblem is immutable (frozen dataclass).""" + p = ValidationProblem(loc=("shape",), message="x", kind="invalid_type") + with pytest.raises(dataclasses.FrozenInstanceError): + # setattr: assigning to a frozen field is an intentional runtime error, + # spelled dynamically so it is not also a static type error. + setattr(p, "message", "y") # noqa: B010 + + +def test_metadata_validation_error_holds_problems() -> None: + """MetadataValidationError carries its problem list and renders them in its message.""" + problems = [ + ValidationProblem(loc=("shape",), message="missing required key", kind="missing_key"), + ValidationProblem( + loc=("data_type",), message="expected a metadata field", kind="invalid_type" + ), + ] + err = MetadataValidationError(problems) + assert err.problems == problems + assert "shape: missing required key" in str(err) + assert "data_type: expected a metadata field" in str(err) + + +def test_prefix_prepends_loc_head() -> None: + """_prefix prepends a loc head to each problem's loc.""" + problems = [ValidationProblem(loc=("name",), message="expected str", kind="invalid_type")] + prefixed = _prefix(0, problems) + assert prefixed == [ + ValidationProblem(loc=(0, "name"), message="expected str", kind="invalid_type") + ] + + +# --- Stricter v2/v3 field validation and error kinds ------------------------- + + +def test_v2_dtype_must_be_string_or_records() -> None: + """A non-string, non-records v2 dtype is rejected with an invalid_type problem.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dtype": 42} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dtype",), "invalid_type")] + + +def test_v2_structured_dtype_records_accepted() -> None: + """A structured v2 dtype (field records, optionally nested/shaped) validates.""" + dtype = (("a", " None: + """A field record with the wrong arity is rejected.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dtype": (("a",),)} + problems = validate_array_metadata_v2(doc) + assert [p.loc for p in problems] == [("dtype",)] + + +def test_v2_order_literal_enforced() -> None: + """An order other than 'C' or 'F' is rejected with an invalid_value problem.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"order": "Q"} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("order",), "invalid_value")] + + +def test_v2_compressor_must_be_codec_or_none() -> None: + """A compressor that is not null or a codec config mapping is rejected.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"compressor": "zlib"} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("compressor",), "invalid_type")] + + +def test_v2_compressor_requires_string_id() -> None: + """A compressor mapping without a string id is rejected.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"compressor": {"level": 3}} + problems = validate_array_metadata_v2(doc) + assert [p.loc for p in problems] == [("compressor",)] + + +def test_v2_filters_must_be_codec_sequence_or_none() -> None: + """Filters that are not null or a sequence of codec configs are rejected.""" + for bad in (7, (5,), "gzip"): + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"filters": bad} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("filters",), "invalid_type")], bad + + +def test_v2_shape_and_chunks_must_have_equal_rank() -> None: + """Raw v2 metadata requires one chunk length per array dimension.""" + doc = dict(ZarrV2ArrayMetadata.create_default(shape=(2, 3)).to_json()) + doc["chunks"] = (1,) + + assert [(p.loc, p.kind) for p in validate_array_metadata_v2(doc)] == [ + (("chunks",), "invalid_value") + ] + with pytest.raises(MetadataValidationError, match="same number of dimensions"): + ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) + + +def test_v2_filters_must_be_nonempty_when_present() -> None: + """A non-null v2 filter sequence contains one or more codec configurations.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) + doc["filters"] = () + + assert [(p.loc, p.kind) for p in validate_array_metadata_v2(doc)] == [ + (("filters",), "invalid_value") + ] + with pytest.raises(MetadataValidationError, match="at least one filter"): + ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) + + +def test_v2_dimension_separator_literal_enforced() -> None: + """A dimension_separator other than '.' or '/' is rejected.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dimension_separator": "-"} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] + + +def test_v2_zarr_format_literal_enforced() -> None: + """A v2 document claiming zarr_format 3 is rejected with an invalid_value problem.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"zarr_format": 3} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] + + +def test_v3_zarr_format_literal_enforced() -> None: + """A v3 document claiming zarr_format 2 is rejected with an invalid_value problem.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"zarr_format": 2} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] + + +@pytest.mark.parametrize( + ("document", "validate"), + [ + pytest.param( + dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"zarr_format": 2.0}, + validate_array_metadata_v2, + id="v2", + ), + pytest.param( + dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"zarr_format": 3.0}, + validate_array_metadata_v3, + id="v3", + ), + ], +) +def test_array_zarr_format_rejects_float( + document: object, validate: Callable[[object], list[ValidationProblem]] +) -> None: + """Integer-valued floats do not satisfy integer format literals.""" + assert [(p.loc, p.kind) for p in validate(document)] == [(("zarr_format",), "invalid_value")] + + +def test_array_v2_rejects_unknown_document_member() -> None: + """The closed v2 merged-document shape rejects undeclared members.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"unexpected": 1} + + assert [(p.loc, p.kind) for p in validate_array_metadata_v2(doc)] == [ + (("unexpected",), "invalid_value") + ] + + +def test_array_v3_from_json_materializes_abstract_containers() -> None: + """A flexible input mapping becomes the canonical dict/tuple model shape.""" + doc = UserDict(dict(ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json())) + doc["shape"] = range(2) + + model = ZarrV3ArrayMetadata.from_json(doc) + + assert model.shape == (0, 1) + assert type(model.shape) is tuple + + +def test_from_key_value_rejects_non_standard_json_constant() -> None: + """Store JSON decoding rejects JavaScript NaN/Infinity constants.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) + doc["fill_value"] = float("nan") + raw = json.dumps(doc) + + with pytest.raises(MetadataValidationError, match="invalid JSON"): + ZarrV3ArrayMetadata.from_key_value({"zarr.json": raw.encode()}) + + +def test_to_key_value_rejects_non_finite_model_value() -> None: + """Strict encoding prevents directly-constructed models from writing invalid JSON.""" + model = ZarrV3ArrayMetadata.create_default(fill_value=float("nan")) + + with pytest.raises(ValueError, match="JSON compliant"): + model.to_key_value() + + +def test_v3_node_type_literal_enforced() -> None: + """A v3 array document claiming node_type 'group' is rejected.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"node_type": "group"} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("node_type",), "invalid_value")] + + +def test_missing_key_kind_is_machine_readable() -> None: + """A missing required key is distinguishable by kind, without message matching.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) + del doc["chunk_key_encoding"] + problems = validate_array_metadata_v3(doc) + assert problems == [ + ValidationProblem(("chunk_key_encoding",), "missing required key", "missing_key") + ] + + +# --- Unified error channels --------------------------------------------------- + + +def test_from_key_value_invalid_json_raises_metadata_error() -> None: + """Undecodable store bytes raise MetadataValidationError (kind invalid_json), not JSONDecodeError.""" + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV3ArrayMetadata.from_key_value({"zarr.json": b"{not json"}) + assert [p.kind for p in exc_info.value.problems] == ["invalid_json"] + + +def test_from_key_value_invalid_utf8_raises_metadata_error() -> None: + """Invalid UTF-8 store bytes use the same invalid_json error channel.""" + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV3ArrayMetadata.from_key_value({"zarr.json": b"\x80"}) + assert [p.kind for p in exc_info.value.problems] == ["invalid_json"] + + +def test_v2_from_key_value_scalar_root_raises_metadata_error() -> None: + """A scalar .zarray document fails through the unified metadata error channel.""" + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV2ArrayMetadata.from_key_value({".zarray": b"null"}) + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + ((), "invalid_type") + ] + + +def test_from_key_value_missing_key_kind() -> None: + """A missing store key surfaces as a missing_key problem at the store-key loc.""" + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV2ArrayMetadata.from_key_value({}) + assert exc_info.value.problems == [ + ValidationProblem((".zarray",), "missing store key", "missing_key") + ] + + +def test_extra_fields_overlap_raises_metadata_error() -> None: + """The extra-fields overlap invariant raises MetadataValidationError (a ValueError).""" + with pytest.raises(MetadataValidationError, match="Extra fields") as exc_info: + ZarrV3ArrayMetadata.create_default(extra_fields={"shape": {"must_understand": False}}) + assert [p.kind for p in exc_info.value.problems] == ["invalid_value"] + + +def test_extension_point_fields_annotated_with_role_alias() -> None: + """Extension-point fields are annotated with ZarrV3MetadataField (the + logical role), not ZarrV3NamedConfig (the current serialized form), so a + future widening of the field union does not move annotation sites.""" + assert ZarrV3MetadataField is ZarrV3NamedConfig + annotations = ZarrV3ArrayMetadata.__annotations__ + for field_name in ("data_type", "chunk_grid", "chunk_key_encoding"): + assert annotations[field_name] == "ZarrV3MetadataField" + for field_name in ("codecs", "storage_transformers"): + assert annotations[field_name] == "tuple[ZarrV3MetadataField, ...]" + + +# --- Adversarial-probe fixes: documents that used to pass validation --------- + + +def test_shape_rejects_json_booleans() -> None: + """JSON booleans are not integers: shape/chunks containing true/false are + rejected (bool is an int subclass in Python, so isinstance alone passes).""" + v3 = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"shape": (True, True)} + assert [p.loc for p in validate_array_metadata_v3(v3)] == [("shape",)] + v2 = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"chunks": (True,)} + assert [p.loc for p in validate_array_metadata_v2(v2)] == [("chunks",)] + + +def test_shape_rejects_negative_dimensions() -> None: + """Dimension lengths must be non-negative; a negative entry is invalid_value.""" + v3 = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"shape": (-1,)} + assert [(p.loc, p.kind) for p in validate_array_metadata_v3(v3)] == [ + (("shape",), "invalid_value") + ] + v2 = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"chunks": (-5,)} + assert [(p.loc, p.kind) for p in validate_array_metadata_v2(v2)] == [ + (("chunks",), "invalid_value") + ] + + +def test_dimension_names_length_must_match_shape() -> None: + """dimension_names must have one entry per dimension of shape.""" + doc = dict(ZarrV3ArrayMetadata.create_default(shape=(10,)).to_json()) | { + "dimension_names": ("x", "y", "z") + } + assert [(p.loc, p.kind) for p in validate_array_metadata_v3(doc)] == [ + (("dimension_names",), "invalid_value") + ] + + +def test_attributes_values_must_be_json() -> None: + """Attribute values are JSON-checked recursively (like fill_value), so a + non-serializable value is a validation problem, not a later TypeError.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"attributes": {"a": {1, 2}}} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("attributes", "a"), "invalid_type")] + + +def test_configuration_values_must_be_json() -> None: + """Configuration values are JSON-checked recursively, so an int-keyed dict + cannot pass validation and be silently rewritten by json.dumps.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | { + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": {1: 2}}} + } + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [ + (("chunk_grid", "configuration", "chunk_shape"), "invalid_type") + ] + + +def test_v3_extension_keys_must_be_strings() -> None: + """A non-string top-level key cannot be represented by a v3 document type.""" + doc: dict[object, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) + doc[1] = {"must_understand": False} + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + ((), "invalid_type") + ] + + +def test_v3_extension_values_must_be_json() -> None: + """Extension payloads are JSON-checked before a model is constructed.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) + doc["ext"] = {"must_understand": False, "payload": object()} + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + (("ext", "payload"), "invalid_type") + ] + + +def test_v3_json_extension_without_waiver_is_preserved_as_must_understand() -> None: + """A JSON extension without an explicit false waiver remains must-understand.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) + doc["ext"] = 1 + parsed = parse_array_metadata_v3(doc) + assert is_array_metadata_v3(parsed) + model = ZarrV3ArrayMetadata.from_json(doc) + assert model.extra_fields["ext"] == 1 + assert model.must_understand_fields == {"ext": 1} + + +def test_v2_codec_configuration_values_must_be_json() -> None: + """Non-JSON codec parameters are rejected for compressors and filters.""" + for field, value, expected_loc in ( + ("compressor", {"id": "x", "payload": object()}, ("compressor", "payload")), + ("filters", ({"id": "x", "payload": object()},), ("filters", 0, "payload")), + ): + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) + doc[field] = value + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v2(doc)] == [ + (expected_loc, "invalid_type") + ] + + +def test_dimension_sequences_reject_binary_values() -> None: + """Binary buffers are not JSON arrays even though they are integer sequences.""" + v3 = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"shape": b"\x02"} + v2 = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"chunks": b"\x02"} + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(v3)] == [ + (("shape",), "invalid_type") + ] + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v2(v2)] == [ + (("chunks",), "invalid_type") + ] + + +def test_array_parsers_normalize_json_lists_before_narrowing() -> None: + """Parsers return tuple-backed document types while guards reject raw list forms.""" + v3_raw = json.loads(json.dumps(ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json())) + v2_raw = json.loads(json.dumps(ZarrV2ArrayMetadata.create_default(shape=(2,)).to_json())) + + assert validate_array_metadata_v3(v3_raw) == [] + assert validate_array_metadata_v2(v2_raw) == [] + assert not is_array_metadata_v3(v3_raw) + assert not is_array_metadata_v2(v2_raw) + + v3_parsed = parse_array_metadata_v3(v3_raw) + v2_parsed = parse_array_metadata_v2(v2_raw) + assert isinstance(v3_parsed["shape"], tuple) + assert isinstance(v3_parsed["codecs"], tuple) + assert isinstance(v2_parsed["shape"], tuple) + assert isinstance(v2_parsed["chunks"], tuple) + + +def test_array_guards_reject_noncanonical_nested_json() -> None: + """Document guards cannot narrow values that only parsers can materialize.""" + v3 = dict(ZarrV3ArrayMetadata.create_default().to_json()) + v3["fill_value"] = range(2) + v2 = dict(ZarrV2ArrayMetadata.create_default().to_json()) + v2["fill_value"] = range(2) + + assert not is_array_metadata_v3(v3) + assert not is_array_metadata_v2(v2) + assert parse_array_metadata_v3(v3)["fill_value"] == (0, 1) + assert parse_array_metadata_v2(v2)["fill_value"] == (0, 1) + + +# --- must_understand partition (spec: MUST fail to open unrecognized fields) -- + + +def test_must_understand_fields_partition() -> None: + """must_understand_fields contains every extra field not explicitly waived + with must_understand: false, including implicitly-true and non-mapping + fields, so a reader can discharge the spec's fail-to-open duty by + subtracting the extensions it recognizes.""" + model = ZarrV3ArrayMetadata.create_default( + extra_fields={ + "ext_a": {"name": "a", "must_understand": False}, + "ext_b": {"name": "b"}, + "ext_c": {"name": "c", "must_understand": True}, + "ext_d": 123, + } + ) + assert set(model.must_understand_fields) == {"ext_b", "ext_c", "ext_d"} + recognized = {"ext_b"} + assert model.must_understand_fields.keys() - recognized == {"ext_c", "ext_d"} + + +def test_must_understand_fields_empty_when_all_waived() -> None: + """must_understand_fields is empty when every extra field is explicitly waived.""" + model = ZarrV3ArrayMetadata.create_default( + extra_fields={"ext_a": {"name": "a", "must_understand": False}} + ) + assert model.must_understand_fields == {} + + +def test_dimension_names_null_field_rejected() -> None: + """A dimension_names field whose VALUE is null is invalid: the spec permits + null as an element (an unnamed dimension), never as the field value — "not + specified" is spelled by omitting the key. Consumers bridging from an + in-memory None sentinel must drop the key, not write null.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"dimension_names": None} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dimension_names",), "invalid_type")] + # and the model's own None spelling correctly maps to key absence + assert ( + "dimension_names" not in ZarrV3ArrayMetadata.create_default(dimension_names=UNSET).to_json() + ) + + +# --- create_default derives the chunk grid from shape ------------------------ + + +def test_v3_create_default_chunk_grid_follows_shape() -> None: + """Overriding shape without chunk_grid derives a consistent default grid: + one chunk covering the array (chunk_shape == shape), instead of silently + keeping the scalar default's 0-d grid.""" + model = ZarrV3ArrayMetadata.create_default(shape=(100, 100)) + assert model.chunk_grid == ZarrV3NamedConfig( + name="regular", configuration={"chunk_shape": (100, 100)} + ) + + +def test_v3_create_default_explicit_chunk_grid_respected() -> None: + """An explicit chunk_grid override wins over the shape-derived default.""" + grid = ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": (10, 10)}) + model = ZarrV3ArrayMetadata.create_default(shape=(100, 100), chunk_grid=grid) + assert model.chunk_grid == grid + + +def test_v2_create_default_chunks_follow_shape() -> None: + """Overriding shape without chunks derives chunks == shape.""" + model = ZarrV2ArrayMetadata.create_default(shape=(100, 100)) + assert model.chunks == (100, 100) + + +def test_v2_create_default_explicit_chunks_respected() -> None: + """An explicit chunks override wins over the shape-derived default.""" + model = ZarrV2ArrayMetadata.create_default(shape=(100, 100), chunks=(10, 10)) + assert model.chunks == (10, 10) + + +def test_v3_create_default_zero_length_dimensions() -> None: + """chunk_shape == shape is spec-sound even with zero-length dimensions: + 'The chunk shape elements are non-zero when the corresponding dimensions + of the arrays have non-zero length' — the constraint is conditional, so a + zero chunk length is permitted exactly where the dimension is empty.""" + model = ZarrV3ArrayMetadata.create_default(shape=(0, 3)) + assert model.chunk_grid.configuration["chunk_shape"] == (0, 3) + + +def test_create_default_derivation_is_one_way() -> None: + """Overriding the chunk grid (v3) or chunks (v2) without shape leaves the + scalar default shape=() untouched: a user-supplied chunk_grid is an + extension point taken verbatim, and deriving shape from it would require + interpreting grid configurations, which the model layer never does.""" + grid = ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": (10, 10)}) + v3 = ZarrV3ArrayMetadata.create_default(chunk_grid=grid) + assert v3.shape == () + assert v3.chunk_grid == grid + v2 = ZarrV2ArrayMetadata.create_default(chunks=(10, 10)) + assert v2.shape == () + assert v2.chunks == (10, 10) + + +# --- v2 dimension_separator default (roborev job 426) ------------------------- + + +def test_v2_absent_dimension_separator_means_dot() -> None: + """A .zarray that omits dimension_separator uses the v2 convention default + '.', not '/': chunk keys of real-world default-separator v2 arrays look + like '0.0'. The model normalizes the absent key to an explicit '.' — a + semantics-preserving spelling normalization.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) + del doc["dimension_separator"] + model = ZarrV2ArrayMetadata.from_json(doc) + assert model.dimension_separator == "." + assert model.to_json()["dimension_separator"] == "." + + +def test_v2_from_key_value_without_separator_means_dot() -> None: + """The .zarray store-file path applies the same '.' default for an absent + dimension_separator key.""" + doc = { + k: v + for k, v in ZarrV2ArrayMetadata.create_default().to_json().items() + if k not in ("dimension_separator", "attributes") + } + import json as _json + + model = ZarrV2ArrayMetadata.from_key_value({".zarray": _json.dumps(doc).encode()}) + assert model.dimension_separator == "." + + +def test_v2_null_dimension_separator_rejected() -> None: + """dimension_separator may be absent, '.', or '/' — never null: the + document grammar has no null spelling for this field.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dimension_separator": None} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] + + +def test_dimension_names_absent_and_all_null_are_distinct() -> None: + """An absent dimension_names field and an explicit all-null one are + semantically different documents: the explicit form says every dimension + has a name, which is null; absence says there are no dimension names. + The model preserves the distinction (UNSET vs a tuple of Nones), and both + spellings round-trip faithfully.""" + absent_doc = dict(ZarrV3ArrayMetadata.create_default(shape=(2, 3)).to_json()) + explicit_doc = absent_doc | {"dimension_names": (None, None)} + + absent = ZarrV3ArrayMetadata.from_json(absent_doc) + explicit = ZarrV3ArrayMetadata.from_json(explicit_doc) + + assert absent.dimension_names is UNSET + assert explicit.dimension_names == (None, None) + assert absent != explicit + assert "dimension_names" not in absent.to_json() + assert absent.to_json() == absent_doc + assert explicit.to_json() == explicit_doc diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py new file mode 100644 index 0000000000..4b8c22b84d --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -0,0 +1,572 @@ +"""Tests for the group and consolidated metadata models in `zarr_metadata.model`.""" + +import copy +import dataclasses +import json +from collections import UserDict +from collections.abc import Callable + +import pytest + +from tests.model._cases import mutate_nested_containers +from zarr_metadata.model import UNSET +from zarr_metadata.model._array import ZarrV3ArrayMetadata +from zarr_metadata.model._group import ( + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV2GroupMetadataPartial, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, + ZarrV3GroupMetadataPartial, +) +from zarr_metadata.model._validation import ( + MetadataValidationError, + ValidationProblem, + is_group_metadata_v2, + is_group_metadata_v3, + parse_group_metadata_v2, + parse_group_metadata_v3, + validate_group_metadata_v2, + validate_group_metadata_v3, +) + +# --- ZarrV3GroupMetadata --------------------------------------------------- + + +def test_group_v3_roundtrip() -> None: + """A v3 group document round-trips through the model unchanged.""" + doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": (1, 2)}} + model = ZarrV3GroupMetadata.from_json(doc) + assert model.to_json() == doc + + +def test_group_v3_omits_empty_attributes() -> None: + """to_json omits the attributes key when attributes is empty.""" + model = ZarrV3GroupMetadata.create_default() + assert "attributes" not in model.to_json() + + +def test_group_v3_lists_become_tuples() -> None: + """from_json converts JSON arrays in attributes to tuples.""" + doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": [1, 2]}} + model = ZarrV3GroupMetadata.from_json(doc) + assert model.attributes == {"a": (1, 2)} + + +def test_group_v3_extra_fields_roundtrip() -> None: + """Unknown top-level keys land in extra_fields and reappear in to_json.""" + doc = { + "zarr_format": 3, + "node_type": "group", + "my_extension": {"name": "thing", "must_understand": False}, + } + model = ZarrV3GroupMetadata.from_json(doc) + assert model.extra_fields == {"my_extension": {"name": "thing", "must_understand": False}} + assert model.to_json() == doc + + +def test_group_v3_json_extra_field_roundtrips_as_must_understand() -> None: + """A non-object extra field is preserved and implicitly requires understanding.""" + doc = {"zarr_format": 3, "node_type": "group", "ext": [1, 2]} + model = ZarrV3GroupMetadata.from_json(doc) + assert model.to_json()["ext"] == (1, 2) + assert model.must_understand_fields == {"ext": (1, 2)} + + +def test_group_v3_extra_fields_overlap_rejected() -> None: + """Constructing a v3 group model with extra_fields shadowing a standard key raises.""" + with pytest.raises(ValueError, match="Extra fields"): + ZarrV3GroupMetadata( + attributes={}, + consolidated_metadata=UNSET, + extra_fields={"node_type": {"name": "x", "must_understand": False}}, + ) + + +def test_group_v3_consolidated_extra_field_rejected() -> None: + """extra_fields may not shadow the consolidated_metadata convention key.""" + with pytest.raises(ValueError, match="Extra fields"): + ZarrV3GroupMetadata( + attributes={}, + consolidated_metadata=UNSET, + extra_fields={"consolidated_metadata": {"name": "x", "must_understand": False}}, + ) + + +def test_group_v3_missing_required_key() -> None: + """parse_group_metadata_v3 reports each missing required key.""" + with pytest.raises(MetadataValidationError, match="node_type"): + parse_group_metadata_v3({"zarr_format": 3}) + + +def test_group_v3_bad_attributes() -> None: + """parse_group_metadata_v3 rejects a non-mapping attributes value.""" + with pytest.raises(MetadataValidationError, match="attributes"): + parse_group_metadata_v3({"zarr_format": 3, "node_type": "group", "attributes": 5}) + + +@pytest.mark.parametrize( + ("document", "validate"), + [ + pytest.param( + {"zarr_format": 2.0}, + validate_group_metadata_v2, + id="v2", + ), + pytest.param( + {"zarr_format": 3.0, "node_type": "group"}, + validate_group_metadata_v3, + id="v3", + ), + ], +) +def test_group_zarr_format_rejects_float( + document: object, validate: Callable[[object], list[ValidationProblem]] +) -> None: + """Integer-valued floats do not satisfy integer format literals.""" + assert [(p.loc, p.kind) for p in validate(document)] == [(("zarr_format",), "invalid_value")] + + +def test_group_v2_rejects_unknown_document_member() -> None: + """The closed v2 merged-document shape rejects undeclared members.""" + assert [(p.loc, p.kind) for p in validate_group_metadata_v2({"zarr_format": 2, "x": 1})] == [ + (("x",), "invalid_value") + ] + + +@pytest.mark.parametrize( + ("parse", "document"), + [ + pytest.param(parse_group_metadata_v2, {"zarr_format": 2}, id="v2"), + pytest.param( + parse_group_metadata_v3, + {"zarr_format": 3, "node_type": "group"}, + id="v3", + ), + ], +) +def test_group_parser_materializes_abstract_mapping( + parse: Callable[[object], object], document: dict[str, object] +) -> None: + """A successful group parser always returns the declared concrete TypedDict shape.""" + parsed = parse(UserDict(document)) + + assert type(parsed) is dict + assert parsed == document + + +def test_group_guards_reject_noncanonical_nested_json() -> None: + """Document guards cannot narrow values that only parsers can materialize.""" + v3 = {"zarr_format": 3, "node_type": "group", "extension": range(2)} + v2 = {"zarr_format": 2, "attributes": {"values": range(2)}} + + assert not is_group_metadata_v3(v3) + assert not is_group_metadata_v2(v2) + assert parse_group_metadata_v3(v3)["extension"] == (0, 1) + assert parse_group_metadata_v2(v2)["attributes"] == {"values": (0, 1)} + + +def test_group_v3_extension_fields_are_validated() -> None: + """Group extension payloads must be JSON values with a must-understand flag.""" + doc = { + "zarr_format": 3, + "node_type": "group", + "ext": {"must_understand": False, "payload": object()}, + } + assert [(problem.loc, problem.kind) for problem in validate_group_metadata_v3(doc)] == [ + (("ext", "payload"), "invalid_type") + ] + + +def test_group_v3_key_value_roundtrip() -> None: + """from_key_value(to_key_value()) is the identity for v3 groups.""" + model = ZarrV3GroupMetadata.create_default(attributes={"a": 1}) + assert ZarrV3GroupMetadata.from_key_value(model.to_key_value()) == model + + +def test_group_v3_update() -> None: + """update replaces the given fields and returns a new instance.""" + base = ZarrV3GroupMetadata.create_default() + updated = base.update(attributes={"a": 1}) + assert updated.attributes == {"a": 1} + assert base.attributes == {} + + +# --- ZarrV2GroupMetadata --------------------------------------------------- + + +def test_group_v2_key_value_split() -> None: + """v2 to_key_value writes .zgroup and .zattrs; from_key_value merges them.""" + model = ZarrV2GroupMetadata.create_default(attributes={"a": 1}) + kv = model.to_key_value() + assert set(kv) == {".zgroup", ".zattrs"} + assert json.loads(kv[".zgroup"]) == {"zarr_format": 2} + assert ZarrV2GroupMetadata.from_key_value(kv) == model + + +@pytest.mark.parametrize("extra_key", ["attributes", "vendor_extension"]) +def test_v2_group_from_key_value_rejects_zgroup_extra_members(extra_key: str) -> None: + """Raw `.zgroup` documents reject every non-spec member.""" + doc: dict[str, object] = {"zarr_format": 2, extra_key: {}} + + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV2GroupMetadata.from_key_value({".zgroup": json.dumps(doc).encode()}) + + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + ((extra_key,), "invalid_value") + ] + + +def test_group_v2_zattrs_presence_round_trips() -> None: + """A v2 group with no .zattrs file parses with UNSET attributes and emits + no .zattrs; an explicit empty .zattrs stays a file — the stores remain + distinct through a round-trip.""" + absent = ZarrV2GroupMetadata.from_key_value({".zgroup": b'{"zarr_format": 2}'}) + assert absent.attributes is UNSET + assert ".zattrs" not in absent.to_key_value() + explicit = ZarrV2GroupMetadata.from_key_value( + {".zgroup": b'{"zarr_format": 2}', ".zattrs": b"{}"} + ) + assert explicit.attributes == {} + assert ".zattrs" in explicit.to_key_value() + assert absent != explicit + + +def test_group_v2_json_roundtrip() -> None: + """A merged-form v2 group document round-trips through the model unchanged.""" + doc = {"zarr_format": 2, "attributes": {"a": 1}} + model = ZarrV2GroupMetadata.from_json(doc) + assert model.to_json() == doc + + +def test_group_v2_omits_empty_attributes() -> None: + """to_json omits the attributes key when attributes is empty.""" + assert "attributes" not in ZarrV2GroupMetadata.create_default().to_json() + + +def test_group_v2_not_a_mapping() -> None: + """parse_group_metadata_v2 rejects a non-mapping document.""" + with pytest.raises(MetadataValidationError, match="expected a mapping"): + parse_group_metadata_v2([1, 2, 3]) + + +def test_group_v2_missing_required_key() -> None: + """parse_group_metadata_v2 reports a missing zarr_format key.""" + with pytest.raises(MetadataValidationError, match="zarr_format"): + parse_group_metadata_v2({}) + + +# --- Partial TypedDict drift guards ----------------------------------------- + + +def test_group_partial_keys_match_settable_model_fields() -> None: + """Each group partial TypedDict must list exactly the settable model fields. + + Guards against drift: adding/removing a settable field on a group model + without updating its `*Partial` TypedDict fails here. + """ + for model_cls, partial_cls in ( + (ZarrV3GroupMetadata, ZarrV3GroupMetadataPartial), + (ZarrV2GroupMetadata, ZarrV2GroupMetadataPartial), + ): + settable = {f.name for f in dataclasses.fields(model_cls) if f.init} + assert set(partial_cls.__annotations__) == settable + + +# --- ZarrV3ConsolidatedMetadata -------------------------------------------- + + +def test_consolidated_v3_roundtrip() -> None: + """A v3 group with inline consolidated metadata round-trips, with child + entries parsed into array/group models.""" + child = ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json() + doc = { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": child, "g": {"zarr_format": 3, "node_type": "group"}}, + }, + } + model = ZarrV3GroupMetadata.from_json(doc) + assert isinstance(model.consolidated_metadata, ZarrV3ConsolidatedMetadata) + assert isinstance(model.consolidated_metadata.metadata["a"], ZarrV3ArrayMetadata) + assert isinstance(model.consolidated_metadata.metadata["g"], ZarrV3GroupMetadata) + assert model.to_json() == doc + + +def test_consolidated_v3_must_understand_true_rejected() -> None: + """ZarrV3ConsolidatedMetadata enforces must_understand=False at runtime.""" + with pytest.raises(ValueError, match="must_understand"): + ZarrV3ConsolidatedMetadata(must_understand=True, metadata={}) + + +def test_consolidated_v3_from_json_must_understand_true_rejected() -> None: + """from_json rejects a consolidated document carrying must_understand=true.""" + with pytest.raises(MetadataValidationError, match="must_understand"): + ZarrV3ConsolidatedMetadata.from_json( + {"kind": "inline", "must_understand": True, "metadata": {}} + ) + + +def test_consolidated_v3_entry_without_node_type_rejected() -> None: + """from_json rejects a consolidated entry lacking a recognizable node_type.""" + with pytest.raises(MetadataValidationError, match="node_type"): + ZarrV3ConsolidatedMetadata.from_json( + {"kind": "inline", "must_understand": False, "metadata": {"a": {"zarr_format": 3}}} + ) + + +def test_consolidated_v3_not_a_mapping() -> None: + """from_json rejects a non-mapping consolidated document.""" + with pytest.raises(MetadataValidationError, match="expected a mapping"): + ZarrV3ConsolidatedMetadata.from_json(5) + + +# --- ZarrV2ConsolidatedMetadata -------------------------------------------- + + +def test_consolidated_v2_verbatim_roundtrip() -> None: + """The v2 .zmetadata model holds the flat file-keyed map verbatim, + including nodes that have no .zattrs entry.""" + doc = { + "zarr_consolidated_format": 1, + "metadata": { + ".zgroup": {"zarr_format": 2}, + "a/.zarray": { + "zarr_format": 2, + "shape": (2,), + "chunks": (2,), + "dtype": "|u1", + "fill_value": 0, + "order": "C", + "compressor": None, + "filters": None, + }, + }, + } + model = ZarrV2ConsolidatedMetadata.from_json(doc) + assert model.to_json() == doc + + +def test_consolidated_v2_key_value_roundtrip() -> None: + """from_key_value(to_key_value()) is the identity for .zmetadata documents.""" + model = ZarrV2ConsolidatedMetadata.from_json( + {"zarr_consolidated_format": 1, "metadata": {".zgroup": {"zarr_format": 2}}} + ) + assert ZarrV2ConsolidatedMetadata.from_key_value(model.to_key_value()) == model + + +def test_consolidated_v2_lists_become_tuples() -> None: + """from_json converts JSON arrays inside entries to tuples.""" + doc = { + "zarr_consolidated_format": 1, + "metadata": {"a/.zarray": {"shape": [2, 3]}}, + } + model = ZarrV2ConsolidatedMetadata.from_json(doc) + assert model.metadata == {"a/.zarray": {"shape": (2, 3)}} + + +def test_consolidated_v2_envelope_validation() -> None: + """from_json rejects a .zmetadata document missing the metadata key.""" + with pytest.raises(MetadataValidationError, match="metadata"): + ZarrV2ConsolidatedMetadata.from_json({"zarr_consolidated_format": 1}) + + +def test_consolidated_v2_not_a_mapping() -> None: + """from_json rejects a non-mapping .zmetadata document.""" + with pytest.raises(MetadataValidationError, match="expected a mapping"): + ZarrV2ConsolidatedMetadata.from_json([1]) + + +def test_consolidated_v2_format_literal_enforced() -> None: + """A .zmetadata document must declare consolidated format 1.""" + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV2ConsolidatedMetadata.from_json({"zarr_consolidated_format": 2, "metadata": {}}) + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + (("zarr_consolidated_format",), "invalid_value") + ] + + +def test_consolidated_v2_metadata_values_must_be_json() -> None: + """Non-JSON values in the flat metadata map are rejected during ingestion.""" + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV2ConsolidatedMetadata.from_json( + {"zarr_consolidated_format": 1, "metadata": {".zgroup": object()}} + ) + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + (("metadata", ".zgroup"), "invalid_type") + ] + + +def test_group_v2_from_key_value_scalar_root_raises_metadata_error() -> None: + """A scalar .zgroup document fails through the unified metadata error channel.""" + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV2GroupMetadata.from_key_value({".zgroup": b"null"}) + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + ((), "invalid_type") + ] + + +# --- Literal-value enforcement ----------------------------------------------- + + +def test_group_v3_literals_enforced() -> None: + """A v3 group doc with wrong zarr_format or node_type is rejected with invalid_value.""" + base = ZarrV3GroupMetadata.create_default().to_json() + for key, bad in (("zarr_format", 2), ("node_type", "array")): + problems = validate_group_metadata_v3(dict(base) | {key: bad}) + assert [(p.loc, p.kind) for p in problems] == [((key,), "invalid_value")], key + + +def test_group_v2_zarr_format_literal_enforced() -> None: + """A v2 group doc claiming zarr_format 3 is rejected with invalid_value.""" + problems = validate_group_metadata_v2({"zarr_format": 3}) + assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] + + +# --- Consolidated envelope validated by the group validator ------------------ + + +def test_group_v3_validator_agrees_with_from_json_on_consolidated() -> None: + """The group validator validates the consolidated envelope and entries, so + is_group_metadata_v3 never vouches for a document from_json would reject.""" + bad_docs = ( + # empty envelope: missing kind/must_understand/metadata + {"zarr_format": 3, "node_type": "group", "consolidated_metadata": {}}, + # entry without a recognizable node_type + { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": {"zarr_format": 3}}, + }, + }, + # must_understand: true + { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": True, + "metadata": {}, + }, + }, + ) + for doc in bad_docs: + assert validate_group_metadata_v3(doc) != [], doc + with pytest.raises(MetadataValidationError): + ZarrV3GroupMetadata.from_json(doc) + + +def test_group_v3_valid_consolidated_passes_validator() -> None: + """A well-formed consolidated group validates cleanly (control case).""" + child = ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json() + doc = { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": child, "g": {"zarr_format": 3, "node_type": "group"}}, + }, + } + assert validate_group_metadata_v3(doc) == [] + + +def test_v3_consolidated_rejects_unknown_envelope_member() -> None: + """The inline consolidated envelope is closed and never drops accepted members.""" + doc = { + "kind": "inline", + "must_understand": False, + "metadata": {}, + "unexpected": 1, + } + + with pytest.raises(MetadataValidationError, match="unexpected"): + ZarrV3ConsolidatedMetadata.from_json(doc) + + +def test_v2_consolidated_rejects_unknown_document_member() -> None: + """The v2 consolidated document is closed and never drops accepted members.""" + doc = {"zarr_consolidated_format": 1, "metadata": {}, "unexpected": 1} + + with pytest.raises(MetadataValidationError, match="unexpected"): + ZarrV2ConsolidatedMetadata.from_json(doc) + + +# --- must_understand partition ------------------------------------------------ + + +def test_group_must_understand_fields_partition() -> None: + """The group model partitions extra fields by the spec's implicit-true rule, + like the array model.""" + model = ZarrV3GroupMetadata.create_default( + extra_fields={ + "waived": {"name": "w", "must_understand": False}, + "implicit": {"name": "i"}, + } + ) + assert set(model.must_understand_fields) == {"implicit"} + + +def test_group_v3_null_consolidated_metadata_repaired_to_absence() -> None: + """consolidated_metadata: null was written by a historical zarr-python bug. + Those stores must remain readable, but the bug spelling is not honored: + it is read as absence (UNSET) and never written back — the round-trip + deliberately repairs the document rather than preserving the bug.""" + null_doc = {"zarr_format": 3, "node_type": "group", "consolidated_metadata": None} + assert validate_group_metadata_v3(null_doc) == [] + model = ZarrV3GroupMetadata.from_json(null_doc) + assert model.consolidated_metadata is UNSET + assert "consolidated_metadata" not in model.to_json() + assert model == ZarrV3GroupMetadata.from_json({"zarr_format": 3, "node_type": "group"}) + + +# --- to_json shares no mutable state with the model ------------------------ + +TO_JSON_NO_ALIASING_PARAMS = [ + pytest.param( + ZarrV3GroupMetadata.create_default( + attributes={"a": {"b": [1]}}, + consolidated_metadata=ZarrV3ConsolidatedMetadata( + metadata={ + "child": ZarrV3ArrayMetadata.create_default(attributes={"x": {"y": 1}}), + "grp": ZarrV3GroupMetadata.create_default(attributes={"x": {"y": 1}}), + } + ), + extra_fields={"ext": {"must_understand": False, "cfg": {"x": [1]}}}, + ), + id="v3-group", + ), + pytest.param( + ZarrV2GroupMetadata.create_default(attributes={"a": {"b": [1]}}), + id="v2-group", + ), + pytest.param( + ZarrV3ConsolidatedMetadata( + metadata={"child": ZarrV3ArrayMetadata.create_default(attributes={"x": {"y": 1}})} + ), + id="v3-consolidated", + ), + pytest.param( + ZarrV2ConsolidatedMetadata(metadata={"a/.zarray": {"nested": {"x": [1]}}}), + id="v2-consolidated", + ), +] + + +@pytest.mark.parametrize("model", TO_JSON_NO_ALIASING_PARAMS) +def test_to_json_shares_no_mutable_state_with_model( + model: ZarrV3GroupMetadata + | ZarrV2GroupMetadata + | ZarrV3ConsolidatedMetadata + | ZarrV2ConsolidatedMetadata, +) -> None: + """Mutating a document returned by to_json leaves the model unchanged.""" + baseline = copy.deepcopy(model.to_json()) + mutate_nested_containers(model.to_json()) + assert model.to_json() == baseline diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py new file mode 100644 index 0000000000..e5714bb8fb --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -0,0 +1,302 @@ +"""Executable example: integrating the metadata models with pydantic (v2). + +Pydantic's native dataclass introspection CAN be made to work (see +`test_native_dataclass_introspection_is_possible_but_diverges`): the models +keep their annotation-only imports behind `TYPE_CHECKING`, so a bare +`TypeAdapter(ZarrV3ArrayMetadata)` raises `class-not-fully-defined`, but +`rebuild(_types_namespace=...)` with the names supplied resolves the schema. +It is still the wrong tool: it validates the MODEL SHAPE, not the DOCUMENT — +no `from_json` normalization (a bare-string `data_type` is rejected), and +pydantic's lax coercion silently re-opens holes the library's validators +close (`shape=[True, -5]` coerces to `(1, -5)`; a wrong `dimension_names` +count passes). The recommended integration delegates wholesale — treat the +model as an opaque value: + +- `InstanceOf` makes pydantic's core schema an is-instance check (no field + introspection), +- validation goes through `from_json` (the single source of truth for what + a well-formed document is, including normalization: bare-string metadata + fields, arrays-to-tuples), +- serialization goes through `to_json` (the canonical document form). + +`MetadataValidationError` subclasses `ValueError`, so pydantic converts a +failed parse into its own `ValidationError` with the loc-annotated problem +messages intact. +""" + +from collections.abc import Mapping +from typing import Annotated, Generic, TypeVar + +import pytest +from pydantic import ( + BaseModel, + BeforeValidator, + ConfigDict, + InstanceOf, + PlainSerializer, + PydanticSchemaGenerationError, + PydanticUserError, + TypeAdapter, + ValidationError, + model_validator, +) + +from zarr_metadata import JSONValue +from zarr_metadata.model import ZarrV3ArrayMetadata, ZarrV3NamedConfig + +# --- the integration (this is the example) ----------------------------------- + + +def _as_array_metadata_v3(value: object) -> ZarrV3ArrayMetadata: + """Accept an existing model instance or a raw metadata document.""" + if isinstance(value, ZarrV3ArrayMetadata): + return value + return ZarrV3ArrayMetadata.from_json(value) + + +# return_type is explicit because to_json's own annotation (`ZarrV3ArrayMetadataJSON`) +# is a TYPE_CHECKING-only name pydantic cannot resolve at runtime. +ArrayMetadataV3Field = Annotated[ + InstanceOf[ZarrV3ArrayMetadata], + BeforeValidator(_as_array_metadata_v3), + PlainSerializer(ZarrV3ArrayMetadata.to_json, return_type=dict), +] +"""A pydantic-ready field type for v3 array metadata. + +Validates raw documents via `from_json`, passes model instances through, +and serializes to the canonical document form via `to_json`. +""" + + +class ArrayManifest(BaseModel): + """Example consumer model: a named array with its metadata document.""" + + path: str + metadata: ArrayMetadataV3Field + + +# --- tests pinning the example ------------------------------------------------ + +VALID_DOC = { + "zarr_format": 3, + "node_type": "array", + "shape": [10], + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}}, + "chunk_key_encoding": {"name": "default"}, + "codecs": [{"name": "bytes"}], +} + + +def test_raw_document_is_validated_into_a_model() -> None: + """A raw metadata document on a pydantic field is parsed by from_json, + with the library's normalization applied (tuples, canonical field form).""" + manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) + assert isinstance(manifest.metadata, ZarrV3ArrayMetadata) + assert manifest.metadata.shape == (10,) + assert manifest.metadata.data_type.name == "uint8" + + +def test_model_instance_passes_through() -> None: + """An already-constructed model instance is accepted unchanged.""" + model = ZarrV3ArrayMetadata.from_json(VALID_DOC) + manifest = ArrayManifest(path="a/b", metadata=model) + assert manifest.metadata is model + + +def test_invalid_document_surfaces_problems_in_validation_error() -> None: + """A structurally-invalid document fails pydantic validation, carrying the + loc-annotated problem messages from MetadataValidationError.""" + doc = dict(VALID_DOC) + del doc["chunk_key_encoding"] + with pytest.raises(ValidationError) as exc_info: + ArrayManifest.model_validate({"path": "a/b", "metadata": doc}) + assert "chunk_key_encoding: missing required key" in str(exc_info.value) + + +def test_dump_emits_canonical_document() -> None: + """model_dump serializes the field via to_json — the canonical document, + not pydantic's field-by-field view of the dataclass.""" + manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) + dumped = manifest.model_dump() + assert dumped["metadata"] == manifest.metadata.to_json() + # Empty configurations use the extension-definition shorthand form. + assert dumped["metadata"]["data_type"] == "uint8" + + +def test_json_roundtrip_through_pydantic() -> None: + """model_dump_json output re-validates to an equal manifest (JSON emits + tuples as arrays; from_json converts them back).""" + manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) + revived = ArrayManifest.model_validate_json(manifest.model_dump_json()) + assert revived == manifest + + +def test_type_adapter_standalone() -> None: + """The annotated alias also works without a BaseModel, via TypeAdapter.""" + adapter = TypeAdapter(ArrayMetadataV3Field) + model = adapter.validate_python(VALID_DOC) + assert isinstance(model, ZarrV3ArrayMetadata) + assert adapter.dump_python(model) == model.to_json() + + +# --- the road not taken: native dataclass introspection ---------------------- + + +def test_native_dataclass_introspection_is_not_supported() -> None: + """Pydantic cannot field-introspect the model dataclasses: the UNSET + sentinel (PEP 661, typing_extensions.Sentinel) in the optional-field + annotations has no pydantic schema (as of pydantic 2.13), so even the + rebuild-with-namespace recipe fails. Introspection was already the wrong + tool before the sentinel existed — it validated the model shape rather + than the document, and its lax coercion re-opened validator holes (e.g. + shape=[True, -5] coerced to (1, -5)) — so the delegation patterns above + are the only supported integrations. If this test ever fails because + pydantic learned to handle sentinels, revisit whether the introspection + path needs its divergences documented again.""" + from zarr_metadata._common import JSONValue + from zarr_metadata.model import UNSET + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON, ZarrV3ExtensionField + + def build_and_use() -> None: + adapter = TypeAdapter(ZarrV3ArrayMetadata) + adapter.rebuild( + force=True, + _types_namespace={ + "JSONValue": JSONValue, + "ZarrV3ExtensionField": ZarrV3ExtensionField, + "ZarrV3MetadataFieldJSON": ZarrV3MetadataFieldJSON, + "ZarrV3ArrayMetadataJSON": ZarrV3ArrayMetadataJSON, + "ZarrV3NamedConfig": ZarrV3NamedConfig, + "ZarrV3MetadataField": ZarrV3NamedConfig, + "UNSET": UNSET, + }, + ) + adapter.validate_python({}) + + with pytest.raises((AttributeError, PydanticSchemaGenerationError, PydanticUserError)): + build_and_use() + + +# --- a first-class pydantic model, engine-backed (the pydantic-zarr pattern) -- +# +# When a consumer wants a real BaseModel — JSON schema generation, and +# generics for typed attributes, as in pydantic-zarr's ArraySpec — the model +# fields are pydantic-native, but validation and serialization still route +# through the library: a mode="before" validator canonicalizes every input +# document with from_json(...).to_json(), so the structural validators and +# normalization run BEFORE pydantic parses fields (no coercion divergence), +# and the document form is the bridge in both directions. + +AttrsT = TypeVar("AttrsT") + + +class NamedConfig(BaseModel): + """Pydantic mirror of a normalized metadata extension envelope.""" + + name: str + configuration: dict[str, JSONValue] = {} + must_understand: bool = True + + +class ArrayMetadataV3Spec(BaseModel, Generic[AttrsT]): + """A pydantic-native, attribute-typed view of a v3 array metadata document. + + The library is the engine: every input is canonicalized and structurally + validated by `ZarrV3ArrayMetadata.from_json` before pydantic sees the + fields, and `to_document` / `to_metadata_model` emit through the library. + """ + + model_config = ConfigDict(frozen=True) + + zarr_format: int = 3 + node_type: str = "array" + shape: tuple[int, ...] + data_type: NamedConfig + chunk_grid: NamedConfig + chunk_key_encoding: NamedConfig + fill_value: JSONValue + codecs: tuple[NamedConfig, ...] + attributes: AttrsT + dimension_names: tuple[str | None, ...] | None = None + storage_transformers: tuple[NamedConfig, ...] = () + + @model_validator(mode="before") + @classmethod + def _canonicalize(cls, data: object) -> object: + """Route every input document through the library's validation and + normalization; pydantic then parses only canonical documents.""" + if isinstance(data, Mapping): + doc = dict(ZarrV3ArrayMetadata.from_json(data).to_json()) + for key in ("data_type", "chunk_grid", "chunk_key_encoding"): + if isinstance(doc[key], str): + doc[key] = {"name": doc[key]} + for key in ("codecs", "storage_transformers"): + doc[key] = tuple( + {"name": item} if isinstance(item, str) else item for item in doc.get(key, ()) + ) + doc.setdefault("attributes", {}) + return doc + return data + + def to_metadata_model(self) -> ZarrV3ArrayMetadata: + """Bridge back to the canonical model, via the document form. + + In the document, "no dimension names" is key-absence, not null; the + pydantic-side None translates to dropping the key. + """ + doc = self.model_dump() + if doc["dimension_names"] is None: + del doc["dimension_names"] + return ZarrV3ArrayMetadata.from_json(doc) + + def to_document(self) -> dict[str, object]: + """The canonical document (omit-empty conventions applied).""" + return dict(self.to_metadata_model().to_json()) + + +class MicroscopyAttrs(BaseModel): + """Example of consumer-typed attributes, pydantic-zarr style.""" + + resolution_um: float + + +def test_spec_typed_attributes() -> None: + """The generic parameter types the attributes, so consumers get validated, + attribute-level access — the pydantic-zarr ArraySpec pattern.""" + doc = dict(VALID_DOC) | {"attributes": {"resolution_um": 0.5}} + spec = ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(doc) + assert spec.attributes.resolution_um == 0.5 + assert spec.data_type == NamedConfig(name="uint8") + + +def test_spec_engine_validates_before_pydantic() -> None: + """The library's structural validation runs before pydantic's parsing, so + coercion cannot re-open validator holes (contrast with the native + introspection test above, where [True, -5] coerced to (1, -5)).""" + with pytest.raises(ValidationError, match="shape"): + ArrayMetadataV3Spec[MicroscopyAttrs].model_validate( + dict(VALID_DOC) | {"shape": [True, -5], "attributes": {"resolution_um": 0.5}} + ) + + +def test_spec_bridges_to_canonical_model_and_document() -> None: + """to_metadata_model / to_document round-trip through the document form, + and the emitted document matches what the library itself would emit.""" + doc = dict(VALID_DOC) | {"attributes": {"resolution_um": 0.5}} + spec = ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(doc) + model = spec.to_metadata_model() + assert isinstance(model, ZarrV3ArrayMetadata) + assert spec.to_document() == dict(model.to_json()) + # and back: the document revalidates to an equal spec + assert ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(spec.to_document()) == spec + + +def test_spec_json_schema_generation() -> None: + """A real BaseModel means model_json_schema works — the capability the + opaque InstanceOf pattern cannot provide.""" + schema = ArrayMetadataV3Spec[MicroscopyAttrs].model_json_schema() + assert schema["properties"]["shape"]["type"] == "array" + assert "MicroscopyAttrs" in schema["$defs"] diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py new file mode 100644 index 0000000000..d15b3f118c --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -0,0 +1,264 @@ +"""Tests for `zarr_metadata.pydantic`, the optional pydantic field-type module. + +The hand-rolled recipes in `test_pydantic.py` document how the integration +works; this module ships it. Instances are the CORE model classes (no +parallel hierarchy), so values interoperate freely with non-pydantic code. +""" + +import json +import warnings + +import pytest +from jsonschema import Draft202012Validator +from pydantic import BaseModel, TypeAdapter, ValidationError + +import zarr_metadata.pydantic as zmp +from zarr_metadata.model import ( + ZarrV2ArrayMetadata, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV3ArrayMetadata, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, + ZarrV3NamedConfig, +) + +V3_ARRAY_DOC = dict(ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()) +V2_ARRAY_DOC = dict(ZarrV2ArrayMetadata.create_default(shape=(4,), chunks=(2,)).to_json()) +V3_GROUP_DOC = {"zarr_format": 3, "node_type": "group", "attributes": {"a": 1}} +V2_GROUP_DOC = {"zarr_format": 2, "attributes": {"a": 1}} +V3_CONSOLIDATED_DOC = { + "kind": "inline", + "must_understand": False, + "metadata": {"a": dict(V3_ARRAY_DOC)}, +} +V2_CONSOLIDATED_DOC = { + "zarr_consolidated_format": 1, + "metadata": {".zgroup": {"zarr_format": 2}}, +} + +FIELD_CASES = [ + pytest.param(zmp.ZarrV3ArrayMetadata, ZarrV3ArrayMetadata, V3_ARRAY_DOC, id="array-v3"), + pytest.param(zmp.ZarrV2ArrayMetadata, ZarrV2ArrayMetadata, V2_ARRAY_DOC, id="array-v2"), + pytest.param(zmp.ZarrV3GroupMetadata, ZarrV3GroupMetadata, V3_GROUP_DOC, id="group-v3"), + pytest.param(zmp.ZarrV2GroupMetadata, ZarrV2GroupMetadata, V2_GROUP_DOC, id="group-v2"), + pytest.param( + zmp.ZarrV3ConsolidatedMetadata, + ZarrV3ConsolidatedMetadata, + V3_CONSOLIDATED_DOC, + id="consolidated-v3", + ), + pytest.param( + zmp.ZarrV2ConsolidatedMetadata, + ZarrV2ConsolidatedMetadata, + V2_CONSOLIDATED_DOC, + id="consolidated-v2", + ), + pytest.param(zmp.ZarrV3MetadataField, ZarrV3NamedConfig, {"name": "bytes"}, id="field-v3"), +] + + +@pytest.mark.parametrize(("field_type", "model_cls", "doc"), FIELD_CASES) +def test_field_type_validates_and_dumps_canonically( + field_type: object, model_cls: type, doc: dict[str, object] +) -> None: + """Each field type parses its raw document into the CORE model class, + passes existing instances through unchanged, and dumps the canonical + document via to_json.""" + adapter = TypeAdapter(field_type) + model = adapter.validate_python(doc) + assert type(model) is model_cls + assert adapter.validate_python(model) is model + assert adapter.dump_python(model) == model.to_json() + + +def test_core_instances_interoperate() -> None: + """A core model instance (e.g. handed out by zarr-python) drops straight + into a pydantic field — the reason the module ships Annotated aliases over + the core classes rather than pydantic-aware subclasses.""" + + class Manifest(BaseModel): + metadata: zmp.ZarrV3ArrayMetadata + + core = ZarrV3ArrayMetadata.from_json(V3_ARRAY_DOC) + manifest = Manifest(metadata=core) + assert manifest.metadata is core + + +def test_validation_error_carries_problems() -> None: + """A defective document fails with the library's loc-annotated messages.""" + + class Manifest(BaseModel): + metadata: zmp.ZarrV3ArrayMetadata + + doc = dict(V3_ARRAY_DOC) + del doc["chunk_key_encoding"] + with pytest.raises(ValidationError, match="chunk_key_encoding: missing required key"): + Manifest.model_validate({"metadata": doc}) + + +def test_json_schema_generation() -> None: + """model_json_schema works, describing the document form each field accepts.""" + + class Manifest(BaseModel): + metadata: zmp.ZarrV3ArrayMetadata + codec: zmp.ZarrV3MetadataField + + schema = Manifest.model_json_schema() + metadata_schema = schema["$defs"]["ZarrV3ArrayMetadataJSON"] + assert schema["properties"]["metadata"]["$ref"] == "#/$defs/ZarrV3ArrayMetadataJSON" + assert metadata_schema["required"] == [ + "zarr_format", + "node_type", + "data_type", + "shape", + "chunk_grid", + "chunk_key_encoding", + "fill_value", + "codecs", + ] + assert metadata_schema["properties"]["zarr_format"] == { + "const": 3, + "title": "Zarr Format", + "type": "integer", + } + assert schema["properties"]["codec"]["anyOf"] == [ + {"type": "string"}, + {"$ref": "#/$defs/ZarrV3NamedConfigJSON"}, + ] + + +def test_json_schema_generation_emits_no_warnings() -> None: + """Consumers can generate every public integration schema without warning filters.""" + field_types = ( + zmp.ZarrV3ArrayMetadata, + zmp.ZarrV2ArrayMetadata, + zmp.ZarrV3GroupMetadata, + zmp.ZarrV2GroupMetadata, + zmp.ZarrV3ConsolidatedMetadata, + zmp.ZarrV2ConsolidatedMetadata, + zmp.ZarrV3MetadataField, + ) + + with warnings.catch_warnings(): + warnings.simplefilter("error") + for field_type in field_types: + TypeAdapter(field_type).json_schema() + + +def test_v2_recursive_structured_dtype_is_in_pydantic_schema() -> None: + """The schema accepts nested structured dtypes supported by the v2 specification.""" + doc = json.loads(json.dumps(V2_ARRAY_DOC)) + doc["dtype"] = [["outer", [["inner", " None: + adapter = TypeAdapter(field_type) + with pytest.raises(ValidationError): + adapter.validate_python(document) + assert list(Draft202012Validator(adapter.json_schema()).iter_errors(document)) + + +def test_v3_array_schema_rejects_empty_codecs() -> None: + """The generated schema mirrors the runtime non-empty codec pipeline rule.""" + doc = json.loads(json.dumps(V3_ARRAY_DOC)) + doc["codecs"] = [] + + _assert_runtime_and_schema_reject(zmp.ZarrV3ArrayMetadata, doc) + + +def test_array_schemas_reject_negative_dimensions() -> None: + """Both array schemas mirror the runtime non-negative dimension rule.""" + for field_type, source in ( + (zmp.ZarrV3ArrayMetadata, V3_ARRAY_DOC), + (zmp.ZarrV2ArrayMetadata, V2_ARRAY_DOC), + ): + doc = json.loads(json.dumps(source)) + doc["shape"] = [-1] + _assert_runtime_and_schema_reject(field_type, doc) + + +def test_v2_array_schema_rejects_empty_filters() -> None: + """The v2 schema mirrors the runtime one-or-more filter rule.""" + doc = json.loads(json.dumps(V2_ARRAY_DOC)) + doc["filters"] = [] + + _assert_runtime_and_schema_reject(zmp.ZarrV2ArrayMetadata, doc) + + +@pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) +def test_v3_array_schema_rejects_false_at_mandatory_extension_points(field: str) -> None: + """Mandatory v3 extension points cannot opt out of understanding.""" + doc = json.loads(json.dumps(V3_ARRAY_DOC)) + doc[field] = {"name": "example", "must_understand": False} + + _assert_runtime_and_schema_reject(zmp.ZarrV3ArrayMetadata, doc) + + +def test_metadata_field_schema_rejects_unknown_members() -> None: + """Named-configuration envelopes are closed in both runtime and schema validation.""" + _assert_runtime_and_schema_reject( + zmp.ZarrV3MetadataField, + {"name": "example", "unexpected": 1}, + ) + + +@pytest.mark.parametrize( + ("field_type", "source"), + [ + (zmp.ZarrV2ArrayMetadata, V2_ARRAY_DOC), + (zmp.ZarrV2GroupMetadata, V2_GROUP_DOC), + (zmp.ZarrV2ConsolidatedMetadata, V2_CONSOLIDATED_DOC), + ], +) +def test_v2_schema_rejects_unknown_document_members( + field_type: object, source: dict[str, object] +) -> None: + """Closed v2 merged documents expose their runtime boundary in JSON Schema.""" + doc = json.loads(json.dumps(source)) + doc["unexpected"] = 1 + + _assert_runtime_and_schema_reject(field_type, doc) + + +def test_v3_array_schema_allows_unknown_extension_fields() -> None: + """Schema constraints do not close the v3 top-level extension namespace.""" + doc = json.loads(json.dumps(V3_ARRAY_DOC)) + doc["vendor_extension"] = {"anything": [1, 2]} + adapter = TypeAdapter(zmp.ZarrV3ArrayMetadata) + + assert adapter.validate_python(doc).extra_fields == {"vendor_extension": {"anything": (1, 2)}} + assert Draft202012Validator(adapter.json_schema()).is_valid(doc) + + +def test_json_roundtrip() -> None: + """model_dump_json output re-validates to an equal pydantic model.""" + + class Manifest(BaseModel): + metadata: zmp.ZarrV3ArrayMetadata + + manifest = Manifest.model_validate({"metadata": V3_ARRAY_DOC}) + assert Manifest.model_validate_json(manifest.model_dump_json()) == manifest + + +def test_metadata_field_serializes_shorthand_and_false_object() -> None: + """The optional integration exposes the core model's canonical extension form.""" + adapter = TypeAdapter(zmp.ZarrV3MetadataField) + assert adapter.dump_python(adapter.validate_python({"name": "bytes"})) == "bytes" + assert adapter.dump_python( + adapter.validate_python({"name": "optional", "must_understand": False}) + ) == {"name": "optional", "must_understand": False} + + +def test_core_package_does_not_import_pydantic() -> None: + """Importing zarr_metadata (in a fresh interpreter) must not import + pydantic: the integration is opt-in via zarr_metadata.pydantic.""" + import subprocess + import sys + + code = "import sys, zarr_metadata; assert 'pydantic' not in sys.modules, 'leaked'" + subprocess.run([sys.executable, "-c", code], check=True) diff --git a/packages/zarr-metadata/tests/model/test_sentinel.py b/packages/zarr-metadata/tests/model/test_sentinel.py new file mode 100644 index 0000000000..252a4cdd30 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_sentinel.py @@ -0,0 +1,89 @@ +"""Tests for the pickling and copying behavior of the `UNSET` sentinel. + +The sentinel's contract is identity, so it must never be reconstructed from +state. typing_extensions >= 4.16 pickles sentinels by reference (a lookup of +the sentinel's name on its defining module), which preserves the singleton +across process boundaries; these tests pin that behavior, since models hold +`UNSET` as field values and must survive pickling and deep-copying. + +The model round-trip tests compare whole structures: dataclass equality +compares every field, and `UNSET` compares by identity, so an impostor +sentinel produced by state-based pickling would fail the equality check. +""" + +from __future__ import annotations + +import copy +import pickle + +import pytest +from typing_extensions import Sentinel + +from zarr_metadata.model import ( + UNSET, + ZarrV2ArrayMetadata, + ZarrV2GroupMetadata, + ZarrV3ArrayMetadata, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, +) + +# Whole-model cases covering the states we know are problematic for +# serialization: every optional-key field in the UNSET (absent) state, the +# same fields in the present state (including present-but-empty, which must +# stay distinct from absent), and UNSET nested inside consolidated metadata. +MODEL_CASES = { + "array-v3-dimension-names-unset": ZarrV3ArrayMetadata.create_default(shape=(4,)), + "array-v3-dimension-names-set": ZarrV3ArrayMetadata.create_default(shape=(2, 2)).update( + dimension_names=("x", None) + ), + "array-v2-attributes-unset": ZarrV2ArrayMetadata.create_default(shape=(4,)), + "array-v2-attributes-empty": ZarrV2ArrayMetadata.create_default(shape=(4,), attributes={}), + "group-v2-attributes-unset": ZarrV2GroupMetadata.create_default(), + "group-v2-attributes-set": ZarrV2GroupMetadata.create_default(attributes={"a": 1}), + "group-v3-consolidated-unset": ZarrV3GroupMetadata.create_default(), + "group-v3-consolidated-with-unset-inside": ZarrV3GroupMetadata.create_default( + consolidated_metadata=ZarrV3ConsolidatedMetadata( + metadata={ + "child": ZarrV3ArrayMetadata.create_default(shape=(4,)), + "subgroup": ZarrV3GroupMetadata.create_default(), + } + ) + ), +} + + +def test_unset_pickle_round_trip_preserves_identity() -> None: + restored = pickle.loads(pickle.dumps(UNSET)) + assert restored is UNSET + + +def test_unset_copy_preserves_identity() -> None: + assert copy.copy(UNSET) is UNSET + assert copy.deepcopy(UNSET) is UNSET + + +@pytest.mark.parametrize("model", MODEL_CASES.values(), ids=MODEL_CASES.keys()) +def test_model_pickle_round_trip( + model: ZarrV2ArrayMetadata | ZarrV3ArrayMetadata | ZarrV2GroupMetadata | ZarrV3GroupMetadata, +) -> None: + restored = pickle.loads(pickle.dumps(model)) + assert restored == model + + +@pytest.mark.parametrize("model", MODEL_CASES.values(), ids=MODEL_CASES.keys()) +def test_model_deepcopy( + model: ZarrV2ArrayMetadata | ZarrV3ArrayMetadata | ZarrV2GroupMetadata | ZarrV3GroupMetadata, +) -> None: + assert copy.deepcopy(model) == model + + +def test_non_importable_sentinel_fails_to_pickle() -> None: + """Sentinels pickle by reference, never by state. A sentinel that is not + an importable attribute of its module has no reference to pickle, so + dumping it must fail loudly — a successful dump here would mean the + implementation regressed to state-based pickling, which would produce + identity-breaking impostor objects on the receiving side.""" + local_sentinel = Sentinel("local_sentinel") + with pytest.raises((pickle.PicklingError, TypeError)): + pickle.dumps(local_sentinel) diff --git a/packages/zarr-metadata/tests/test_partial_equivalence.py b/packages/zarr-metadata/tests/test_partial_equivalence.py index 995a6a21e1..33492b2356 100644 --- a/packages/zarr-metadata/tests/test_partial_equivalence.py +++ b/packages/zarr-metadata/tests/test_partial_equivalence.py @@ -14,17 +14,17 @@ import pytest -from zarr_metadata.v2.array import ArrayMetadataV2, ArrayMetadataV2Partial -from zarr_metadata.v2.group import GroupMetadataV2, GroupMetadataV2Partial -from zarr_metadata.v3.array import ArrayMetadataV3, ArrayMetadataV3Partial -from zarr_metadata.v3.group import GroupMetadataV3, GroupMetadataV3Partial +from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON, ZarrV2ArrayMetadataJSONPartial +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON, ZarrV2GroupMetadataJSONPartial +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON, ZarrV3ArrayMetadataJSONPartial +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON, ZarrV3GroupMetadataJSONPartial # (full, partial) pairs to check. Add new pairs here as more are introduced. PAIRS: list[tuple[type, type]] = [ - (ArrayMetadataV3, ArrayMetadataV3Partial), - (GroupMetadataV3, GroupMetadataV3Partial), - (ArrayMetadataV2, ArrayMetadataV2Partial), - (GroupMetadataV2, GroupMetadataV2Partial), + (ZarrV3ArrayMetadataJSON, ZarrV3ArrayMetadataJSONPartial), + (ZarrV3GroupMetadataJSON, ZarrV3GroupMetadataJSONPartial), + (ZarrV2ArrayMetadataJSON, ZarrV2ArrayMetadataJSONPartial), + (ZarrV2GroupMetadataJSON, ZarrV2GroupMetadataJSONPartial), ] diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index d3270579c3..e65c680fd1 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -1,5 +1,7 @@ """Test that the curated front-door names are accessible from the top-level zarr_metadata package.""" +import importlib +import pkgutil import re from typing import get_args @@ -21,26 +23,43 @@ def _group_rank(s: str) -> int: EXPECTED = [ # Category A — metadata-document types - "ArrayMetadataV2", - "ArrayMetadataV2Partial", - "ZArrayMetadata", - "GroupMetadataV2", - "GroupMetadataV2Partial", - "ZGroupMetadata", - "ConsolidatedMetadataV2", - "ZAttrsMetadata", - "CodecMetadataV2", - "ArrayMetadataV3", - "ArrayMetadataV3Partial", - "ExtensionFieldV3", - "GroupMetadataV3", - "GroupMetadataV3Partial", - "ConsolidatedMetadataV3", - "NamedConfigV3", - "MetadataV3", + "ZarrV2ArrayMetadataJSON", + "ZarrV2ArrayMetadataJSONPartial", + "ZarrV2ZArrayJSON", + "ZarrV2GroupMetadataJSON", + "ZarrV2GroupMetadataJSONPartial", + "ZarrV2ZGroupJSON", + "ZarrV2ConsolidatedMetadataJSON", + "ZarrV2ZAttrsJSON", + "ZarrV2CodecMetadata", + "ZarrV3ArrayMetadataJSON", + "ZarrV3ArrayMetadataJSONPartial", + "ZarrV3ExtensionField", + "ZarrV3GroupMetadataJSON", + "ZarrV3GroupMetadataJSONPartial", + "ZarrV3ConsolidatedMetadataJSON", + "ZarrV3NamedConfigJSON", + "ZarrV3MetadataFieldJSON", "JSONValue", + # Category A' — metadata models (in-memory dataclasses over the documents) + "ZarrV2ArrayMetadata", + "ZarrV2ArrayMetadataPartial", + "ZarrV3ArrayMetadata", + "ZarrV3ArrayMetadataPartial", + "ZarrV2GroupMetadata", + "ZarrV2GroupMetadataPartial", + "ZarrV3GroupMetadata", + "ZarrV3GroupMetadataPartial", + "ZarrV2ConsolidatedMetadata", + "ZarrV3ConsolidatedMetadata", + "ZarrV3NamedConfig", + "ZarrV3MetadataField", + "ValidationProblem", + "MetadataValidationError", + "ProblemKind", + "UNSET", # v2 data-type encoding union - "DataTypeMetadataV2", + "ZarrV2DataTypeMetadata", # Category B — codec canonical unions "BloscCodecMetadata", "BytesCodecMetadata", @@ -129,9 +148,9 @@ def _group_rank(s: str) -> int: "RawBytesFillValue", # Category E — constant+Literal pairs "ARRAY_ORDER_V2", - "ArrayOrderV2", + "ZarrV2ArrayOrder", "ARRAY_DIMENSION_SEPARATOR_V2", - "ArrayDimensionSeparatorV2", + "ZarrV2ArrayDimensionSeparator", "ENDIANNESS", "Endianness", "BYTES_CODEC_NAME", @@ -200,6 +219,109 @@ def test_all_is_grouped_and_unique() -> None: assert len(zm.__all__) == len(set(zm.__all__)) +# --- naming grammar --------------------------------------------------------- + +# Core document/model names: the format version comes first (`ZarrV2` / +# `ZarrV3`), then the CamelCase entity, then an optional role suffix +# (`JSON`, `JSONPartial`, `Partial`, `StoreKey`) — validated loosely here +# because `JSON` decomposes into single-letter words under any strict +# word-splitting regex. +_CORE_NAME = re.compile(r"^ZarrV[23](?:[A-Z][a-z0-9]*)+$") + +# Zarr v3 extension-entity names: the registered entity comes first (`Blosc`, +# `Uint8`, ... — `V2` here is the *entity name* of the v2-compatibility chunk +# key encoding, not a format-version marker, which is always spelled +# `ZarrV2`/`ZarrV3`), followed by exactly one role suffix. +_EXTENSION_ROLES = ( + "CodecConfiguration", + "CodecMetadata", + "CodecName", + "CodecObject", + "ChunkGridConfiguration", + "ChunkGridMetadata", + "ChunkGridName", + "ChunkGridObject", + "ChunkKeyEncodingConfiguration", + "ChunkKeyEncodingMetadata", + "ChunkKeyEncodingName", + "ChunkKeyEncodingObject", + "ChunkKeyEncodingSeparator", + "DataTypeName", + "FillValue", + "Configuration", + "Component", +) +_EXTENSION_NAME = re.compile(r"^(?:[A-Z][a-z0-9]*)+?(?:" + "|".join(_EXTENSION_ROLES) + r")$") + +# Standalone vocabulary: scalar Literal aliases, structural helper shapes, and +# the validation diagnostics. Closed by hand — a new name belongs here only if +# it is genuinely role-less; anything document- or entity-shaped must fit the +# grammars above instead. +_STANDALONE_VOCAB = frozenset( + { + "Base64Bytes", + "BloscCName", + "BloscShuffle", + "CastOutOfRangeMode", + "CastRoundingMode", + "Endianness", + "HexFloat16", + "HexFloat32", + "HexFloat64", + "JSONValue", + "MetadataValidationError", + "NumpyDatetime64", + "NumpyTimeUnit", + "NumpyTimedelta64", + "ProblemKind", + "RectilinearDimSpec", + "ScalarMap", + "ScalarMapEntry", + "ShardingIndexLocation", + "Struct", + "StructField", + "ValidationProblem", + } +) + + +def _public_type_names() -> set[tuple[str, str]]: + """Every (module, CamelCase name) pair exported via a public `__all__`.""" + module_names = {"zarr_metadata"} + for info in pkgutil.walk_packages(zm.__path__, prefix="zarr_metadata."): + if not any(part.startswith("_") for part in info.name.split(".")[1:]): + module_names.add(info.name) + out: set[tuple[str, str]] = set() + for module_name in module_names: + module = importlib.import_module(module_name) + for name in getattr(module, "__all__", ()): + if name.startswith("_") or name.isupper() or name.islower(): + continue + out.add((module_name, name)) + return out + + +def test_public_type_names_comply_with_naming_grammar() -> None: + """Every public type name parses against the package naming grammar: + version-first core names, entity-plus-role extension names, or the closed + standalone vocabulary.""" + exported = _public_type_names() + violations = [ + f"{module}.{name}" + for module, name in sorted(exported) + if name not in _STANDALONE_VOCAB + and not _CORE_NAME.match(name) + and not _EXTENSION_NAME.match(name) + ] + assert not violations, f"names outside the naming grammar: {violations}" + + +def test_standalone_vocab_is_not_stale() -> None: + """Every allowlisted vocabulary name is still actually exported.""" + exported_names = {name for _, name in _public_type_names()} + assert exported_names >= _STANDALONE_VOCAB + + def test_promoted_pairs_drift() -> None: pairs = [ (zm.ENDIANNESS, zm.Endianness), @@ -209,7 +331,7 @@ def test_promoted_pairs_drift() -> None: (zm.NUMPY_TIME_UNIT, zm.NumpyTimeUnit), (zm.CAST_ROUNDING_MODE, zm.CastRoundingMode), (zm.CAST_OUT_OF_RANGE_MODE, zm.CastOutOfRangeMode), - (zm.ARRAY_ORDER_V2, zm.ArrayOrderV2), + (zm.ARRAY_ORDER_V2, zm.ZarrV2ArrayOrder), ] for const, lit in pairs: assert set(const) == set(get_args(lit)) diff --git a/packages/zarr-metadata/tests/v2/array/test_fixtures.py b/packages/zarr-metadata/tests/v2/array/test_fixtures.py index 1576aae8db..578d3647ab 100644 --- a/packages/zarr-metadata/tests/v2/array/test_fixtures.py +++ b/packages/zarr-metadata/tests/v2/array/test_fixtures.py @@ -1,7 +1,7 @@ """Decode v2 array metadata fixtures via pydantic. Each `*.json` file in this directory is a representative on-disk -`.zarray` that should validate cleanly as `ZArrayMetadata` (the strict +`.zarray` that should validate cleanly as `ZarrV2ZArrayJSON` (the strict on-disk shape). User attributes live in sibling `.zattrs` files and are not part of these fixtures. @@ -17,11 +17,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v2.array import ZArrayMetadata +from zarr_metadata.v2.array import ZarrV2ZArrayJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(ZArrayMetadata) +ADAPTER = TypeAdapter(ZarrV2ZArrayJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) diff --git a/packages/zarr-metadata/tests/v2/consolidated/test_fixtures.py b/packages/zarr-metadata/tests/v2/consolidated/test_fixtures.py index 9dad66d074..e802c5bef8 100644 --- a/packages/zarr-metadata/tests/v2/consolidated/test_fixtures.py +++ b/packages/zarr-metadata/tests/v2/consolidated/test_fixtures.py @@ -8,11 +8,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v2.consolidated import ConsolidatedMetadataV2 +from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(ConsolidatedMetadataV2) +ADAPTER = TypeAdapter(ZarrV2ConsolidatedMetadataJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) diff --git a/packages/zarr-metadata/tests/v2/group/test_fixtures.py b/packages/zarr-metadata/tests/v2/group/test_fixtures.py index 1ad88a577b..29652185b2 100644 --- a/packages/zarr-metadata/tests/v2/group/test_fixtures.py +++ b/packages/zarr-metadata/tests/v2/group/test_fixtures.py @@ -1,7 +1,7 @@ """Decode v2 group metadata fixtures via pydantic. Each `*.json` file in this directory is a representative on-disk -`.zgroup` that should validate cleanly as `ZGroupMetadata` (the strict +`.zgroup` that should validate cleanly as `ZarrV2ZGroupJSON` (the strict on-disk shape). User attributes live in sibling `.zattrs` files and are not part of these fixtures. """ @@ -14,11 +14,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v2.group import ZGroupMetadata +from zarr_metadata.v2.group import ZarrV2ZGroupJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(ZGroupMetadata) +ADAPTER = TypeAdapter(ZarrV2ZGroupJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) diff --git a/packages/zarr-metadata/tests/v3/array/test_fixtures.py b/packages/zarr-metadata/tests/v3/array/test_fixtures.py index fccd00d481..c84cc4042b 100644 --- a/packages/zarr-metadata/tests/v3/array/test_fixtures.py +++ b/packages/zarr-metadata/tests/v3/array/test_fixtures.py @@ -1,7 +1,7 @@ """Decode v3 array metadata fixtures via pydantic. Each `*.json` file in this directory is a representative on-disk -`zarr.json` that should validate cleanly as `ArrayMetadataV3`. +`zarr.json` that should validate cleanly as `ZarrV3ArrayMetadataJSON`. Fixtures are named for the variant they exercise (regular vs rectilinear grid, blosc/gzip/zstd/sharding_indexed codecs, named-config dtypes, optional fields, extra fields). @@ -15,11 +15,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v3.array import ArrayMetadataV3 +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(ArrayMetadataV3) +ADAPTER = TypeAdapter(ZarrV3ArrayMetadataJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) diff --git a/packages/zarr-metadata/tests/v3/array/with_extra_field.json b/packages/zarr-metadata/tests/v3/array/with_extra_field.json index 46a7f0f235..bd7a9f5b45 100644 --- a/packages/zarr-metadata/tests/v3/array/with_extra_field.json +++ b/packages/zarr-metadata/tests/v3/array/with_extra_field.json @@ -16,6 +16,6 @@ ], "my_custom_extension": { "must_understand": false, - "purpose": "exercise the extra_items=ExtensionFieldV3 path" + "purpose": "exercise the extra_items=ZarrV3ExtensionField path" } } diff --git a/packages/zarr-metadata/tests/v3/consolidated/test_fixtures.py b/packages/zarr-metadata/tests/v3/consolidated/test_fixtures.py index 4d9e300bae..d052b16986 100644 --- a/packages/zarr-metadata/tests/v3/consolidated/test_fixtures.py +++ b/packages/zarr-metadata/tests/v3/consolidated/test_fixtures.py @@ -8,11 +8,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v3.consolidated import ConsolidatedMetadataV3 +from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(ConsolidatedMetadataV3) +ADAPTER = TypeAdapter(ZarrV3ConsolidatedMetadataJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) diff --git a/packages/zarr-metadata/tests/v3/group/test_fixtures.py b/packages/zarr-metadata/tests/v3/group/test_fixtures.py index 2015d5ce96..ffcdedef2b 100644 --- a/packages/zarr-metadata/tests/v3/group/test_fixtures.py +++ b/packages/zarr-metadata/tests/v3/group/test_fixtures.py @@ -8,11 +8,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v3.group import GroupMetadataV3 +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(GroupMetadataV3) +ADAPTER = TypeAdapter(ZarrV3GroupMetadataJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index c751d6a31c..f5e614a051 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -1292,7 +1292,9 @@ async def open_like(a: ArrayLike, path: str, **kwargs: Any) -> AnyAsyncArray: path : str The path to the new array. **kwargs - Any keyword arguments to pass to the array constructor. + Additional keyword arguments passed to `open_array`. + If `mode` is omitted or `None`, it defaults to `"a"`. Pass `mode="r"` when + opening an existing array from a read-only store. Returns ------- @@ -1300,6 +1302,8 @@ async def open_like(a: ArrayLike, path: str, **kwargs: Any) -> AnyAsyncArray: The opened array. """ like_kwargs = _like_args(a) | kwargs + if like_kwargs.get("mode") is None: + like_kwargs["mode"] = "a" return await open_array(path=path, **like_kwargs) # type: ignore[arg-type] diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 3231837a04..dc12d5f7af 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -1399,11 +1399,13 @@ def open_like(a: ArrayLike, path: str, **kwargs: Any) -> AnyArray: path : str The path to the new array. **kwargs - Any keyword arguments to pass to the array constructor. + Additional keyword arguments passed to `open_array`. + If `mode` is omitted or `None`, it defaults to `"a"`. Pass `mode="r"` when + opening an existing array from a read-only store. Returns ------- - AsyncArray + Array The opened array. """ return Array(sync(async_api.open_like(a, path=path, **kwargs))) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 0aaf89234e..922eaf1498 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -51,6 +51,7 @@ from zarr.core.metadata.io import save_metadata from zarr.core.sync import SyncMixin, sync from zarr.errors import ( + ArrayNotFoundError, ContainsArrayError, ContainsGroupError, GroupNotFoundError, @@ -820,6 +821,70 @@ async def get[DefaultT]( except KeyError: return default + async def get_array(self, path: str) -> AnyAsyncArray: + """Obtain an array member of this group, raising if it is absent or not an array. + + Parameters + ---------- + path : str + Path of the array relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subarray`. + + Returns + ------- + AsyncArray + The array at the given path. + + Raises + ------ + ArrayNotFoundError + If no node exists at the given path. + ContainsGroupError + If the node at the given path is a group rather than an array. + """ + store_path = self.store_path / path + try: + node = await self.getitem(path) + except KeyError as e: + msg = f"No array found in store {store_path.store!r} at path {store_path.path!r}" + raise ArrayNotFoundError(msg) from e + if isinstance(node, AsyncGroup): + msg = f"A group exists in store {store_path.store!r} at path {store_path.path!r}." + raise ContainsGroupError(msg) + return node + + async def get_group(self, path: str) -> AsyncGroup: + """Obtain a group member of this group, raising if it is absent or not a group. + + Parameters + ---------- + path : str + Path of the group relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subsubgroup`. + + Returns + ------- + AsyncGroup + The group at the given path. + + Raises + ------ + GroupNotFoundError + If no node exists at the given path. + ContainsArrayError + If the node at the given path is an array rather than a group. + """ + store_path = self.store_path / path + try: + node = await self.getitem(path) + except KeyError as e: + msg = f"No group found in store {store_path.store!r} at path {store_path.path!r}" + raise GroupNotFoundError(msg) from e + if isinstance(node, AsyncArray): + msg = f"An array exists in store {store_path.store!r} at path {store_path.path!r}." + raise ContainsArrayError(msg) + return node + async def _save_metadata(self, ensure_parents: bool = False) -> None: await save_metadata(self.store_path, self.metadata, ensure_parents=ensure_parents) @@ -1880,6 +1945,74 @@ def get[DefaultT]( except KeyError: return default + def get_array(self, path: str) -> AnyArray: + """Obtain an array member of this group, raising if it is absent or not an array. + + Parameters + ---------- + path : str + Path of the array relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subarray`. + + Returns + ------- + Array + The array at the given path. + + Raises + ------ + ArrayNotFoundError + If no node exists at the given path. + ContainsGroupError + If the node at the given path is a group rather than an array. + + Examples + -------- + ```python + import zarr + from zarr.core.group import Group + group = Group.from_store(zarr.storage.MemoryStore()) + group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") + group.get_array("subarray") + # + ``` + """ + return Array(self._sync(self._async_group.get_array(path))) + + def get_group(self, path: str) -> Group: + """Obtain a group member of this group, raising if it is absent or not a group. + + Parameters + ---------- + path : str + Path of the group relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subsubgroup`. + + Returns + ------- + Group + The group at the given path. + + Raises + ------ + GroupNotFoundError + If no node exists at the given path. + ContainsArrayError + If the node at the given path is an array rather than a group. + + Examples + -------- + ```python + import zarr + from zarr.core.group import Group + group = Group.from_store(zarr.storage.MemoryStore()) + group.create_group(name="subgroup") + group.get_group("subgroup") + # + ``` + """ + return Group(self._sync(self._async_group.get_group(path))) + def __delitem__(self, key: str) -> None: """Delete a group member. diff --git a/src/zarr/storage/_zip.py b/src/zarr/storage/_zip.py index 430b0c3e2a..69ae18bc2c 100644 --- a/src/zarr/storage/_zip.py +++ b/src/zarr/storage/_zip.py @@ -1,12 +1,13 @@ from __future__ import annotations +import io import os import shutil import threading import time import zipfile from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal +from typing import IO, TYPE_CHECKING, Any, Literal from zarr.abc.store import ( ByteRequest, @@ -23,14 +24,67 @@ ZipStoreAccessModeLiteral = Literal["r", "w", "a"] +class _RawReaderAdapter(io.RawIOBase): + """ + Adapt a minimal seekable reader to the `io` interface `zipfile` needs. + + Some file-like objects (e.g. `obstore.ReadableFile`) implement + `read`/`seek`/`tell` but are not `io.IOBase` instances, and their + `read` may return a buffer-protocol object rather than `bytes`. + Wrapping in this adapter plus `io.BufferedReader` yields real `bytes`. + + Reads are clamped to the bytes remaining before EOF: some readers + (obstore < 0.6) raise on short reads rather than returning fewer bytes. + The size is cached, which is safe because the adapter is only used for + read-only access. + """ + + def __init__(self, fileobj: IO[bytes]) -> None: + self._fileobj = fileobj + self._size: int | None = None + + def _get_size(self) -> int: + if self._size is None: + pos = self._fileobj.tell() + self._size = self._fileobj.seek(0, os.SEEK_END) + self._fileobj.seek(pos) + return self._size + + def readable(self) -> bool: + return True + + def seekable(self) -> bool: + return True + + def seek(self, pos: int, whence: int = 0) -> int: + return self._fileobj.seek(pos, whence) + + def tell(self) -> int: + return self._fileobj.tell() + + def readinto(self, b: Any) -> int: + n_requested = min(len(b), self._get_size() - self._fileobj.tell()) + if n_requested <= 0: + return 0 + data = self._fileobj.read(n_requested) + n = len(data) + b[:n] = memoryview(data) + return n + + class ZipStore(Store): """ Store using a ZIP file. Parameters ---------- - path : str - Location of file. + path : str, Path, or IO[bytes] + Location of file, or an open binary file object. A file object must + support `read`, `seek`, and `tell`; objects that are not `io.IOBase` + instances (e.g. an `obstore` reader) are adapted automatically but + can only be used for reading (`mode="r"`). The file object must stay + open for the lifetime of the store, and operations that require a + filesystem location (`clear`, `move`, pickling) are not supported. mode : str, optional One of 'r' to read an existing file, 'w' to truncate and write a new file, 'a' to append to an existing file, or 'x' to exclusively create @@ -58,16 +112,17 @@ class ZipStore(Store): supports_deletes: bool = False supports_listing: bool = True - path: Path + path: Path | None compression: int allowZip64: bool _zf: zipfile.ZipFile _lock: threading.RLock + _fileobj: IO[bytes] | None def __init__( self, - path: Path | str, + path: Path | str | IO[bytes], *, mode: ZipStoreAccessModeLiteral = "r", read_only: bool | None = None, @@ -81,8 +136,28 @@ def __init__( if isinstance(path, str): path = Path(path) - assert isinstance(path, Path) - self.path = path # root? + if isinstance(path, Path): + self.path = path # root? + self._fileobj = None + else: + self.path = None + if not isinstance(path, io.IOBase): + if not all( + callable(getattr(path, attr, None)) for attr in ("read", "seek", "tell") + ): + raise TypeError( + f"expected a path or an open binary file object supporting " + f"read/seek/tell, got {type(path).__name__}" + ) + if mode != "r": + raise TypeError( + f"a file object that is not an io.IOBase instance can only be " + f"opened for reading (mode='r', got mode={mode!r})" + ) + # e.g. an obstore ReadableFile: readable and seekable, but + # not an io object and reads may not return bytes + path = io.BufferedReader(_RawReaderAdapter(path)) + self._fileobj = path self._zmode = mode self.compression = compression @@ -95,7 +170,7 @@ def _sync_open(self) -> None: self._lock = threading.RLock() self._zf = zipfile.ZipFile( - self.path, + self.path if self.path is not None else self._fileobj, # type: ignore[arg-type] mode=self._zmode, compression=self.compression, allowZip64=self.allowZip64, @@ -107,6 +182,13 @@ async def _open(self) -> None: self._sync_open() def __getstate__(self) -> dict[str, Any]: + if self.path is None: + # A path-backed store pickles its path and reopens the file on + # unpickling; an open file object cannot be serialized that way. + raise TypeError( + "cannot pickle a ZipStore backed by a file-like object; " + "construct the store from a path instead" + ) # We need a copy to not modify the state of the original store state = self.__dict__.copy() for attr in ["_zf", "_lock"]: @@ -130,6 +212,10 @@ async def clear(self) -> None: # docstring inherited with self._lock: self._check_writable() + if self.path is None: + raise NotImplementedError( + "clear() is not supported for a ZipStore backed by a file-like object" + ) self._zf.close() os.remove(self.path) self._zf = zipfile.ZipFile( @@ -137,13 +223,19 @@ async def clear(self) -> None: ) def __str__(self) -> str: + if self.path is None: + return f"zip://{self._fileobj!r}" return f"zip://{self.path}" def __repr__(self) -> str: return f"ZipStore('{self}')" def __eq__(self, other: object) -> bool: - return isinstance(other, type(self)) and self.path == other.path + return ( + isinstance(other, type(self)) + and self.path == other.path + and self._fileobj is other._fileobj + ) def _get( self, @@ -297,6 +389,10 @@ async def move(self, path: Path | str) -> None: """ Move the store to another path. """ + if self.path is None: + raise NotImplementedError( + "move() is not supported for a ZipStore backed by a file-like object" + ) if isinstance(path, str): path = Path(path) self.close() diff --git a/tests/test_api.py b/tests/test_api.py index 1b4414ae63..cbe8ea3b44 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -168,6 +168,53 @@ async def test_array_like_creation( assert np.all(Array(new_arr)[:] == expect_fill) +@pytest.mark.parametrize("mode_kwargs", [{}, {"mode": None}]) +async def test_open_like_creates_array_by_default( + zarr_format: ZarrFormat, mode_kwargs: dict[str, None] +) -> None: + ref_arr = zarr.create_array( + store={}, + shape=(11, 12), + dtype="uint8", + chunks=(11, 12), + zarr_format=zarr_format, + fill_value=100, + ) + + new_arr = await zarr.api.asynchronous.open_like( + ref_arr, + path="foo", + store={}, + zarr_format=zarr_format, + **mode_kwargs, + ) + + assert new_arr.shape == ref_arr.shape + assert new_arr.chunks == ref_arr.chunks + assert new_arr.dtype == ref_arr.dtype + assert np.all(Array(new_arr)[:] == ref_arr.fill_value) + + +async def test_open_like_default_mode_rejects_read_only_store( + zarr_format: ZarrFormat, +) -> None: + ref_arr = zarr.create_array( + store={}, + shape=(11, 12), + dtype="uint8", + chunks=(11, 12), + zarr_format=zarr_format, + ) + + with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"): + await zarr.api.asynchronous.open_like( + ref_arr, + path="foo", + store=MemoryStore(read_only=True), + zarr_format=zarr_format, + ) + + # TODO: parametrize over everything this function takes @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_create_array(store: Store, zarr_format: ZarrFormat) -> None: diff --git a/tests/test_group.py b/tests/test_group.py index 1acd5551ca..29377a5392 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -41,8 +41,10 @@ from zarr.core.metadata.v3 import ArrayV3Metadata from zarr.core.sync import _collect_aiterator, sync from zarr.errors import ( + ArrayNotFoundError, ContainsArrayError, ContainsGroupError, + GroupNotFoundError, MetadataValidationError, ZarrUserWarning, ) @@ -101,7 +103,7 @@ async def test_create_creates_parents(store: Store, zarr_format: ZarrFormat) -> root = await zarr.api.asynchronous.open_group( store=store, ) - agroup = await root.getitem("a") + agroup = await root.get_group("a") assert agroup.attrs == {"key": "value"} # create a child node with a couple intermediates @@ -446,6 +448,77 @@ def test_group_get_with_default(store: Store, zarr_format: ZarrFormat) -> None: assert result.attrs["foo"] == "bar" +def test_group_get_array(store: Store, zarr_format: ZarrFormat) -> None: + """ + `Group.get_array` returns the array at the given path, for both direct child names + and nested paths, and the result is statically typed as an Array. + """ + group = Group.from_store(store, zarr_format=zarr_format) + subgroup = group.create_group(name="subgroup") + subarray = group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") + subsubarray = subgroup.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") + + observed = group.get_array("subarray") + assert isinstance(observed, Array) + assert observed == subarray + assert group.get_array("subgroup/subarray") == subsubarray + + +def test_group_get_array_missing(store: Store, zarr_format: ZarrFormat) -> None: + """ + `Group.get_array` raises `ArrayNotFoundError` when no node exists at the given path. + """ + group = Group.from_store(store, zarr_format=zarr_format) + with pytest.raises(ArrayNotFoundError, match="No array found in store"): + group.get_array("missing") + + +def test_group_get_array_wrong_node_type(store: Store, zarr_format: ZarrFormat) -> None: + """ + `Group.get_array` raises `ContainsGroupError` when the node at the given path is a + group rather than an array. + """ + group = Group.from_store(store, zarr_format=zarr_format) + group.create_group(name="subgroup") + with pytest.raises(ContainsGroupError, match="A group exists in store"): + group.get_array("subgroup") + + +def test_group_get_group(store: Store, zarr_format: ZarrFormat) -> None: + """ + `Group.get_group` returns the group at the given path, for both direct child names + and nested paths, and the result is statically typed as a Group. + """ + group = Group.from_store(store, zarr_format=zarr_format) + subgroup = group.create_group(name="subgroup") + subsubgroup = subgroup.create_group(name="subsubgroup") + + observed = group.get_group("subgroup") + assert isinstance(observed, Group) + assert observed == subgroup + assert group.get_group("subgroup/subsubgroup") == subsubgroup + + +def test_group_get_group_missing(store: Store, zarr_format: ZarrFormat) -> None: + """ + `Group.get_group` raises `GroupNotFoundError` when no node exists at the given path. + """ + group = Group.from_store(store, zarr_format=zarr_format) + with pytest.raises(GroupNotFoundError, match="No group found in store"): + group.get_group("missing") + + +def test_group_get_group_wrong_node_type(store: Store, zarr_format: ZarrFormat) -> None: + """ + `Group.get_group` raises `ContainsArrayError` when the node at the given path is an + array rather than a group. + """ + group = Group.from_store(store, zarr_format=zarr_format) + group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") + with pytest.raises(ContainsArrayError, match="An array exists in store"): + group.get_group("subarray") + + @pytest.mark.parametrize("consolidated", [True, False]) def test_group_delitem(store: Store, zarr_format: ZarrFormat, consolidated: bool) -> None: """ @@ -1469,7 +1542,7 @@ async def test_group_getitem_consolidated(self, store: Store) -> None: # On disk, we've consolidated all the metadata in the root zarr.json group = await zarr.api.asynchronous.open(store=store) - rg0 = await group.getitem("g0") + rg0 = await group.get_group("g0") expected = ConsolidatedMetadata( metadata={ @@ -1490,10 +1563,10 @@ async def test_group_getitem_consolidated(self, store: Store) -> None: ) assert rg0.metadata.consolidated_metadata == expected - rg1 = await rg0.getitem("g1") + rg1 = await rg0.get_group("g1") assert rg1.metadata.consolidated_metadata == expected.metadata["g1"].consolidated_metadata - rg2 = await rg1.getitem("g2") + rg2 = await rg1.get_group("g2") assert rg2.metadata.consolidated_metadata == ConsolidatedMetadata(metadata={}) async def test_group_delitem_consolidated(self, store: Store) -> None: diff --git a/tests/test_metadata/test_consolidated.py b/tests/test_metadata/test_consolidated.py index 3596d2bcaa..e6087435fe 100644 --- a/tests/test_metadata/test_consolidated.py +++ b/tests/test_metadata/test_consolidated.py @@ -111,12 +111,10 @@ async def test_getitem_consolidated_empty_leaf_group( group = await zarr.api.asynchronous.open_consolidated( store=memory_store, zarr_format=zarr_format ) - raw = await group.getitem("raw") - assert isinstance(raw, zarr.AsyncGroup) + raw = await group.get_group("raw") assert raw.metadata.consolidated_metadata is not None - varm = await raw.getitem("varm") - assert isinstance(varm, zarr.AsyncGroup) + varm = await raw.get_group("varm") assert varm.metadata.consolidated_metadata == ConsolidatedMetadata(metadata={}) async def test_open_consolidated_false_raises(self) -> None: @@ -770,8 +768,7 @@ async def test_absolute_path_for_subgroup(self, memory_store: zarr.storage.Memor await zarr.api.asynchronous.consolidate_metadata(memory_store) group = await zarr.api.asynchronous.open_group(store=memory_store) - subgroup = await group.getitem("/a") - assert isinstance(subgroup, AsyncGroup) + subgroup = await group.get_group("/a") members = [x async for x in subgroup.keys()] # noqa: SIM118 assert members == ["b"] diff --git a/tests/test_store/test_zip.py b/tests/test_store/test_zip.py index ed69114b51..32b18c5273 100644 --- a/tests/test_store/test_zip.py +++ b/tests/test_store/test_zip.py @@ -1,6 +1,8 @@ from __future__ import annotations +import io import os +import pickle import shutil import tempfile import zipfile @@ -20,7 +22,6 @@ import zarr from zarr import create_array from zarr.core.buffer import Buffer, cpu, default_buffer_prototype -from zarr.core.group import Group from zarr.core.sync import sync from zarr.storage import ZipStore from zarr.testing.store import StoreTests @@ -139,13 +140,13 @@ def test_externally_zipped_store(self, tmp_path: Path) -> None: zarr_path = tmp_path / "foo.zarr" root = zarr.open_group(store=zarr_path, mode="w") root.require_group("foo") - assert isinstance(foo := root["foo"], Group) # noqa: RUF018 + foo = root.get_group("foo") foo["bar"] = np.array([1]) shutil.make_archive(str(zarr_path), "zip", zarr_path) zip_path = tmp_path / "foo.zarr.zip" zipped = zarr.open_group(ZipStore(zip_path, mode="r"), mode="r") assert list(zipped.keys()) == list(root.keys()) - assert isinstance(group := zipped["foo"], Group) + group = zipped.get_group("foo") assert list(group.keys()) == list(group.keys()) async def test_list_without_explicit_open(self, tmp_path: Path) -> None: @@ -188,6 +189,163 @@ async def test_move(self, tmp_path: Path) -> None: assert np.array_equal(array[...], np.arange(10)) +class TestZipStoreFileObj: + """ZipStore backed by an open binary file-like object instead of a path.""" + + @pytest.fixture + def zip_bytes(self, tmp_path: Path) -> bytes: + path = tmp_path / "data.zip" + store = ZipStore(path, mode="w") + zarr.create_array(store, data=np.arange(10), chunks=(5,)) + store.close() + return path.read_bytes() + + def test_read_from_fileobj(self, zip_bytes: bytes) -> None: + # an existing archive can be read through any seekable binary reader + store = ZipStore(io.BytesIO(zip_bytes), mode="r") + array = zarr.open_array(store, mode="r") + assert np.array_equal(array[...], np.arange(10)) + assert store.path is None + + def test_write_to_fileobj(self) -> None: + # a writable file object receives the archive; the bytes it holds + # after close() are a complete, reopenable zip + buffer = io.BytesIO() + store = ZipStore(buffer, mode="w", read_only=False) + zarr.create_array(store, data=np.arange(4)) + store.close() + + roundtrip = ZipStore(io.BytesIO(buffer.getvalue()), mode="r") + array = zarr.open_array(roundtrip, mode="r") + assert np.array_equal(array[...], np.arange(4)) + + async def test_clear_unsupported(self, zip_bytes: bytes) -> None: + # clear() requires a filesystem location, so it raises a clear error + # for file-object-backed stores + store = ZipStore(io.BytesIO(zip_bytes), mode="a", read_only=False) + store._sync_open() + with pytest.raises(NotImplementedError, match="clear.*file-like"): + await store.clear() + + async def test_move_unsupported(self, zip_bytes: bytes) -> None: + # move() requires a filesystem location, so it raises a clear error + # for file-object-backed stores + store = ZipStore(io.BytesIO(zip_bytes), mode="a", read_only=False) + store._sync_open() + with pytest.raises(NotImplementedError, match="move.*file-like"): + await store.move("elsewhere.zip") + + def test_invalid_file_object_rejected(self) -> None: + # objects without read/seek/tell are rejected at construction, not + # deep inside zipfile + with pytest.raises(TypeError, match="read/seek/tell"): + ZipStore(42, mode="r") # type: ignore[arg-type] + + @pytest.mark.parametrize("mode", ["w", "a", "x"]) + def test_non_iobase_reader_write_modes_rejected(self, zip_bytes: bytes, mode: str) -> None: + # readers that are not io.IOBase instances are adapted for reading + # only; write modes are rejected at construction with a clear error + class MinimalReader: + def __init__(self, data: bytes) -> None: + self._buffer = io.BytesIO(data) + + def read(self, size: int, /) -> bytes: + return self._buffer.read(size) + + def seek(self, pos: int, whence: int = 0, /) -> int: + return self._buffer.seek(pos, whence) + + def tell(self) -> int: + return self._buffer.tell() + + with pytest.raises(TypeError, match="opened for reading"): + ZipStore(MinimalReader(zip_bytes), mode=mode, read_only=False) # type: ignore[arg-type] + + def test_fsspec_file(self, tmp_path: Path, zip_bytes: bytes) -> None: + # a file opened through fsspec (already an io.IOBase) is used directly; + # fsspec's local filesystem stands in for a remote one + fsspec = pytest.importorskip("fsspec") + + path = tmp_path / "fsspec.zip" + path.write_bytes(zip_bytes) + with fsspec.open(f"local://{path}", "rb") as fileobj: + store = ZipStore(fileobj, mode="r") + array = zarr.open_array(store, mode="r") + assert np.array_equal(array[...], np.arange(10)) + assert store.path is None + + def test_obstore_reader(self, tmp_path: Path, zip_bytes: bytes) -> None: + # obstore's ReadableFile is not an io.IOBase and its read() returns a + # buffer-protocol object; ZipStore adapts it via _RawReaderAdapter + obstore = pytest.importorskip("obstore") + from obstore.store import LocalStore as ObstoreLocalStore + + (tmp_path / "obstore.zip").write_bytes(zip_bytes) + reader = obstore.open_reader(ObstoreLocalStore(str(tmp_path)), "obstore.zip") + store = ZipStore(reader, mode="r") + array = zarr.open_array(store, mode="r") + assert np.array_equal(array[...], np.arange(10)) + + def test_raw_reader_adapter_eof(self) -> None: + from zarr.storage._zip import _RawReaderAdapter + + class MinimalReader: + """Non-io.IOBase reader exposing only read/seek/tell, like obstore.""" + + def __init__(self, data: bytes) -> None: + self._buffer = io.BytesIO(data) + + def read(self, size: int, /) -> bytes: + return self._buffer.read(size) + + def seek(self, pos: int, whence: int = 0, /) -> int: + return self._buffer.seek(pos, whence) + + def tell(self) -> int: + return self._buffer.tell() + + # the adapter must clamp reads to EOF: some readers (obstore < 0.6) + # raise on short reads instead of returning fewer bytes + data = b"0123456789" + adapter = _RawReaderAdapter(MinimalReader(data)) # type: ignore[arg-type] + + # A read straddling EOF returns only the remaining bytes. + adapter.seek(len(data) - 3) + buf = bytearray(8) + assert adapter.readinto(buf) == 3 + assert bytes(buf[:3]) == data[-3:] + + # A read at EOF returns 0. + assert adapter.tell() == len(data) + assert adapter.readinto(bytearray(8)) == 0 + + def test_pickle_fileobj_raises(self, zip_bytes: bytes) -> None: + # an open file object cannot be reliably serialized, so pickling a + # file-object-backed store raises with a pointer at the alternative + store = ZipStore(io.BytesIO(zip_bytes), mode="r") + with pytest.raises(TypeError, match="cannot pickle a ZipStore backed by a file-like"): + pickle.dumps(store) + + def test_pickle_path_backed_roundtrip(self, tmp_path: Path, zip_bytes: bytes) -> None: + # path-backed stores remain picklable: the path is serialized and the + # archive is reopened on unpickling + path = tmp_path / "pickled.zip" + path.write_bytes(zip_bytes) + store = ZipStore(path, mode="r") + unpickled = pickle.loads(pickle.dumps(store)) + array = zarr.open_array(unpickled, mode="r") + assert np.array_equal(array[...], np.arange(10)) + + def test_str_and_eq(self, zip_bytes: bytes) -> None: + # file-object-backed stores stringify with the object repr and + # compare equal only when backed by the very same file object + fileobj = io.BytesIO(zip_bytes) + store = ZipStore(fileobj, mode="r") + assert str(store).startswith("zip://<") + assert store == ZipStore(fileobj, mode="r") + assert store != ZipStore(io.BytesIO(zip_bytes), mode="r") + + class ZipStoreLifecycleMachine(RuleBasedStateMachine): """Drive a ZipStore through construct / open / write / close transitions.