From eefa424df0e7cfd1683ca7fb27996384141fea72 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:11:51 +0200 Subject: [PATCH 01/24] chore(deps): bump the actions group with 3 updates (#4181) Bumps the actions group with 3 updates: [actions/labeler](https://github.com/actions/labeler), [actions/attest](https://github.com/actions/attest) and [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action). Updates `actions/labeler` from 6.2.0 to 7.0.0 - [Release notes](https://github.com/actions/labeler/releases) - [Commits](https://github.com/actions/labeler/compare/b8dd2d9be0f68b860e7dae5dae7d772984eacd6d...bf12e9b00b37c5c0ca2b87b79b2daf7891dbda13) Updates `actions/attest` from 4.1.1 to 4.2.0 - [Release notes](https://github.com/actions/attest/releases) - [Changelog](https://github.com/actions/attest/blob/main/RELEASE.md) - [Commits](https://github.com/actions/attest/compare/a1948c3f048ba23858d222213b7c278aabede763...f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6) Updates `zizmorcore/zizmor-action` from 0.5.7 to 0.6.0 - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/192e21d79ab29983730a13d1382995c2307fbcaa...6599ee8b7a49aef6a770f63d261d214911a7ce02) --- updated-dependencies: - dependency-name: actions/labeler dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: actions/attest dependency-version: 4.2.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: zizmorcore/zizmor-action dependency-version: 0.6.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/needs_release_notes.yml | 2 +- .github/workflows/releases.yml | 2 +- .github/workflows/zarr-metadata-release.yml | 4 ++-- .github/workflows/zizmor.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/needs_release_notes.yml b/.github/workflows/needs_release_notes.yml index fa555d1478..e001e8cd43 100644 --- a/.github/workflows/needs_release_notes.yml +++ b/.github/workflows/needs_release_notes.yml @@ -21,7 +21,7 @@ jobs: pull-requests: write # Required to add labels to PRs runs-on: ubuntu-latest steps: - - uses: actions/labeler@b8dd2d9be0f68b860e7dae5dae7d772984eacd6d # v6.2.0 + - uses: actions/labeler@bf12e9b00b37c5c0ca2b87b79b2daf7891dbda13 # v7.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} sync-labels: true diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 4d460f4a56..fe0d09f300 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -81,7 +81,7 @@ jobs: name: releases path: dist - name: Generate artifact attestation - uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0 with: subject-path: dist/* - name: Publish package to PyPI diff --git a/.github/workflows/zarr-metadata-release.yml b/.github/workflows/zarr-metadata-release.yml index 5021f79d2e..bc9ecf9871 100644 --- a/.github/workflows/zarr-metadata-release.yml +++ b/.github/workflows/zarr-metadata-release.yml @@ -82,7 +82,7 @@ jobs: path: dist - name: Generate artifact attestation - uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0 with: subject-path: dist/* @@ -107,7 +107,7 @@ jobs: path: dist - name: Generate artifact attestation - uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0 with: subject-path: dist/* diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 6250426bae..1567bea713 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -32,4 +32,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7 + uses: zizmorcore/zizmor-action@6599ee8b7a49aef6a770f63d261d214911a7ce02 # v0.6.0 From d1533e1cea576cba0466794bfbbe83afa3f8bfac Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 27 Jul 2026 21:48:06 +0200 Subject: [PATCH 02/24] docs: add blog --- docs/blog/index.md | 0 docs/blog/posts/3.3.0-release.md | 57 ++++++++++++++++++++++++++++++++ mkdocs.yml | 3 ++ 3 files changed, 60 insertions(+) create mode 100644 docs/blog/index.md create mode 100644 docs/blog/posts/3.3.0-release.md diff --git a/docs/blog/index.md b/docs/blog/index.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/blog/posts/3.3.0-release.md b/docs/blog/posts/3.3.0-release.md new file mode 100644 index 0000000000..7d2d456fd5 --- /dev/null +++ b/docs/blog/posts/3.3.0-release.md @@ -0,0 +1,57 @@ +# Zarr-Python 3.3.0 + +We're happy to announce the release of version 3.3.0 of Zarr-Python. It's been a while since our last release, +and we're bringing some exciting additions in 3.3.0. For the full release notes, see [NEEDS INPUT], otherwise stick around for an overview of two performance-centric highlights of this release. + +## Faster low-latency storage + +[NEEDS INPUT] + + +### The cost of async overhead + +Zarr-Python 3.x uses async routines for fetching data and decoding chunks. In terms of code, this means our store (data fetching) and codec (chunk decoding) APIs are both async. This makes +I/O against high-latency storage backends like cloud object storage efficient. But for *low-latency* storage, like in-process memory or the file system, async routines add measurable overhead and offer no benefit. Async only adds value when there's work to be done while waiting for I/O to complete, but when I/O latency is low, it completes too quickly to run anything while waiting, and we are left paying the performance bill for obligatory async task scheduling that offered no value. + +This performance problem became acute when Zarr-Python users reported that in-memory array indexing workloads ran *slower* in Zarr-Python 3.x relative to Zarr-Python 2.18 (issue [NEEDS INPUT]). Fortunately this performance regression had a straightforward fix (I don't say "easy" because it was a lot of work). + +### Synchronous execution restores performance + +If async overhead makes low-latency storage slow, does *removing* that overhead restore performance? Yes, it does! + +In a series of PRs ([NEEDS INPUT]) we defined synchronous versions of our storage and codec APIs, and then combined them in a new codec orchestration class called `FusedCodecPipeline`. The `FusedCodecPipeline` is an alternative to the default (the `BatchedCodecPipeline`) that gives large speedups for low-latency storage. + +On this author's 10-core Apple M4 laptop, the `FusedCodecPipeline` delivers the following results against memory-backed arrays: +- uncompressed writes are *~4 times faster* +- uncompressed reads are *~5 times faster* +- compressed writes are *~2 times faster* +- compressed reads are *~2 times faster* + +These numbers came from a [benchmark script](https://gist.github.com/d-v-b/f01da0980fdcafe51e8726e21e344632). Run this yourself to get a sense of how the `FusedCodecPipeline` works on your system. + +Getting good numbers requires choosing the right level of thread-based parallelism for your workload, which is part of the runtime configuration of the `FusedCodecPipeline`. For uncompressed chunks there's no CPU-bound work to do after fetching a chunk and so +thread-based parallelism is worse than useless and slows things down. But for compressed chunks, threading offers a substantial payoff. + +### How to use it + +[NEEDS INPUT] code example showing how to set the global config or use the `config` attribute of an array + +## Faster sharded reads + +[NEEDS INPUT] + +### How sharding works + +Chunks encoded with the `sharding_indexed` contain subchunks, or "inner chunks". For example, if the `chunk_grid` field of the array metadata declares an "outer chunk" size of, say `(10, 10)`, a `sharding_indexed` codec in the `codecs` field could declare an "inner chunk" size of `(5, 5)`. Readers accessing such a chunk will observe a stored object (a stream of bytes) that decodes to an array with size `(10, 10)` (the "outer chunk"), which is comprised of four separate, contiguous byte ranges that each decode to a `(5, 5)` inner chunk. Each inner chunk occupies its own byte range in the outer chunk. A reader can satisfy a request for 3 inner chunks by issuing 3 separate byte-range requests. That's what previous versions of Zarr-Python did. + +### Interval equivalence + +Byte ranges, being intervals, obey some combination rules: the values in two half-open intervals `[a, b), [b, c)` can be captured by the single interval `[a, c)`. That means a reader could get all three inner chunks with *one* byte-range request by requesting a range of bytes starting with the first byte of the first subchunk and ending with the last byte of the last subchunk. When individual requests are expensive, this kind of optimization is worth a lot; we could use one request instead of three, in the previous example. + +### Byte range coalescing + +We call this procedure -- rewriting `N` byte range requests into `M` byte range requests, where `M` <= `N` -- "byte range coalescing", and it's a new performance optimization shipping in Zarr-Python 3.3.0. It's on by default with base settings we think are good, so most users won't need to tune anything. If you do need to adjust some settings, see the full documentation of the feature: [NEEDS INPUT] link to official docs + +## Going faster + +Landing these two enhancements taught us a lot about the performance-sensitive areas in Zarr-Python. We could probably invest much more time in adding or changing abstractions, writing code for special cases, etc. But we should also look to wrapping lower-level libraries that simply have an easier path to performance by virtue of being written in a compiled language. The [`zarrs`](https://zarrs.dev/) project is an ecosystem of Zarr tools written in Rust, with [extremely high performance](https://book.zarrs.dev/#-zarrs-is-fast-). Is there a `zarrs` binding in Zarr-Python's future? I hope so! We are keenly observing development of [`zarrista`](https://developmentseed.org/zarrista/latest/) as a proof-of-concept for what a Python-`zarrs` binding layer might look like. Stay tuned! diff --git a/mkdocs.yml b/mkdocs.yml index 46bfc1764c..c8d74632d6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -94,6 +94,9 @@ nav: - ' zarr.zeros_like': api/zarr/functions/zeros_like.md - release-notes.md - contributing.md + - Blog: + - blog/index.md + - blog/posts/3.3.0-release.md hooks: - mkdocs_hooks.py From 0104cec57ba48ee9be97b7b903da7f0e28b0aa28 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Mon, 27 Jul 2026 21:56:57 +0200 Subject: [PATCH 03/24] Update docs/blog/posts/3.3.0-release.md Co-authored-by: Max Jones <14077947+maxrjones@users.noreply.github.com> --- docs/blog/posts/3.3.0-release.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/blog/posts/3.3.0-release.md b/docs/blog/posts/3.3.0-release.md index 7d2d456fd5..001cddfaf7 100644 --- a/docs/blog/posts/3.3.0-release.md +++ b/docs/blog/posts/3.3.0-release.md @@ -54,4 +54,4 @@ We call this procedure -- rewriting `N` byte range requests into `M` byte range ## Going faster -Landing these two enhancements taught us a lot about the performance-sensitive areas in Zarr-Python. We could probably invest much more time in adding or changing abstractions, writing code for special cases, etc. But we should also look to wrapping lower-level libraries that simply have an easier path to performance by virtue of being written in a compiled language. The [`zarrs`](https://zarrs.dev/) project is an ecosystem of Zarr tools written in Rust, with [extremely high performance](https://book.zarrs.dev/#-zarrs-is-fast-). Is there a `zarrs` binding in Zarr-Python's future? I hope so! We are keenly observing development of [`zarrista`](https://developmentseed.org/zarrista/latest/) as a proof-of-concept for what a Python-`zarrs` binding layer might look like. Stay tuned! +Landing these two enhancements taught us a lot about the performance-sensitive areas in Zarr-Python. We could probably invest much more time in tuning the performance of the Python library by adding or changing abstractions, writing code for special cases, etc. But we should also look to wrapping lower-level libraries that simply have an easier path to performance by virtue of being written in a compiled language. The [`zarrs`](https://zarrs.dev/) project is an ecosystem of Zarr tools written in Rust, with [extremely high performance](https://book.zarrs.dev/#-zarrs-is-fast-). Is there a `zarrs` binding in Zarr-Python's future? I hope so! We are keenly observing development of [`zarrista`](https://developmentseed.org/zarrista/latest/) as a proof-of-concept for what a Python-`zarrs` binding layer might look like. Stay tuned! From 4a1a44186919d63379e4eb91e571e31106cdae70 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 27 Jul 2026 21:58:17 +0200 Subject: [PATCH 04/24] docs(blog): render post list and fill in release-post details Configure the Material blog plugin so docs/blog/index.md renders a reverse-chronological post list instead of a blank page. The plugin ships with mkdocs-material, which is already pinned, so no new dependency is needed. Archive and category indexes are disabled -- one post doesn't justify the extra navigation. Fill in the [NEEDS INPUT] placeholders in the 3.3.0 post from the release notes and user guide: the #3524 regression report, the #3885 / #3925 / #3987 / #3004 pull requests, runnable config examples for both FusedCodecPipeline and the coalescing knobs, and links to the release notes and config guide. Also correct two accuracy issues: FusedCodecPipeline is opt-in and experimental (not on by default, unlike coalescing), and the sharding example now consistently says four inner chunks rather than switching to three mid-paragraph. Assisted-by: ClaudeCode:claude-opus-5 --- docs/blog/.authors.yml | 6 ++ docs/blog/index.md | 3 + docs/blog/posts/3.3.0-release.md | 114 ++++++++++++++++++++++++++++--- mkdocs.yml | 9 ++- 4 files changed, 121 insertions(+), 11 deletions(-) create mode 100644 docs/blog/.authors.yml diff --git a/docs/blog/.authors.yml b/docs/blog/.authors.yml new file mode 100644 index 0000000000..f756cdcaa9 --- /dev/null +++ b/docs/blog/.authors.yml @@ -0,0 +1,6 @@ +authors: + d-v-b: + name: Davis Bennett + description: Zarr-Python core developer + avatar: https://github.com/d-v-b.png + url: https://github.com/d-v-b diff --git a/docs/blog/index.md b/docs/blog/index.md index e69de29bb2..fca29e2578 100644 --- a/docs/blog/index.md +++ b/docs/blog/index.md @@ -0,0 +1,3 @@ +# Blog + +News, release highlights, and design notes from the Zarr-Python developers. diff --git a/docs/blog/posts/3.3.0-release.md b/docs/blog/posts/3.3.0-release.md index 7d2d456fd5..e659a781d8 100644 --- a/docs/blog/posts/3.3.0-release.md +++ b/docs/blog/posts/3.3.0-release.md @@ -1,27 +1,42 @@ +--- +date: 2026-07-15 +authors: + - d-v-b +categories: + - Release +--- + # Zarr-Python 3.3.0 We're happy to announce the release of version 3.3.0 of Zarr-Python. It's been a while since our last release, -and we're bringing some exciting additions in 3.3.0. For the full release notes, see [NEEDS INPUT], otherwise stick around for an overview of two performance-centric highlights of this release. +and we're bringing some exciting additions in 3.3.0. For the full release notes, see the [3.3.0 release notes](../../release-notes.md), otherwise stick around for an overview of two performance-centric highlights of this release. + + ## Faster low-latency storage -[NEEDS INPUT] - +Relevant issues and pull requests: + +- [#3524](https://github.com/zarr-developers/zarr-python/issues/3524) -- the performance report that started this work +- [#3885](https://github.com/zarr-developers/zarr-python/pull/3885) -- synchronous codec APIs and the `FusedCodecPipeline` ### The cost of async overhead Zarr-Python 3.x uses async routines for fetching data and decoding chunks. In terms of code, this means our store (data fetching) and codec (chunk decoding) APIs are both async. This makes I/O against high-latency storage backends like cloud object storage efficient. But for *low-latency* storage, like in-process memory or the file system, async routines add measurable overhead and offer no benefit. Async only adds value when there's work to be done while waiting for I/O to complete, but when I/O latency is low, it completes too quickly to run anything while waiting, and we are left paying the performance bill for obligatory async task scheduling that offered no value. -This performance problem became acute when Zarr-Python users reported that in-memory array indexing workloads ran *slower* in Zarr-Python 3.x relative to Zarr-Python 2.18 (issue [NEEDS INPUT]). Fortunately this performance regression had a straightforward fix (I don't say "easy" because it was a lot of work). +This performance problem became acute when Zarr-Python users reported that in-memory array indexing workloads ran *slower* in Zarr-Python 3.x relative to Zarr-Python 2.18 ([#3524](https://github.com/zarr-developers/zarr-python/issues/3524)). Fortunately this performance regression had a straightforward fix (I don't say "easy" because it was a lot of work). ### Synchronous execution restores performance If async overhead makes low-latency storage slow, does *removing* that overhead restore performance? Yes, it does! -In a series of PRs ([NEEDS INPUT]) we defined synchronous versions of our storage and codec APIs, and then combined them in a new codec orchestration class called `FusedCodecPipeline`. The `FusedCodecPipeline` is an alternative to the default (the `BatchedCodecPipeline`) that gives large speedups for low-latency storage. +In [#3885](https://github.com/zarr-developers/zarr-python/pull/3885) we defined synchronous versions of our storage and codec APIs -- the `SyncByteGetter` and `SyncByteSetter` protocols, plus a `get_ranges_sync` method on the `Store` ABC -- and then combined them in a new codec orchestration class called `FusedCodecPipeline`. The `FusedCodecPipeline` is an opt-in alternative to the default (the `BatchedCodecPipeline`) that gives large speedups for low-latency storage. It is currently marked [experimental](../../user-guide/experimental.md), so we may change it as we learn more; the default pipeline is untouched, and existing code keeps working unless you opt in. + +The win here is *not* a faster compressor or a different on-disk format -- the bytes written are byte-for-byte identical. It is the removal of async scheduling overhead (including some [nasty `asyncio.to_thread` overhead](https://github.com/python/cpython/issues/136084)), plus a few vectorized fast paths for dense, uncompressed shards. On this author's 10-core Apple M4 laptop, the `FusedCodecPipeline` delivers the following results against memory-backed arrays: + - uncompressed writes are *~4 times faster* - uncompressed reads are *~5 times faster* - compressed writes are *~2 times faster* @@ -34,23 +49,102 @@ thread-based parallelism is worse than useless and slows things down. But for co ### How to use it -[NEEDS INPUT] code example showing how to set the global config or use the `config` attribute of an array +Select the pipeline through the [runtime configuration](../../user-guide/config.md) by setting `codec_pipeline.path`. Set it globally to affect every array created or opened afterwards: + +```python exec="true" session="blog-330" source="above" +import zarr + +zarr.config.set( + {"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"} +) +``` + +Or scope it to a block of code by using `zarr.config.set` as a context manager, which is the safer choice if you only want the new pipeline for part of your program: + +```python exec="true" session="blog-330" source="above" result="ansi" +import numpy as np +import zarr +from zarr.storage import MemoryStore + +with zarr.config.set( + {"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"} +): + arr = zarr.create_array( + store=MemoryStore(), + shape=(1000, 1000), + chunks=(100, 100), + shards=(1000, 1000), + dtype="float32", + ) + arr[:] = np.random.random((1000, 1000)).astype("float32") + result = arr[:] + +print(result.shape) +``` + +Thread-based parallelism is configured separately, via `codec_pipeline.max_workers`. It defaults to `None`, meaning a pool sized to `os.cpu_count()`. As noted above, memory-backed and uncompressed workloads often do better with a single worker: + +```python exec="true" session="blog-330" source="above" +import zarr + +# A fixed-size thread pool for codec compute. +zarr.config.set({"codec_pipeline.max_workers": 8}) + +# Or "auto", sized to the number of CPUs. +zarr.config.set({"codec_pipeline.max_workers": None}) +``` + +To return to the default pipeline, set `codec_pipeline.path` back to the batched implementation: + +```python exec="true" session="blog-330" source="above" +import zarr + +zarr.config.set( + {"codec_pipeline.path": "zarr.core.codec_pipeline.BatchedCodecPipeline"} +) +``` ## Faster sharded reads -[NEEDS INPUT] +Relevant issues and pull requests: + +- [#3004](https://github.com/zarr-developers/zarr-python/pull/3004) -- optimize partial shard reads +- [#3925](https://github.com/zarr-developers/zarr-python/pull/3925) -- `Store.get_ranges` for concurrent, coalesced multi-range reads +- [#3987](https://github.com/zarr-developers/zarr-python/pull/3987) -- control coalescing through `ArrayConfig` and the runtime config ### How sharding works -Chunks encoded with the `sharding_indexed` contain subchunks, or "inner chunks". For example, if the `chunk_grid` field of the array metadata declares an "outer chunk" size of, say `(10, 10)`, a `sharding_indexed` codec in the `codecs` field could declare an "inner chunk" size of `(5, 5)`. Readers accessing such a chunk will observe a stored object (a stream of bytes) that decodes to an array with size `(10, 10)` (the "outer chunk"), which is comprised of four separate, contiguous byte ranges that each decode to a `(5, 5)` inner chunk. Each inner chunk occupies its own byte range in the outer chunk. A reader can satisfy a request for 3 inner chunks by issuing 3 separate byte-range requests. That's what previous versions of Zarr-Python did. +Chunks encoded with the `sharding_indexed` codec contain subchunks, or "inner chunks". For example, if the `chunk_grid` field of the array metadata declares an "outer chunk" size of, say `(10, 10)`, a `sharding_indexed` codec in the `codecs` field could declare an "inner chunk" size of `(5, 5)`. Readers accessing such a chunk will observe a stored object (a stream of bytes) that decodes to an array with size `(10, 10)` (the "outer chunk"), which is comprised of four separate, contiguous byte ranges that each decode to a `(5, 5)` inner chunk. Each inner chunk occupies its own byte range in the outer chunk. A reader can satisfy a request for all four inner chunks by issuing four separate byte-range requests. That's what previous versions of Zarr-Python did. ### Interval equivalence -Byte ranges, being intervals, obey some combination rules: the values in two half-open intervals `[a, b), [b, c)` can be captured by the single interval `[a, c)`. That means a reader could get all three inner chunks with *one* byte-range request by requesting a range of bytes starting with the first byte of the first subchunk and ending with the last byte of the last subchunk. When individual requests are expensive, this kind of optimization is worth a lot; we could use one request instead of three, in the previous example. +Byte ranges, being intervals, obey some combination rules: the values in two half-open intervals `[a, b), [b, c)` can be captured by the single interval `[a, c)`. That means a reader could get all four inner chunks with *one* byte-range request by requesting a range of bytes starting with the first byte of the first subchunk and ending with the last byte of the last subchunk. When individual requests are expensive, this kind of optimization is worth a lot; we could use one request instead of four, in the previous example. ### Byte range coalescing -We call this procedure -- rewriting `N` byte range requests into `M` byte range requests, where `M` <= `N` -- "byte range coalescing", and it's a new performance optimization shipping in Zarr-Python 3.3.0. It's on by default with base settings we think are good, so most users won't need to tune anything. If you do need to adjust some settings, see the full documentation of the feature: [NEEDS INPUT] link to official docs +We call this procedure -- rewriting `N` byte range requests into `M` byte range requests, where `M` <= `N` -- "byte range coalescing", and it's a new performance optimization shipping in Zarr-Python 3.3.0. Unlike the `FusedCodecPipeline`, this one is on by default with base settings we think are good, so most users won't need to tune anything. + +Two knobs control it, both documented in the [runtime configuration guide](../../user-guide/config.md). Nearby byte ranges in the same shard are merged into a single request when the gap between them is no larger than `array.sharding_coalesce_max_gap_bytes` (default 1 MiB) and the merged read stays within `array.sharding_coalesce_max_bytes` (default 16 MiB). The gap threshold is what trades wasted bytes against saved requests: raising it reads more data you didn't ask for, in exchange for fewer round trips. + +You can set them globally, or per array by passing `config={...}` to [`zarr.create_array`][]: + +```python exec="true" session="blog-330" source="above" result="ansi" +import zarr +from zarr.storage import MemoryStore + +arr = zarr.create_array( + store=MemoryStore(), + shape=(1000, 1000), + chunks=(100, 100), + shards=(1000, 1000), + dtype="float32", + config={ + "sharding_coalesce_max_gap_bytes": 4 << 20, # 4 MiB + "sharding_coalesce_max_bytes": 64 << 20, # 64 MiB + }, +) +print(arr.shape) +``` ## Going faster diff --git a/mkdocs.yml b/mkdocs.yml index c8d74632d6..0c9813f1ae 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -96,7 +96,6 @@ nav: - contributing.md - Blog: - blog/index.md - - blog/posts/3.3.0-release.md hooks: - mkdocs_hooks.py @@ -155,6 +154,14 @@ extra_css: plugins: - autorefs + - blog: + blog_dir: blog + post_dir: "{blog}/posts" + post_url_format: "{slug}" + # The blog is a simple reverse-chronological list of posts; the archive + # and category indexes add navigation we don't have the volume to justify. + archive: false + categories: false - search - markdown-exec - mkdocstrings: From 0b727571268c8962717ab5c23965c4642fe6bd36 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 27 Jul 2026 19:31:14 -0700 Subject: [PATCH 05/24] feat: ZipStore accepts open binary file-like objects (#4187) Allows constructing a ZipStore from any seekable binary reader, enabling zip archives on remote storage: - io objects (BytesIO, fsspec file objects) are used directly - minimal readers that are not io.IOBase instances and whose read() may return buffer-protocol objects rather than bytes (e.g. obstore.ReadableFile) are adapted via a small io.RawIOBase wrapper when opened for reading clear()/move() raise NotImplementedError for file-object-backed stores. Co-authored-by: Claude Fable 5 --- changes/4187.feature.md | 4 + docs/user-guide/storage.md | 13 +++ src/zarr/storage/_zip.py | 114 +++++++++++++++++++++++-- tests/test_store/test_zip.py | 159 +++++++++++++++++++++++++++++++++++ 4 files changed, 281 insertions(+), 9 deletions(-) create mode 100644 changes/4187.feature.md diff --git a/changes/4187.feature.md b/changes/4187.feature.md new file mode 100644 index 0000000000..87133e2034 --- /dev/null +++ b/changes/4187.feature.md @@ -0,0 +1,4 @@ +`ZipStore` now accepts an open binary file-like object in place of a path, enabling +zip archives on remote storage (e.g. a file opened with `fsspec` or an +`obstore.ReadableFile`). Operations that require a filesystem location +(`clear`, `move`) raise `NotImplementedError` for file-object-backed stores. diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index 7e0154b2a0..0ba6202c76 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -124,6 +124,19 @@ array = zarr.create_array(store=store, shape=(2,), dtype='float64') print(array) ``` +In place of a path, `ZipStore` also accepts an open binary file object (for +example a file opened with `fsspec`, or an `obstore` reader), enabling zip +archives on remote storage. The file must stay open for as long as the store +is in use: + +```python exec="true" session="storage" source="above" result="ansi" +store.close() +f = open('data.zip', mode='rb') # must stay open while the store is used +array = zarr.open_array(store=zarr.storage.ZipStore(f), mode='r') +print(array[:]) +f.close() +``` + ### Remote Store The [`zarr.storage.FsspecStore`][] stores the contents of a Zarr hierarchy following the same diff --git a/src/zarr/storage/_zip.py b/src/zarr/storage/_zip.py index 430b0c3e2a..69ae18bc2c 100644 --- a/src/zarr/storage/_zip.py +++ b/src/zarr/storage/_zip.py @@ -1,12 +1,13 @@ from __future__ import annotations +import io import os import shutil import threading import time import zipfile from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal +from typing import IO, TYPE_CHECKING, Any, Literal from zarr.abc.store import ( ByteRequest, @@ -23,14 +24,67 @@ ZipStoreAccessModeLiteral = Literal["r", "w", "a"] +class _RawReaderAdapter(io.RawIOBase): + """ + Adapt a minimal seekable reader to the `io` interface `zipfile` needs. + + Some file-like objects (e.g. `obstore.ReadableFile`) implement + `read`/`seek`/`tell` but are not `io.IOBase` instances, and their + `read` may return a buffer-protocol object rather than `bytes`. + Wrapping in this adapter plus `io.BufferedReader` yields real `bytes`. + + Reads are clamped to the bytes remaining before EOF: some readers + (obstore < 0.6) raise on short reads rather than returning fewer bytes. + The size is cached, which is safe because the adapter is only used for + read-only access. + """ + + def __init__(self, fileobj: IO[bytes]) -> None: + self._fileobj = fileobj + self._size: int | None = None + + def _get_size(self) -> int: + if self._size is None: + pos = self._fileobj.tell() + self._size = self._fileobj.seek(0, os.SEEK_END) + self._fileobj.seek(pos) + return self._size + + def readable(self) -> bool: + return True + + def seekable(self) -> bool: + return True + + def seek(self, pos: int, whence: int = 0) -> int: + return self._fileobj.seek(pos, whence) + + def tell(self) -> int: + return self._fileobj.tell() + + def readinto(self, b: Any) -> int: + n_requested = min(len(b), self._get_size() - self._fileobj.tell()) + if n_requested <= 0: + return 0 + data = self._fileobj.read(n_requested) + n = len(data) + b[:n] = memoryview(data) + return n + + class ZipStore(Store): """ Store using a ZIP file. Parameters ---------- - path : str - Location of file. + path : str, Path, or IO[bytes] + Location of file, or an open binary file object. A file object must + support `read`, `seek`, and `tell`; objects that are not `io.IOBase` + instances (e.g. an `obstore` reader) are adapted automatically but + can only be used for reading (`mode="r"`). The file object must stay + open for the lifetime of the store, and operations that require a + filesystem location (`clear`, `move`, pickling) are not supported. mode : str, optional One of 'r' to read an existing file, 'w' to truncate and write a new file, 'a' to append to an existing file, or 'x' to exclusively create @@ -58,16 +112,17 @@ class ZipStore(Store): supports_deletes: bool = False supports_listing: bool = True - path: Path + path: Path | None compression: int allowZip64: bool _zf: zipfile.ZipFile _lock: threading.RLock + _fileobj: IO[bytes] | None def __init__( self, - path: Path | str, + path: Path | str | IO[bytes], *, mode: ZipStoreAccessModeLiteral = "r", read_only: bool | None = None, @@ -81,8 +136,28 @@ def __init__( if isinstance(path, str): path = Path(path) - assert isinstance(path, Path) - self.path = path # root? + if isinstance(path, Path): + self.path = path # root? + self._fileobj = None + else: + self.path = None + if not isinstance(path, io.IOBase): + if not all( + callable(getattr(path, attr, None)) for attr in ("read", "seek", "tell") + ): + raise TypeError( + f"expected a path or an open binary file object supporting " + f"read/seek/tell, got {type(path).__name__}" + ) + if mode != "r": + raise TypeError( + f"a file object that is not an io.IOBase instance can only be " + f"opened for reading (mode='r', got mode={mode!r})" + ) + # e.g. an obstore ReadableFile: readable and seekable, but + # not an io object and reads may not return bytes + path = io.BufferedReader(_RawReaderAdapter(path)) + self._fileobj = path self._zmode = mode self.compression = compression @@ -95,7 +170,7 @@ def _sync_open(self) -> None: self._lock = threading.RLock() self._zf = zipfile.ZipFile( - self.path, + self.path if self.path is not None else self._fileobj, # type: ignore[arg-type] mode=self._zmode, compression=self.compression, allowZip64=self.allowZip64, @@ -107,6 +182,13 @@ async def _open(self) -> None: self._sync_open() def __getstate__(self) -> dict[str, Any]: + if self.path is None: + # A path-backed store pickles its path and reopens the file on + # unpickling; an open file object cannot be serialized that way. + raise TypeError( + "cannot pickle a ZipStore backed by a file-like object; " + "construct the store from a path instead" + ) # We need a copy to not modify the state of the original store state = self.__dict__.copy() for attr in ["_zf", "_lock"]: @@ -130,6 +212,10 @@ async def clear(self) -> None: # docstring inherited with self._lock: self._check_writable() + if self.path is None: + raise NotImplementedError( + "clear() is not supported for a ZipStore backed by a file-like object" + ) self._zf.close() os.remove(self.path) self._zf = zipfile.ZipFile( @@ -137,13 +223,19 @@ async def clear(self) -> None: ) def __str__(self) -> str: + if self.path is None: + return f"zip://{self._fileobj!r}" return f"zip://{self.path}" def __repr__(self) -> str: return f"ZipStore('{self}')" def __eq__(self, other: object) -> bool: - return isinstance(other, type(self)) and self.path == other.path + return ( + isinstance(other, type(self)) + and self.path == other.path + and self._fileobj is other._fileobj + ) def _get( self, @@ -297,6 +389,10 @@ async def move(self, path: Path | str) -> None: """ Move the store to another path. """ + if self.path is None: + raise NotImplementedError( + "move() is not supported for a ZipStore backed by a file-like object" + ) if isinstance(path, str): path = Path(path) self.close() diff --git a/tests/test_store/test_zip.py b/tests/test_store/test_zip.py index ed69114b51..0d8dadd18a 100644 --- a/tests/test_store/test_zip.py +++ b/tests/test_store/test_zip.py @@ -1,6 +1,8 @@ from __future__ import annotations +import io import os +import pickle import shutil import tempfile import zipfile @@ -188,6 +190,163 @@ async def test_move(self, tmp_path: Path) -> None: assert np.array_equal(array[...], np.arange(10)) +class TestZipStoreFileObj: + """ZipStore backed by an open binary file-like object instead of a path.""" + + @pytest.fixture + def zip_bytes(self, tmp_path: Path) -> bytes: + path = tmp_path / "data.zip" + store = ZipStore(path, mode="w") + zarr.create_array(store, data=np.arange(10), chunks=(5,)) + store.close() + return path.read_bytes() + + def test_read_from_fileobj(self, zip_bytes: bytes) -> None: + # an existing archive can be read through any seekable binary reader + store = ZipStore(io.BytesIO(zip_bytes), mode="r") + array = zarr.open_array(store, mode="r") + assert np.array_equal(array[...], np.arange(10)) + assert store.path is None + + def test_write_to_fileobj(self) -> None: + # a writable file object receives the archive; the bytes it holds + # after close() are a complete, reopenable zip + buffer = io.BytesIO() + store = ZipStore(buffer, mode="w", read_only=False) + zarr.create_array(store, data=np.arange(4)) + store.close() + + roundtrip = ZipStore(io.BytesIO(buffer.getvalue()), mode="r") + array = zarr.open_array(roundtrip, mode="r") + assert np.array_equal(array[...], np.arange(4)) + + async def test_clear_unsupported(self, zip_bytes: bytes) -> None: + # clear() requires a filesystem location, so it raises a clear error + # for file-object-backed stores + store = ZipStore(io.BytesIO(zip_bytes), mode="a", read_only=False) + store._sync_open() + with pytest.raises(NotImplementedError, match="clear.*file-like"): + await store.clear() + + async def test_move_unsupported(self, zip_bytes: bytes) -> None: + # move() requires a filesystem location, so it raises a clear error + # for file-object-backed stores + store = ZipStore(io.BytesIO(zip_bytes), mode="a", read_only=False) + store._sync_open() + with pytest.raises(NotImplementedError, match="move.*file-like"): + await store.move("elsewhere.zip") + + def test_invalid_file_object_rejected(self) -> None: + # objects without read/seek/tell are rejected at construction, not + # deep inside zipfile + with pytest.raises(TypeError, match="read/seek/tell"): + ZipStore(42, mode="r") # type: ignore[arg-type] + + @pytest.mark.parametrize("mode", ["w", "a", "x"]) + def test_non_iobase_reader_write_modes_rejected(self, zip_bytes: bytes, mode: str) -> None: + # readers that are not io.IOBase instances are adapted for reading + # only; write modes are rejected at construction with a clear error + class MinimalReader: + def __init__(self, data: bytes) -> None: + self._buffer = io.BytesIO(data) + + def read(self, size: int, /) -> bytes: + return self._buffer.read(size) + + def seek(self, pos: int, whence: int = 0, /) -> int: + return self._buffer.seek(pos, whence) + + def tell(self) -> int: + return self._buffer.tell() + + with pytest.raises(TypeError, match="opened for reading"): + ZipStore(MinimalReader(zip_bytes), mode=mode, read_only=False) # type: ignore[arg-type] + + def test_fsspec_file(self, tmp_path: Path, zip_bytes: bytes) -> None: + # a file opened through fsspec (already an io.IOBase) is used directly; + # fsspec's local filesystem stands in for a remote one + fsspec = pytest.importorskip("fsspec") + + path = tmp_path / "fsspec.zip" + path.write_bytes(zip_bytes) + with fsspec.open(f"local://{path}", "rb") as fileobj: + store = ZipStore(fileobj, mode="r") + array = zarr.open_array(store, mode="r") + assert np.array_equal(array[...], np.arange(10)) + assert store.path is None + + def test_obstore_reader(self, tmp_path: Path, zip_bytes: bytes) -> None: + # obstore's ReadableFile is not an io.IOBase and its read() returns a + # buffer-protocol object; ZipStore adapts it via _RawReaderAdapter + obstore = pytest.importorskip("obstore") + from obstore.store import LocalStore as ObstoreLocalStore + + (tmp_path / "obstore.zip").write_bytes(zip_bytes) + reader = obstore.open_reader(ObstoreLocalStore(str(tmp_path)), "obstore.zip") + store = ZipStore(reader, mode="r") + array = zarr.open_array(store, mode="r") + assert np.array_equal(array[...], np.arange(10)) + + def test_raw_reader_adapter_eof(self) -> None: + from zarr.storage._zip import _RawReaderAdapter + + class MinimalReader: + """Non-io.IOBase reader exposing only read/seek/tell, like obstore.""" + + def __init__(self, data: bytes) -> None: + self._buffer = io.BytesIO(data) + + def read(self, size: int, /) -> bytes: + return self._buffer.read(size) + + def seek(self, pos: int, whence: int = 0, /) -> int: + return self._buffer.seek(pos, whence) + + def tell(self) -> int: + return self._buffer.tell() + + # the adapter must clamp reads to EOF: some readers (obstore < 0.6) + # raise on short reads instead of returning fewer bytes + data = b"0123456789" + adapter = _RawReaderAdapter(MinimalReader(data)) # type: ignore[arg-type] + + # A read straddling EOF returns only the remaining bytes. + adapter.seek(len(data) - 3) + buf = bytearray(8) + assert adapter.readinto(buf) == 3 + assert bytes(buf[:3]) == data[-3:] + + # A read at EOF returns 0. + assert adapter.tell() == len(data) + assert adapter.readinto(bytearray(8)) == 0 + + def test_pickle_fileobj_raises(self, zip_bytes: bytes) -> None: + # an open file object cannot be reliably serialized, so pickling a + # file-object-backed store raises with a pointer at the alternative + store = ZipStore(io.BytesIO(zip_bytes), mode="r") + with pytest.raises(TypeError, match="cannot pickle a ZipStore backed by a file-like"): + pickle.dumps(store) + + def test_pickle_path_backed_roundtrip(self, tmp_path: Path, zip_bytes: bytes) -> None: + # path-backed stores remain picklable: the path is serialized and the + # archive is reopened on unpickling + path = tmp_path / "pickled.zip" + path.write_bytes(zip_bytes) + store = ZipStore(path, mode="r") + unpickled = pickle.loads(pickle.dumps(store)) + array = zarr.open_array(unpickled, mode="r") + assert np.array_equal(array[...], np.arange(10)) + + def test_str_and_eq(self, zip_bytes: bytes) -> None: + # file-object-backed stores stringify with the object repr and + # compare equal only when backed by the very same file object + fileobj = io.BytesIO(zip_bytes) + store = ZipStore(fileobj, mode="r") + assert str(store).startswith("zip://<") + assert store == ZipStore(fileobj, mode="r") + assert store != ZipStore(io.BytesIO(zip_bytes), mode="r") + + class ZipStoreLifecycleMachine(RuleBasedStateMachine): """Drive a ZipStore through construct / open / write / close transitions. From df8fce4b03df177e3bff19f0776e1487f4ea4eca Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 28 Jul 2026 10:25:42 +0200 Subject: [PATCH 06/24] docs: drop concrete performance figures from the 3.3.0 release note The `FusedCodecPipeline` entry advertised "up to ~24x writes / ~14x reads on many-chunks-per-shard layouts, more with compression". Those figures did not reproduce on a 10-core M4: the best measured result across the repo's own benchmark layouts was ~3.3x, and the many-chunks-per-shard layout the claim is scoped to is the one shape that gets no threading at all, because the pool fans out over outer chunks and that layout has a single outer chunk per operation. Also drop "no regressions on compute-bound workloads": zstd measured 0.4-0.5x (i.e. a regression) on single-shard layouts. Numbers that depend on hardware and layout belong in benchmark output, not in a release note, so state the improvement qualitatively instead. Assisted-by: ClaudeCode:claude-opus-5 --- docs/release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index 3fd8a5f360..e2332b21f9 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -14,7 +14,7 @@ concurrently. ([#3004](https://github.com/zarr-developers/zarr-python/issues/3004)) - Added a `subchunk_write_order` option to `ShardingCodec` to control the physical order of subchunks within a shard. Supported values are `morton`, `unordered`, `lexicographic`, and `colexicographic`. `unordered` makes no guarantee about subchunk layout. This setting affects only on-disk layout, not the data read back, and is not persisted in array metadata: it applies per codec instance and is not recovered when reopening a sharded array. ([#3826](https://github.com/zarr-developers/zarr-python/issues/3826)) - Added `SyncByteGetter` and `SyncByteSetter` runtime-checkable protocols and a `get_ranges_sync` method on the `Store` ABC. These let custom byte getters/setters opt into the synchronous codec pipeline's fast path for in-memory IO, which the sharding codec uses for its inner chunks. ([#3885](https://github.com/zarr-developers/zarr-python/issues/3885)) -- Added `FusedCodecPipeline`, an opt-in codec pipeline that runs codec compute synchronously and in bulk (avoiding the per-chunk async scheduling overhead of the default `BatchedCodecPipeline`), giving large speedups for sharded arrays (up to ~24x writes / ~14x reads on many-chunks-per-shard layouts, more with compression) and no regressions on compute-bound workloads. The default `BatchedCodecPipeline` is unchanged for standard configurations, so existing code keeps working unless you opt in; enable the new pipeline with `zarr.config.set({"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"})`. ([#3885](https://github.com/zarr-developers/zarr-python/issues/3885)) +- Added `FusedCodecPipeline`, an opt-in codec pipeline that runs codec compute synchronously and in bulk (avoiding the per-chunk async scheduling overhead of the default `BatchedCodecPipeline`), giving large speedups for sharded arrays. The default `BatchedCodecPipeline` is unchanged for standard configurations, so existing code keeps working unless you opt in; enable the new pipeline with `zarr.config.set({"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"})`. ([#3885](https://github.com/zarr-developers/zarr-python/issues/3885)) - Add `zarr.abc.store.Store.get_ranges` for concurrent, coalesced multi-range reads from a single key. The method is defined on the `Store` ABC with a default implementation built on `Store.get`, so every store inherits a working version; stores with native multi-range backends (e.g. `FsspecStore`) can override for efficiency. Coalescing knobs (`max_concurrency`, `max_gap_bytes`, `max_coalesced_bytes`) are passed as keyword arguments to `get_ranges`. Failures from underlying fetches surface as a `BaseExceptionGroup` (PEP 654); callers should use `except*` to filter for specific exception types such as `FileNotFoundError`. ([#3925](https://github.com/zarr-developers/zarr-python/issues/3925)) - Two new fields on `ArrayConfig` control how the sharding codec coalesces partial-shard reads: `sharding_coalesce_max_gap_bytes` (default 1 MiB) and `sharding_coalesce_max_bytes` (default 16 MiB). When reading multiple chunks from the same shard, nearby byte ranges are merged into a single request to the store if separated by no more than `sharding_coalesce_max_gap_bytes` and the merged read stays within `sharding_coalesce_max_bytes`. Defaults are seeded from the matching `array.sharding_coalesce_max_gap_bytes` / `array.sharding_coalesce_max_bytes` keys in [`zarr.config`][] at array-creation time, and can be overridden per array by passing `config={...}` to [`zarr.create_array`][]. ([#3987](https://github.com/zarr-developers/zarr-python/issues/3987)) From 02b54cc9afe3cab37145a690a4b47acaf4574085 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 28 Jul 2026 10:44:31 +0200 Subject: [PATCH 07/24] docs: add runnable examples for the 3.3.0 performance features Replace the personal gist link in the 3.3.0 blog post with two self-contained examples that ship with the docs, following the existing examples/ convention (PEP 723 header, README, docs page, nav entry). codec_pipeline_performance measures BatchedCodecPipeline against FusedCodecPipeline across three layouts, with and without compression, at one and at cpu_count workers. It reports speedups below 1.00x rather than hiding them, so the zstd single-shard regression is visible. sharding_coalescing demonstrates byte-range coalescing by counting the fetches the coalescer plans and timing them against a latency-wrapped store. Two details are needed to make the effect observable and are documented in the example: the selection must be strided (a contiguous read produces adjacent ranges that merge under any setting), and LatencyStore must not forward get_ranges to the wrapped store, or the latency is bypassed and every configuration looks identical. Both use timeit.Timer for timing rather than a hand-rolled loop, and operator.setitem/getitem via functools.partial rather than lambdas. Assisted-by: ClaudeCode:claude-opus-5 --- docs/blog/posts/3.3.0-release.md | 22 +- .../examples/codec_pipeline_performance.md | 7 + .../examples/sharding_coalescing.md | 7 + examples/codec_pipeline_performance/README.md | 52 +++++ .../codec_pipeline_performance.py | 191 +++++++++++++++++ examples/sharding_coalescing/README.md | 58 +++++ .../sharding_coalescing.py | 198 ++++++++++++++++++ mkdocs.yml | 2 + 8 files changed, 531 insertions(+), 6 deletions(-) create mode 100644 docs/user-guide/examples/codec_pipeline_performance.md create mode 100644 docs/user-guide/examples/sharding_coalescing.md create mode 100644 examples/codec_pipeline_performance/README.md create mode 100644 examples/codec_pipeline_performance/codec_pipeline_performance.py create mode 100644 examples/sharding_coalescing/README.md create mode 100644 examples/sharding_coalescing/sharding_coalescing.py diff --git a/docs/blog/posts/3.3.0-release.md b/docs/blog/posts/3.3.0-release.md index ed78a3c5c3..cebbcc92aa 100644 --- a/docs/blog/posts/3.3.0-release.md +++ b/docs/blog/posts/3.3.0-release.md @@ -25,7 +25,7 @@ Relevant issues and pull requests: Zarr-Python 3.x uses async routines for fetching data and decoding chunks. In terms of code, this means our store (data fetching) and codec (chunk decoding) APIs are both async. This makes I/O against high-latency storage backends like cloud object storage efficient. But for *low-latency* storage, like in-process memory or the file system, async routines add measurable overhead and offer no benefit. Async only adds value when there's work to be done while waiting for I/O to complete, but when I/O latency is low, it completes too quickly to run anything while waiting, and we are left paying the performance bill for obligatory async task scheduling that offered no value. -This performance problem became acute when Zarr-Python users reported that in-memory array indexing workloads ran *slower* in Zarr-Python 3.x relative to Zarr-Python 2.18 ([#3524](https://github.com/zarr-developers/zarr-python/issues/3524)). Fortunately this performance regression had a straightforward fix (I don't say "easy" because it was a lot of work). +This performance problem became acute when Zarr-Python users reported that in-memory array indexing workloads ran *slower* in Zarr-Python 3.1.3 relative to Zarr-Python 2.18.7 ([#3524](https://github.com/zarr-developers/zarr-python/issues/3524)). Fortunately this performance regression had a straightforward fix (I don't say "easy" because it was a lot of work). ### Synchronous execution restores performance @@ -42,7 +42,7 @@ On this author's 10-core Apple M4 laptop, the `FusedCodecPipeline` delivers the - compressed writes are *~2 times faster* - compressed reads are *~2 times faster* -These numbers came from a [benchmark script](https://gist.github.com/d-v-b/f01da0980fdcafe51e8726e21e344632). Run this yourself to get a sense of how the `FusedCodecPipeline` works on your system. +These numbers came from a [runnable example](../../user-guide/examples/codec_pipeline_performance.md) that ships with the documentation. Run it yourself to get a sense of how the `FusedCodecPipeline` behaves on your system -- the answer depends on your hardware, your array layout, and whether you compress. Getting good numbers requires choosing the right level of thread-based parallelism for your workload, which is part of the runtime configuration of the `FusedCodecPipeline`. For uncompressed chunks there's no CPU-bound work to do after fetching a chunk and so thread-based parallelism is worse than useless and slows things down. But for compressed chunks, threading offers a substantial payoff. @@ -82,15 +82,21 @@ with zarr.config.set( print(result.shape) ``` -Thread-based parallelism is configured separately, via `codec_pipeline.max_workers`. It defaults to `None`, meaning a pool sized to `os.cpu_count()`. As noted above, memory-backed and uncompressed workloads often do better with a single worker: +Thread-based parallelism is configured separately, via `codec_pipeline.max_workers`. It defaults to `None`, meaning a pool sized to `os.cpu_count()`. Note that this setting is read *only* by the `FusedCodecPipeline` -- the default `BatchedCodecPipeline` ignores it, so tuning it without opting in above does nothing. + +As noted, memory-backed and uncompressed workloads often do better with a single worker, which runs everything inline on the calling thread: ```python exec="true" session="blog-330" source="above" import zarr -# A fixed-size thread pool for codec compute. +# No thread pool: run codec compute inline. Often best for uncompressed, +# memory-backed arrays, where there's no CPU-bound work to overlap. +zarr.config.set({"codec_pipeline.max_workers": 1}) + +# A fixed-size thread pool, which pays off once compression is in play. zarr.config.set({"codec_pipeline.max_workers": 8}) -# Or "auto", sized to the number of CPUs. +# Or back to the default, sized to the number of CPUs. zarr.config.set({"codec_pipeline.max_workers": None}) ``` @@ -126,6 +132,8 @@ We call this procedure -- rewriting `N` byte range requests into `M` byte range Two knobs control it, both documented in the [runtime configuration guide](../../user-guide/config.md). Nearby byte ranges in the same shard are merged into a single request when the gap between them is no larger than `array.sharding_coalesce_max_gap_bytes` (default 1 MiB) and the merged read stays within `array.sharding_coalesce_max_bytes` (default 16 MiB). The gap threshold is what trades wasted bytes against saved requests: raising it reads more data you didn't ask for, in exchange for fewer round trips. +For a runnable demonstration -- counting the store requests saved and timing them against a store with simulated latency -- see the [sharded read coalescing example](../../user-guide/examples/sharding_coalescing.md). + You can set them globally, or per array by passing `config={...}` to [`zarr.create_array`][]: ```python exec="true" session="blog-330" source="above" result="ansi" @@ -148,4 +156,6 @@ print(arr.shape) ## Going faster -Landing these two enhancements taught us a lot about the performance-sensitive areas in Zarr-Python. We could probably invest much more time in tuning the performance of the Python library by adding or changing abstractions, writing code for special cases, etc. But we should also look to wrapping lower-level libraries that simply have an easier path to performance by virtue of being written in a compiled language. The [`zarrs`](https://zarrs.dev/) project is an ecosystem of Zarr tools written in Rust, with [extremely high performance](https://book.zarrs.dev/#-zarrs-is-fast-). Is there a `zarrs` binding in Zarr-Python's future? I hope so! We are keenly observing development of [`zarrista`](https://developmentseed.org/zarrista/latest/) as a proof-of-concept for what a Python-`zarrs` binding layer might look like. Stay tuned! +The work released in this PR is just the first step of a larger performance-oriented direction for Zarr-Python. Landing these two enhancements taught us a *lot* about the performance-sensitive areas of the library. We can and will invest more time in performance tuning, e.g. by adding or changing abstractions, writing code for special cases, etc. + +But we should also look to wrapping lower-level libraries that simply have an easier path to performance by virtue of being written in a compiled language. The [`zarrs`](https://zarrs.dev/) project is an ecosystem of Zarr tools written in Rust, with [extremely high performance](https://book.zarrs.dev/#-zarrs-is-fast-). Is there a `zarrs` binding in Zarr-Python's future? I hope so! We are keenly observing development of [`zarrista`](https://developmentseed.org/zarrista/latest/) as a proof-of-concept for what a Python-`zarrs` binding layer might look like. Stay tuned! diff --git a/docs/user-guide/examples/codec_pipeline_performance.md b/docs/user-guide/examples/codec_pipeline_performance.md new file mode 100644 index 0000000000..f21e31636e --- /dev/null +++ b/docs/user-guide/examples/codec_pipeline_performance.md @@ -0,0 +1,7 @@ +--8<-- "examples/codec_pipeline_performance/README.md" + +## Source Code + +```python exec="false" reason="pymdownx snippet include directive, not python source" +--8<-- "examples/codec_pipeline_performance/codec_pipeline_performance.py" +``` diff --git a/docs/user-guide/examples/sharding_coalescing.md b/docs/user-guide/examples/sharding_coalescing.md new file mode 100644 index 0000000000..8b2e054af5 --- /dev/null +++ b/docs/user-guide/examples/sharding_coalescing.md @@ -0,0 +1,7 @@ +--8<-- "examples/sharding_coalescing/README.md" + +## Source Code + +```python exec="false" reason="pymdownx snippet include directive, not python source" +--8<-- "examples/sharding_coalescing/sharding_coalescing.py" +``` diff --git a/examples/codec_pipeline_performance/README.md b/examples/codec_pipeline_performance/README.md new file mode 100644 index 0000000000..e590fe725a --- /dev/null +++ b/examples/codec_pipeline_performance/README.md @@ -0,0 +1,52 @@ +# Codec Pipeline Performance + +This example compares the default `BatchedCodecPipeline` against the opt-in +`FusedCodecPipeline` across several array layouts, with and without compression. + +A *codec pipeline* turns chunks of array data into stored bytes and back, running +the configured codecs and performing the storage IO. The default +`BatchedCodecPipeline` schedules both asynchronously -- roughly one coroutine per +chunk operation. That model pays off for high-latency stores, where there is +useful work to do while waiting on IO. For low-latency stores (in-process memory, +the local filesystem) the IO completes too quickly for the overlap to be worth +its cost, and the async scheduling becomes pure overhead. + +`FusedCodecPipeline` runs codec compute and synchronous IO synchronously, +removing that overhead. It is +[experimental](https://zarr.readthedocs.io/en/stable/user-guide/experimental/) +and opt-in; the default pipeline is unchanged. + +## What it shows + +- How to select a pipeline with `zarr.config.set`, and why the array must be + *created* inside the config block: the pipeline class is resolved at array + construction time and then travels with the array. +- That the benefit depends strongly on layout and on whether compression is in + play. Some configurations are slower under the fused pipeline -- the script + reports speedups below 1.00x rather than hiding them. +- That `codec_pipeline.max_workers` is read only by `FusedCodecPipeline`; the + default pipeline ignores it entirely. + +## Running + +```bash +uv run codec_pipeline_performance.py +``` + +The script has no arguments and writes only to an in-memory store. + +## Interpreting the output + +The numbers are specific to your CPU, your Python build, and the layouts chosen +here. They are a measurement of your machine, not a published benchmark -- treat +a single run as indicative and re-measure against your own data and store before +switching pipelines in production. + +Two effects are worth watching for: + +- **Threading helps only when there is work to spread.** The pool parallelizes + over *outer* chunks. Uncompressed data has almost no CPU-bound codec work, so + more workers mostly add overhead. +- **Compression can reverse the result.** On layouts where the fused pipeline + cannot overlap compression work, the async pipeline's overlap wins and the + measured speedup drops below 1. diff --git a/examples/codec_pipeline_performance/codec_pipeline_performance.py b/examples/codec_pipeline_performance/codec_pipeline_performance.py new file mode 100644 index 0000000000..83b4dd1027 --- /dev/null +++ b/examples/codec_pipeline_performance/codec_pipeline_performance.py @@ -0,0 +1,191 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "zarr @ git+https://github.com/zarr-developers/zarr-python.git@main", +# "numpy", +# ] +# /// + +""" +Compare the performance of the `BatchedCodecPipeline` and the `FusedCodecPipeline`. + +The default `BatchedCodecPipeline` schedules storage IO and codec compute +asynchronously -- roughly one coroutine per chunk operation. That is the right +model for high-latency stores, where there is useful work to do while waiting +for IO. For *low-latency* stores (in-process memory, the local filesystem) the +IO completes too quickly for the overlap to pay for itself, and the async +scheduling is pure overhead. + +The `FusedCodecPipeline` runs codec compute and synchronous IO synchronously, +removing that overhead. This script measures the difference across a few +layouts so you can see where it helps and where it does not. + +Run it with: + + uv run codec_pipeline_performance.py + +Numbers are hardware-, layout-, and codec-dependent. Treat the output as a +measurement of *your* machine, not as a published benchmark. +""" + +from __future__ import annotations + +import operator +import os +import statistics +import timeit +from dataclasses import dataclass +from functools import partial +from typing import TYPE_CHECKING + +import numpy as np + +import zarr +from zarr.storage import MemoryStore + +if TYPE_CHECKING: + from collections.abc import Callable + +BATCHED = "zarr.core.codec_pipeline.BatchedCodecPipeline" +FUSED = "zarr.core.codec_pipeline.FusedCodecPipeline" + +# zstd level 0 is the Zarr v3 default compressor configuration. +ZSTD = {"name": "zstd", "configuration": {"level": 0, "checksum": False}} + + +@dataclass(frozen=True) +class Layout: + """One array geometry to measure.""" + + label: str + shape: tuple[int, ...] + chunks: tuple[int, ...] + shards: tuple[int, ...] | None + + @property + def detail(self) -> str: + if self.shards is None: + return f"{self.shape[0] // self.chunks[0]} chunks, no sharding" + chunks_per_shard = self.shards[0] // self.chunks[0] + n_shards = self.shape[0] // self.shards[0] + return f"{chunks_per_shard} chunks/shard x {n_shards} shards" + + +def make_data(n: int) -> np.ndarray: + """Semi-random but compressible data, so zstd has real work to do. + + Pure random data is incompressible and pure constant data compresses away + to nothing; either would make the compressed measurements misleading. + """ + period = 256 + pattern = (np.sin(np.linspace(0, 2 * np.pi, period)) * 50 + 128).round().astype(np.uint8) + values = np.tile(pattern, int(np.ceil(n / period)))[:n].astype(np.int16) + rng = np.random.default_rng(0) + values += rng.integers(-1, 2, size=n, dtype=np.int16) + return np.clip(values, 0, 255).astype(np.uint8) + + +def time_call(fn: Callable[[], object], repeat: int = 5) -> float: + """Median wall-clock seconds for one call to `fn`. + + `timeit.Timer` handles the mechanics: it uses `perf_counter` and disables + the cyclic garbage collector for the duration of each timing run, so a + collection triggered by earlier work cannot land in the middle of a + measurement. + + `number=1` because a single call here already moves tens of megabytes -- + the per-call overhead `timeit` is designed to amortize is irrelevant at + this scale, and looping inside a run would only measure the same thing + repeatedly. + + The median across repeats is deliberate: `timeit` documents that the + minimum is the best estimate of the achievable speed, but for IO-and-codec + work the typical case is more informative than the luckiest run, and a + mean would be dragged around by scheduler noise. + """ + return statistics.median(timeit.Timer(fn).repeat(repeat=repeat, number=1)) + + +def measure( + pipeline: str, + layout: Layout, + data: np.ndarray, + compressors: object, + max_workers: int | None, +) -> tuple[float, float]: + """Time one full write and one full read under `pipeline`. + + The pipeline class is read from the global config when the array is + constructed, so the array must be *created* inside the config block. Once + built, the array keeps that pipeline -- which is why the timed calls can + sit outside the `with` block. + """ + settings: dict[str, object] = {"codec_pipeline.path": pipeline} + if pipeline == FUSED: + settings["codec_pipeline.max_workers"] = max_workers + + with zarr.config.set(settings): + array = zarr.create_array( + store=MemoryStore(), + shape=layout.shape, + chunks=layout.chunks, + shards=layout.shards, + dtype="uint8", + compressors=compressors, + ) + + write = time_call(partial(operator.setitem, array, Ellipsis, data)) + read = time_call(partial(operator.getitem, array, Ellipsis)) + + # Guard against measuring something that silently produced wrong data. + if not np.array_equal(array[...], data): + raise AssertionError(f"round trip mismatch for {pipeline}") + return write, read + + +def main() -> None: + n_cpu = os.cpu_count() or 1 + size = 64_000_000 # 64 MB of uint8 + + layouts = ( + Layout("flat", (size,), (100_000,), None), + Layout("many shards", (size,), (100_000,), (100_000 * 100,)), + Layout("one big shard", (size,), (100_000,), (size,)), + ) + + data = make_data(size) + print(f"zarr {zarr.__version__} | {n_cpu} CPUs | {data.nbytes / 1e6:.0f} MB uint8\n") + + for compressor_label, compressors in (("uncompressed", None), ("zstd", ZSTD)): + print(f"===== {compressor_label} =====") + for layout in layouts: + print(f"\n{layout.label} ({layout.detail})") + print(f" {'workers':>7} {'op':<6} {'batched':>9} {'fused':>9} {'speedup':>9}") + base_w, base_r = measure(BATCHED, layout, data, compressors, None) + + # For uncompressed data there is no CPU-bound codec work to spread + # across threads, so a thread pool only adds overhead. With + # compression it can pay off. Measure both rather than asserting. + for workers in (1, n_cpu): + fused_w, fused_r = measure(FUSED, layout, data, compressors, workers) + for op, base, fused in (("write", base_w, fused_w), ("read", base_r, fused_r)): + print( + f" {workers:>7} {op:<6} {base:>8.3f}s {fused:>8.3f}s {base / fused:>8.2f}x" + ) + print() + + print( + "Notes:\n" + " * A speedup below 1.00x means the fused pipeline was slower for that case.\n" + " * Threading only helps when there is CPU-bound codec work to overlap and\n" + " more than one outer chunk to spread across threads. The thread pool\n" + " parallelizes over outer chunks, never over the inner chunks of a single\n" + " shard, so layouts with few outer chunks see little benefit from more\n" + " workers no matter how many inner chunks they contain.\n" + " * `codec_pipeline.max_workers` is read only by the FusedCodecPipeline;\n" + " the default BatchedCodecPipeline ignores it." + ) + + +if __name__ == "__main__": + main() diff --git a/examples/sharding_coalescing/README.md b/examples/sharding_coalescing/README.md new file mode 100644 index 0000000000..51afbab354 --- /dev/null +++ b/examples/sharding_coalescing/README.md @@ -0,0 +1,58 @@ +# Sharded Read Coalescing + +This example demonstrates byte-range coalescing for partial reads of sharded +arrays, a performance optimization added in Zarr-Python 3.3.0 and enabled by +default. + +A shard is one stored object containing many inner chunks, each occupying its own +byte range. Reading N inner chunks could mean N separate byte-range requests. +Because byte ranges are intervals, nearby ranges can be merged: `[a, b)` and +`[b, c)` together cover `[a, c)`, so a single request can serve both. Merging +trades reading some bytes you did not ask for against issuing fewer requests -- +worthwhile whenever a request is expensive, as with object storage. + +## What it shows + +- Reading scattered inner chunks from one shard with coalescing **off** issues + one store request per inner chunk; with the **default** settings the same read + collapses to a single request. +- The resulting wall-clock difference against a store with simulated latency. +- That coalescing changes only *how* data is fetched, never *what* is returned -- + the script asserts both configurations produce identical arrays. +- A case where coalescing changes nothing: a contiguous selection already has + adjacent byte ranges, so it merges under any setting. + +## Running + +```bash +uv run sharding_coalescing.py +``` + +## How the comparison is set up + +Two details make the effect observable, and both are worth understanding if you +adapt this script: + +- **The selection must have gaps.** A contiguous read produces adjacent byte + ranges that merge regardless of configuration. The strided selections skip + inner chunks, creating the gaps that the `sharding_coalesce_max_gap_bytes` + budget decides whether to bridge. +- **Latency must be charged per merged fetch.** `LatencyStore` adds latency in + `get`, but `WrapperStore.get_ranges` forwards directly to the wrapped store and + would bypass it. The example subclasses `LatencyStore` to inherit the `Store` + ABC's `get_ranges`, which runs the coalescer over its own `get`. + +## Configuration + +Two settings control the behavior, both settable globally via `zarr.config` or +per array via `config=` on `zarr.create_array` / `Array.with_config`: + +| Setting | Default | Meaning | +| --- | --- | --- | +| `sharding_coalesce_max_gap_bytes` | 1 MiB | Merge two ranges only if the gap between them is no larger than this | +| `sharding_coalesce_max_bytes` | 16 MiB | Never let a merged read exceed this size | + +Setting the gap to `0` merges only exactly-adjacent ranges, which approximates +the pre-3.3.0 behavior; that is how the example emulates the old path. Raising +the gap reads more unwanted bytes in exchange for fewer round trips -- the right +value depends on how expensive a request is against how fast your link is. diff --git a/examples/sharding_coalescing/sharding_coalescing.py b/examples/sharding_coalescing/sharding_coalescing.py new file mode 100644 index 0000000000..ae8557bc92 --- /dev/null +++ b/examples/sharding_coalescing/sharding_coalescing.py @@ -0,0 +1,198 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "zarr @ git+https://github.com/zarr-developers/zarr-python.git@main", +# "numpy", +# ] +# /// + +""" +Demonstrate byte-range coalescing for partial reads of sharded arrays. + +A shard is a single stored object holding many inner chunks, each occupying +its own byte range. Reading N inner chunks could mean issuing N separate +byte-range requests to the store. Because byte ranges are intervals, a reader +can instead merge nearby ranges: `[a, b)` and `[b, c)` together cover +`[a, c)`, so one request can serve both. Merging trades reading some bytes you +did not ask for against issuing fewer requests -- a good trade whenever a +request is expensive, which is the normal case for object storage. + +Zarr-Python 3.3.0 does this automatically. Two settings control it: + + * `sharding_coalesce_max_gap_bytes` (default 1 MiB) -- merge two ranges only + if the gap between them is no larger than this. + * `sharding_coalesce_max_bytes` (default 16 MiB) -- never let a merged read + exceed this size. + +Setting the gap to 0 disables merging of non-adjacent ranges, which +approximates the pre-3.3.0 behavior. This script compares the two, counting +store requests and measuring wall-clock time against a store with simulated +latency. + +Run it with: + + uv run sharding_coalescing.py +""" + +from __future__ import annotations + +import operator +import statistics +import timeit +from contextlib import contextmanager +from functools import partial +from typing import TYPE_CHECKING + +import numpy as np + +import zarr +import zarr.core._coalesce as coalesce_module +from zarr.abc.store import ByteRequest, RangeByteRequest, Store +from zarr.storage import MemoryStore +from zarr.testing.store import LatencyStore + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator, Sequence + +# Emulates the pre-3.3.0 behavior: with a zero gap budget, only ranges that are +# exactly adjacent get merged, so scattered inner chunks are fetched one by one. +NO_COALESCING = {"sharding_coalesce_max_gap_bytes": 0} + +# The shipped defaults. Spelled out here so the comparison is explicit rather +# than relying on whatever the global config happens to be. +DEFAULT_COALESCING = { + "sharding_coalesce_max_gap_bytes": 1 << 20, # 1 MiB + "sharding_coalesce_max_bytes": 16 << 20, # 16 MiB +} + +GET_LATENCY_S = 0.005 # 5 ms per request, a modest stand-in for object storage + + +class PerRequestLatencyStore(LatencyStore): + """A `LatencyStore` that charges latency per *merged byte-range fetch*. + + `WrapperStore.get_ranges` forwards straight to the wrapped store, which + would skip this class's latency-bearing `get` and make every configuration + look identical. Falling back to the `Store` ABC's `get_ranges` runs the + coalescer over `self.get` instead, so each merged fetch pays the latency -- + which is the cost coalescing exists to reduce. + """ + + get_ranges = Store.get_ranges + + +@contextmanager +def counting_requests() -> Iterator[Callable[[], int]]: + """Count the store fetches issued inside the block. + + Wraps the coalescing planner rather than the store: every merged group it + returns, plus every range it declined to merge, becomes exactly one fetch. + Counting here rather than at the store means the number reported is the + planner's decision, which is precisely what the settings control. + """ + original = coalesce_module.coalesce_ranges + total = 0 + + def counting_coalesce_ranges( + byte_ranges: Sequence[ByteRequest | None], + *, + max_gap_bytes: int, + max_coalesced_bytes: int, + ) -> tuple[ + list[list[tuple[int, RangeByteRequest]]], + list[tuple[int, ByteRequest | None]], + ]: + nonlocal total + groups, uncoalescable = original( + byte_ranges, + max_gap_bytes=max_gap_bytes, + max_coalesced_bytes=max_coalesced_bytes, + ) + total += len(groups) + len(uncoalescable) + return groups, uncoalescable + + coalesce_module.coalesce_ranges = counting_coalesce_ranges + try: + yield lambda: total + finally: + coalesce_module.coalesce_ranges = original + + +def measure_read(array: zarr.Array, selection: slice) -> tuple[int, float]: + """Return (store fetches, median seconds) for reading `selection`.""" + read = partial(operator.getitem, array, selection) + + with counting_requests() as fetches: + result = read() + requests = fetches() + + # `timeit.Timer` supplies the loop, `perf_counter`, and GC handling. The + # median of several runs keeps one unlucky run from dominating. + elapsed = statistics.median(timeit.Timer(read).repeat(repeat=5, number=1)) + + assert result.size > 0 # a read that returned nothing would time as "fast" + return requests, elapsed + + +def main() -> None: + n = 8192 + chunk = 64 + inner_chunks = n // chunk + + base = MemoryStore() + source = (np.arange(n, dtype="uint64") % 251).astype("uint8") + + # One shard holding every inner chunk, uncompressed so inner-chunk byte + # offsets stay predictable and the demonstration is easy to reason about. + writable = zarr.create_array( + store=base, shape=(n,), chunks=(chunk,), shards=(n,), dtype="uint8", compressors=None + ) + writable[:] = source + + store = PerRequestLatencyStore(base, get_latency=GET_LATENCY_S) + + print(f"zarr {zarr.__version__}") + print(f"array: {n} uint8 values, {inner_chunks} inner chunks of {chunk} in a single shard") + print(f"store: MemoryStore wrapped with {GET_LATENCY_S * 1000:.0f} ms of latency per request\n") + + # A strided selection touches inner chunks with unread chunks in between, + # so there are real gaps for the coalescer to bridge. A contiguous + # selection would merge under any setting, since its ranges are adjacent. + selections = { + "every 2nd inner chunk": slice(None, None, chunk * 2), + "every 4th inner chunk": slice(None, None, chunk * 4), + "contiguous quarter": slice(0, n // 4), + } + + header = f"{'selection':<24} {'coalescing':<12} {'requests':>9} {'time':>10}" + print(header) + print("-" * len(header)) + + for label, selection in selections.items(): + results = {} + for mode, config in (("off", NO_COALESCING), ("default", DEFAULT_COALESCING)): + array = zarr.open_array(store=store, mode="r").with_config(config) + requests, elapsed = measure_read(array, selection) + results[mode] = (requests, elapsed) + print(f"{label:<24} {mode:<12} {requests:>9} {elapsed * 1000:>9.1f}ms") + + off_requests, off_time = results["off"] + on_requests, on_time = results["default"] + if on_requests < off_requests: + print( + f"{'':<24} {'->':<12} " + f"{off_requests // on_requests:>8}x fewer {off_time / on_time:>9.1f}x faster" + ) + else: + print(f"{'':<24} {'->':<12} {'no change (ranges already adjacent)':>30}") + print() + + # Correctness is the point: coalescing must not change what you read back. + for mode, config in (("off", NO_COALESCING), ("default", DEFAULT_COALESCING)): + array = zarr.open_array(store=store, mode="r").with_config(config) + assert np.array_equal(array[::128], source[::128]), mode + print("Both configurations return identical data; coalescing only changes how it is fetched.") + + +if __name__ == "__main__": + main() diff --git a/mkdocs.yml b/mkdocs.yml index 0c9813f1ae..647bc40a2a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -31,6 +31,8 @@ nav: - Examples: - user-guide/examples/custom_dtype.md - user-guide/examples/rectilinear_chunks.md + - user-guide/examples/codec_pipeline_performance.md + - user-guide/examples/sharding_coalescing.md - API Reference: - api/zarr/index.md - ' zarr.abc': From 25324c4256d95305b638d068691fa9030c500f3f Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 28 Jul 2026 10:46:41 +0200 Subject: [PATCH 08/24] docs: use slice(None) instead of Ellipsis in the pipeline example Reads more idiomatically than `Ellipsis` for a whole-array selection, and naming it `everything` makes the three call sites self-describing. Assisted-by: ClaudeCode:claude-opus-5 --- .../codec_pipeline_performance.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/codec_pipeline_performance/codec_pipeline_performance.py b/examples/codec_pipeline_performance/codec_pipeline_performance.py index 83b4dd1027..9ef24f534f 100644 --- a/examples/codec_pipeline_performance/codec_pipeline_performance.py +++ b/examples/codec_pipeline_performance/codec_pipeline_performance.py @@ -134,11 +134,12 @@ def measure( compressors=compressors, ) - write = time_call(partial(operator.setitem, array, Ellipsis, data)) - read = time_call(partial(operator.getitem, array, Ellipsis)) + everything = slice(None) + write = time_call(partial(operator.setitem, array, everything, data)) + read = time_call(partial(operator.getitem, array, everything)) # Guard against measuring something that silently produced wrong data. - if not np.array_equal(array[...], data): + if not np.array_equal(array[everything], data): raise AssertionError(f"round trip mismatch for {pipeline}") return write, read From 8e579a4e7328aeeb569dc37c697e821a0b724650 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 28 Jul 2026 11:00:42 +0200 Subject: [PATCH 09/24] docs: update blog post --- docs/blog/posts/3.3.0-release.md | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/blog/posts/3.3.0-release.md b/docs/blog/posts/3.3.0-release.md index cebbcc92aa..f7a76f9112 100644 --- a/docs/blog/posts/3.3.0-release.md +++ b/docs/blog/posts/3.3.0-release.md @@ -8,8 +8,8 @@ categories: # Zarr-Python 3.3.0 -We're happy to announce the release of version 3.3.0 of Zarr-Python. It's been a while since our last release, -and we're bringing some exciting additions in 3.3.0. For the full release notes, see the [3.3.0 release notes](../../release-notes.md), otherwise stick around for an overview of two performance-centric highlights of this release. +We're happy to announce the release of version 3.3.0 of Zarr-Python. It's been a while since our last release ([3.2.1](https://github.com/zarr-developers/zarr-python/releases/tag/v3.2.1) dropped in May of this year), +and we're bringing some exciting additions to the latest version. For the full release notes, see the [3.3.0 release notes](../../release-notes.md), otherwise stick around for an overview of two performance-centric highlights of this release. @@ -33,7 +33,7 @@ If async overhead makes low-latency storage slow, does *removing* that overhead In [#3885](https://github.com/zarr-developers/zarr-python/pull/3885) we defined synchronous versions of our storage and codec APIs -- the `SyncByteGetter` and `SyncByteSetter` protocols, plus a `get_ranges_sync` method on the `Store` ABC -- and then combined them in a new codec orchestration class called `FusedCodecPipeline`. The `FusedCodecPipeline` is an opt-in alternative to the default (the `BatchedCodecPipeline`) that gives large speedups for low-latency storage. It is currently marked [experimental](../../user-guide/experimental.md), so we may change it as we learn more; the default pipeline is untouched, and existing code keeps working unless you opt in. -The win here is *not* a faster compressor or a different on-disk format -- the bytes written are byte-for-byte identical. It is the removal of async scheduling overhead (including some [nasty `asyncio.to_thread` overhead](https://github.com/python/cpython/issues/136084)), plus a few vectorized fast paths for dense, uncompressed shards. +The win here is *not* a faster compressor. It is the removal of async scheduling overhead (including some [nasty `asyncio.to_thread` overhead](https://github.com/python/cpython/issues/136084)), plus a few vectorized fast paths for dense, uncompressed shards. On this author's 10-core Apple M4 laptop, the `FusedCodecPipeline` delivers the following results against memory-backed arrays: @@ -42,9 +42,9 @@ On this author's 10-core Apple M4 laptop, the `FusedCodecPipeline` delivers the - compressed writes are *~2 times faster* - compressed reads are *~2 times faster* -These numbers came from a [runnable example](../../user-guide/examples/codec_pipeline_performance.md) that ships with the documentation. Run it yourself to get a sense of how the `FusedCodecPipeline` behaves on your system -- the answer depends on your hardware, your array layout, and whether you compress. +These numbers came from a [runnable example](../../user-guide/examples/codec_pipeline_performance.md) that ships with the documentation. Run it yourself to get a sense of how the `FusedCodecPipeline` behaves on your system -- when and how you use it depends on your hardware, your array layout, and how your chunks are compressed. What's certain is that for in-memory arrays, and arrays saved to the local file system, the `FusedCodecPipeline` is worth a try. -Getting good numbers requires choosing the right level of thread-based parallelism for your workload, which is part of the runtime configuration of the `FusedCodecPipeline`. For uncompressed chunks there's no CPU-bound work to do after fetching a chunk and so +Getting good numbers requires choosing the right level of thread-based parallelism for your workload, which is part of the configuration of the `FusedCodecPipeline`. For uncompressed chunks there's no CPU-bound work to do after fetching a chunk and so thread-based parallelism is worse than useless and slows things down. But for compressed chunks, threading offers a substantial payoff. ### How to use it @@ -120,17 +120,21 @@ Relevant issues and pull requests: ### How sharding works -Chunks encoded with the `sharding_indexed` codec contain subchunks, or "inner chunks". For example, if the `chunk_grid` field of the array metadata declares an "outer chunk" size of, say `(10, 10)`, a `sharding_indexed` codec in the `codecs` field could declare an "inner chunk" size of `(5, 5)`. Readers accessing such a chunk will observe a stored object (a stream of bytes) that decodes to an array with size `(10, 10)` (the "outer chunk"), which is comprised of four separate, contiguous byte ranges that each decode to a `(5, 5)` inner chunk. Each inner chunk occupies its own byte range in the outer chunk. A reader can satisfy a request for all four inner chunks by issuing four separate byte-range requests. That's what previous versions of Zarr-Python did. +Chunks encoded with the `sharding_indexed` codec contain subchunks, or "inner chunks". For example, if the `chunk_grid` field of the array metadata declares an "outer chunk" size of, say `(10, 10)`, a `sharding_indexed` codec in the `codecs` field could declare an "inner chunk" size of `(5, 5)`. Readers accessing such a chunk will observe a stored object (a stream of bytes) that decodes to an array with size `(10, 10)` (the "outer chunk"), which is comprised of four separate, contiguous byte ranges that each decode to a `(5, 5)` inner chunk. Each inner chunk occupies its own byte range in the outer chunk. + +A reader can satisfy a request for all four inner chunks by issuing four separate byte-range requests, or by making a *single* request for a byte range that spans all four inner chunks. The latter option is nice because it cuts down on the number of requests we need. Historically Zarr-Python used this optimization when reading entire outer chunks; in 3.3.0, we use this optimization in more cases, resulting in more efficient I/O patterns for sharded reads. ### Interval equivalence -Byte ranges, being intervals, obey some combination rules: the values in two half-open intervals `[a, b), [b, c)` can be captured by the single interval `[a, c)`. That means a reader could get all four inner chunks with *one* byte-range request by requesting a range of bytes starting with the first byte of the first subchunk and ending with the last byte of the last subchunk. When individual requests are expensive, this kind of optimization is worth a lot; we could use one request instead of four, in the previous example. +Byte ranges, being intervals, obey some combination rules: the values in two half-open intervals `[a, b), [b, c)` can be captured by the single interval `[a, c)`. That means a reader can get multiple inner chunks with *one* byte-range request by requesting a range of bytes starting with the first byte of the first subchunk and ending with the last byte of the last subchunk. When individual requests are expensive, this kind of optimization is worth a lot. + +The requested inner chunks are not necessarily contiguous -- there might be a byte range gap between them. As long as that gap is not too big, its often efficient to fetch the entire byte range, gap included, and pick out the inner chunk byte ranges after I/O is done. ### Byte range coalescing We call this procedure -- rewriting `N` byte range requests into `M` byte range requests, where `M` <= `N` -- "byte range coalescing", and it's a new performance optimization shipping in Zarr-Python 3.3.0. Unlike the `FusedCodecPipeline`, this one is on by default with base settings we think are good, so most users won't need to tune anything. -Two knobs control it, both documented in the [runtime configuration guide](../../user-guide/config.md). Nearby byte ranges in the same shard are merged into a single request when the gap between them is no larger than `array.sharding_coalesce_max_gap_bytes` (default 1 MiB) and the merged read stays within `array.sharding_coalesce_max_bytes` (default 16 MiB). The gap threshold is what trades wasted bytes against saved requests: raising it reads more data you didn't ask for, in exchange for fewer round trips. +Two knobs control it, both documented in the [runtime configuration guide](../../user-guide/config.md). Nearby byte ranges in the same shard are merged into a single request when the gap between them is no larger than `array.sharding_coalesce_max_gap_bytes` (default 1 MiB) and the merged read stays within `array.sharding_coalesce_max_bytes` (default 16 MiB). The gap threshold is what trades wasted bytes against saved requests: raising it reads more data you didn't ask for, in exchange for fewer requests. For a runnable demonstration -- counting the store requests saved and timing them against a store with simulated latency -- see the [sharded read coalescing example](../../user-guide/examples/sharding_coalescing.md). From 8fdeea17553f153152e1be405015886cd5d8d248 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 28 Jul 2026 11:07:01 +0200 Subject: [PATCH 10/24] docs: add request for feedback to post --- docs/blog/posts/3.3.0-release.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/blog/posts/3.3.0-release.md b/docs/blog/posts/3.3.0-release.md index f7a76f9112..3bc1d5d698 100644 --- a/docs/blog/posts/3.3.0-release.md +++ b/docs/blog/posts/3.3.0-release.md @@ -158,6 +158,10 @@ arr = zarr.create_array( print(arr.shape) ``` +## Tell us what you think + +We hope these new features are helpful, and we would appreciate any feedback that helps us improve them, or any other aspect of Zarr-Python. + ## Going faster The work released in this PR is just the first step of a larger performance-oriented direction for Zarr-Python. Landing these two enhancements taught us a *lot* about the performance-sensitive areas of the library. We can and will invest more time in performance tuning, e.g. by adding or changing abstractions, writing code for special cases, etc. From 732d576600e80db81a4090db4143eeed50610927 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 28 Jul 2026 15:04:12 +0200 Subject: [PATCH 11/24] docs: clarify latency claims --- docs/blog/posts/3.3.0-release.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/blog/posts/3.3.0-release.md b/docs/blog/posts/3.3.0-release.md index 3bc1d5d698..ef0cf76698 100644 --- a/docs/blog/posts/3.3.0-release.md +++ b/docs/blog/posts/3.3.0-release.md @@ -33,7 +33,7 @@ If async overhead makes low-latency storage slow, does *removing* that overhead In [#3885](https://github.com/zarr-developers/zarr-python/pull/3885) we defined synchronous versions of our storage and codec APIs -- the `SyncByteGetter` and `SyncByteSetter` protocols, plus a `get_ranges_sync` method on the `Store` ABC -- and then combined them in a new codec orchestration class called `FusedCodecPipeline`. The `FusedCodecPipeline` is an opt-in alternative to the default (the `BatchedCodecPipeline`) that gives large speedups for low-latency storage. It is currently marked [experimental](../../user-guide/experimental.md), so we may change it as we learn more; the default pipeline is untouched, and existing code keeps working unless you opt in. -The win here is *not* a faster compressor. It is the removal of async scheduling overhead (including some [nasty `asyncio.to_thread` overhead](https://github.com/python/cpython/issues/136084)), plus a few vectorized fast paths for dense, uncompressed shards. +The win here is *not* a faster compressor. It is the removal of async scheduling overhead (including some [nasty `asyncio.to_thread` overhead](https://github.com/python/cpython/issues/136084)), plus a few vectorized fast paths for dense, uncompressed shards. And we only expect this new pipeline to accelerate workloads targeting a subset of storage backends, namely any store with methods that advertise low latency. On this author's 10-core Apple M4 laptop, the `FusedCodecPipeline` delivers the following results against memory-backed arrays: From ef48a771088c783f9d2bbee7a55db8d5088b466b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 28 Jul 2026 15:10:20 +0200 Subject: [PATCH 12/24] fix(examples): don't depend on pytest in the coalescing example `tests/test_examples.py` runs each example with `uv run` in an isolated environment containing only the dependencies its PEP 723 header declares. The example imported `LatencyStore` from `zarr.testing.store`, which hard-imports `pytest` at module level, so the script died with ModuleNotFoundError and failed CI on py3.12-min_deps and macOS py3.14. Define the latency wrapper in the example instead, subclassing the public `WrapperStore`. This keeps the script runnable with only zarr and numpy, and makes the mechanism it depends on explicit: `get` sleeps, and `get_ranges` is inherited from the `Store` ABC rather than `WrapperStore`'s forwarding version, so each merged fetch pays the latency once instead of bypassing it. `_with_store` is overridden because `WrapperStore` rebuilds the wrapper when opening read-only and would otherwise drop the latency setting. Assisted-by: ClaudeCode:claude-opus-5 --- examples/sharding_coalescing/README.md | 13 ++++-- .../sharding_coalescing.py | 46 +++++++++++++++---- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/examples/sharding_coalescing/README.md b/examples/sharding_coalescing/README.md index 51afbab354..29ba08c9ce 100644 --- a/examples/sharding_coalescing/README.md +++ b/examples/sharding_coalescing/README.md @@ -37,10 +37,15 @@ adapt this script: ranges that merge regardless of configuration. The strided selections skip inner chunks, creating the gaps that the `sharding_coalesce_max_gap_bytes` budget decides whether to bridge. -- **Latency must be charged per merged fetch.** `LatencyStore` adds latency in - `get`, but `WrapperStore.get_ranges` forwards directly to the wrapped store and - would bypass it. The example subclasses `LatencyStore` to inherit the `Store` - ABC's `get_ranges`, which runs the coalescer over its own `get`. +- **Latency must be charged per merged fetch.** The example defines a small + `WrapperStore` subclass that sleeps in `get`. It deliberately does *not* keep + `WrapperStore.get_ranges`, which forwards straight to the wrapped store and + would bypass the latency entirely; inheriting the `Store` ABC's `get_ranges` + instead runs the coalescer over its own `get`, so each merged fetch pays once. + + `zarr.testing.store` ships a ready-made `LatencyStore`, but importing it pulls + in `pytest`. Defining the wrapper inline keeps the example runnable with only + `zarr` and `numpy` installed. ## Configuration diff --git a/examples/sharding_coalescing/sharding_coalescing.py b/examples/sharding_coalescing/sharding_coalescing.py index ae8557bc92..5da62ed806 100644 --- a/examples/sharding_coalescing/sharding_coalescing.py +++ b/examples/sharding_coalescing/sharding_coalescing.py @@ -36,6 +36,7 @@ from __future__ import annotations +import asyncio import operator import statistics import timeit @@ -48,12 +49,13 @@ import zarr import zarr.core._coalesce as coalesce_module from zarr.abc.store import ByteRequest, RangeByteRequest, Store -from zarr.storage import MemoryStore -from zarr.testing.store import LatencyStore +from zarr.storage import MemoryStore, WrapperStore if TYPE_CHECKING: from collections.abc import Callable, Iterator, Sequence + from zarr.core.buffer import Buffer, BufferPrototype + # Emulates the pre-3.3.0 behavior: with a zero gap budget, only ranges that are # exactly adjacent get merged, so scattered inner chunks are fetched one by one. NO_COALESCING = {"sharding_coalesce_max_gap_bytes": 0} @@ -68,18 +70,44 @@ GET_LATENCY_S = 0.005 # 5 ms per request, a modest stand-in for object storage -class PerRequestLatencyStore(LatencyStore): - """A `LatencyStore` that charges latency per *merged byte-range fetch*. +class PerRequestLatencyStore(WrapperStore[Store]): + """Wraps a store, charging a fixed latency per byte-range fetch. + + `zarr.testing.store` ships a `LatencyStore`, but importing it pulls in + `pytest`; defining the wrapper here keeps this example runnable with only + zarr and numpy installed. - `WrapperStore.get_ranges` forwards straight to the wrapped store, which - would skip this class's latency-bearing `get` and make every configuration - look identical. Falling back to the `Store` ABC's `get_ranges` runs the - coalescer over `self.get` instead, so each merged fetch pays the latency -- - which is the cost coalescing exists to reduce. + Two details matter for the measurement: + + * The latency is applied in `get`, which is what an individual fetch costs. + * `get_ranges` is explicitly *not* overridden to forward to the wrapped + store. `WrapperStore.get_ranges` does forward, which would skip this + class's `get` entirely and make every configuration look identical. + Inheriting the `Store` ABC's implementation instead runs the coalescer + over `self.get`, so each *merged* fetch pays the latency once -- which is + exactly the cost coalescing exists to reduce. """ get_ranges = Store.get_ranges + def __init__(self, store: Store, *, get_latency: float) -> None: + super().__init__(store) + self.get_latency = get_latency + + def _with_store(self, store: Store) -> PerRequestLatencyStore: + # `WrapperStore` rebuilds the wrapper when opening read-only, so the + # latency setting has to be carried across. + return type(self)(store, get_latency=self.get_latency) + + async def get( + self, + key: str, + prototype: BufferPrototype, + byte_range: ByteRequest | None = None, + ) -> Buffer | None: + await asyncio.sleep(self.get_latency) + return await self._store.get(key, prototype, byte_range) + @contextmanager def counting_requests() -> Iterator[Callable[[], int]]: From 6343988b540cbae21e7b738755627c5920a57af3 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 28 Jul 2026 15:14:17 +0200 Subject: [PATCH 13/24] docs: add changelog entry for the blog and examples The contributing guide asks that most pull requests carry a news fragment, and the self-merge policy lists one among the standard merge requirements. The CI check only validates the filename structure of fragments that exist, so it passed without one. Assisted-by: ClaudeCode:claude-opus-5 --- changes/4191.doc.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 changes/4191.doc.md diff --git a/changes/4191.doc.md b/changes/4191.doc.md new file mode 100644 index 0000000000..cb1398dd7c --- /dev/null +++ b/changes/4191.doc.md @@ -0,0 +1,9 @@ +Added a blog section to the documentation, with a post covering two performance +highlights of the 3.3.0 release: the opt-in `FusedCodecPipeline` and byte-range +coalescing for partial reads of sharded arrays. + +Added two runnable examples that accompany the post: +`examples/codec_pipeline_performance` compares the `BatchedCodecPipeline` and +`FusedCodecPipeline` across several array layouts, and +`examples/sharding_coalescing` demonstrates how read coalescing reduces the +number of store requests when reading subregions of a sharded array. From cb93ef80f3d825446fe4fa97499d0787b410f0a9 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Tue, 28 Jul 2026 16:56:48 +0200 Subject: [PATCH 14/24] feat: add type-safe `get_array` and `get_group` methods to `AsyncGroup` and `Group` (#4128) * chore(deps): bump the actions group across 1 directory with 8 updates (#176) Bumps the actions group with 8 updates in the / directory: | Package | From | To | | --- | --- | --- | | [prefix-dev/setup-pixi](https://github.com/prefix-dev/setup-pixi) | `0.9.5` | `0.9.6` | | [codecov/codecov-action](https://github.com/codecov/codecov-action) | `6.0.0` | `6.0.1` | | [github/issue-metrics](https://github.com/github/issue-metrics) | `4.2.2` | `4.2.7` | | [j178/prek-action](https://github.com/j178/prek-action) | `2.0.3` | `2.0.4` | | [actions/upload-artifact](https://github.com/actions/upload-artifact) | `7.0.0` | `7.0.1` | | [actions/download-artifact](https://github.com/actions/download-artifact) | `7.0.0` | `8.0.1` | | [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) | `1.13.0` | `1.14.0` | | [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) | `0.5.3` | `0.5.6` | Updates `prefix-dev/setup-pixi` from 0.9.5 to 0.9.6 - [Release notes](https://github.com/prefix-dev/setup-pixi/releases) - [Commits](https://github.com/prefix-dev/setup-pixi/compare/1b2de7f3351f171c8b4dfeb558c639cb58ed4ec0...5185adfbffb4bd703da3010310260805d89ebb11) Updates `codecov/codecov-action` from 6.0.0 to 6.0.1 - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/57e3a136b779b570ffcdbf80b3bdc90e7fab3de2...e79a6962e0d4c0c17b229090214935d2e33f8354) Updates `github/issue-metrics` from 4.2.2 to 4.2.7 - [Release notes](https://github.com/github/issue-metrics/releases) - [Commits](https://github.com/github/issue-metrics/compare/c9e9838147fd355dace335ba787f01b6641a400a...1e38d5e62363e14db8019ed7d106b9855bdba6cc) Updates `j178/prek-action` from 2.0.3 to 2.0.4 - [Release notes](https://github.com/j178/prek-action/releases) - [Commits](https://github.com/j178/prek-action/compare/6ad80277337ad479fe43bd70701c3f7f8aa74db3...bdca6f102f98e2b4c7029491a53dfd366469e33d) Updates `actions/upload-artifact` from 7.0.0 to 7.0.1 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v7...043fb46d1a93c77aae656e7c1c64a875d1fc6a0a) Updates `actions/download-artifact` from 7.0.0 to 8.0.1 - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v7...3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c) Updates `pypa/gh-action-pypi-publish` from 1.13.0 to 1.14.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.13.0...cef221092ed1bacb1cc03d23a2d87d1d172e277b) Updates `zizmorcore/zizmor-action` from 0.5.3 to 0.5.6 - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/b1d7e1fb5de872772f31590499237e7cce841e8e...5f14fd08f7cf1cb1609c1e344975f152c7ee938d) --- updated-dependencies: - dependency-name: prefix-dev/setup-pixi dependency-version: 0.9.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: codecov/codecov-action dependency-version: 6.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: github/issue-metrics dependency-version: 4.2.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: j178/prek-action dependency-version: 2.0.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/upload-artifact dependency-version: 7.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/download-artifact dependency-version: 8.0.1 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: pypa/gh-action-pypi-publish dependency-version: 1.14.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: zizmorcore/zizmor-action dependency-version: 0.5.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feat: add type-safe get_array and get_group methods to AsyncGroup and Group Add `get_array` and `get_group` methods that return the child node at a given path or raise: `ArrayNotFoundError`/`GroupNotFoundError` when no node exists, and `ContainsGroupError`/`ContainsArrayError` when the node is the wrong kind. Paths mirror `getitem` semantics, so nested paths like "subgroup/subarray" work on both the plain and consolidated-metadata lookup routes. Existing tests that fetched a child via `getitem` and then manually narrowed the type with `isinstance` asserts or a walrus expression now use the new methods instead. Assisted-by: ClaudeCode:claude-fable-5 * fix: pass pre-formatted messages to error constructors in get_array/get_group The multi-argument template form of BaseZarrError.__init__ is documented as deprecated; build the message string at the raise site instead, matching every other call site in the codebase. Assisted-by: ClaudeCode:claude-fable-5 * docs: add changelog entry for get_array/get_group Assisted-by: ClaudeCode:claude-fable-5 * docs: demonstrate get_array/get_group in the groups user guide Assisted-by: ClaudeCode:claude-fable-5 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- changes/4128.feature.md | 1 + docs/user-guide/groups.md | 23 ++++ src/zarr/core/group.py | 133 +++++++++++++++++++++++ tests/test_group.py | 81 +++++++++++++- tests/test_metadata/test_consolidated.py | 9 +- tests/test_store/test_zip.py | 5 +- 6 files changed, 239 insertions(+), 13 deletions(-) create mode 100644 changes/4128.feature.md diff --git a/changes/4128.feature.md b/changes/4128.feature.md new file mode 100644 index 0000000000..c62a615ac2 --- /dev/null +++ b/changes/4128.feature.md @@ -0,0 +1 @@ +Added `Group.get_array`, `Group.get_group`, `AsyncGroup.get_array`, and `AsyncGroup.get_group`: type-safe accessors that return the child array or group at a given path, raising `ArrayNotFoundError` / `GroupNotFoundError` if no node exists there, and `ContainsGroupError` / `ContainsArrayError` if the node is not of the requested kind. Unlike `Group.__getitem__`, which returns `Array | Group`, these methods have precise return types. Nested paths like `"subgroup/subarray"` are supported. diff --git a/docs/user-guide/groups.md b/docs/user-guide/groups.md index 337ad39554..7429a03847 100644 --- a/docs/user-guide/groups.md +++ b/docs/user-guide/groups.md @@ -51,6 +51,29 @@ print(root['foo/bar']) print(root['foo/bar/baz']) ``` +Accessing a member with `[]` returns either an [`zarr.Array`][] or a [`zarr.Group`][], depending on +what is stored at the given path. When you expect a node of a particular kind, use +[`zarr.Group.get_array`][] or [`zarr.Group.get_group`][] instead. These methods accept the same +paths as `[]`, but they have precise return types and raise an error if no node exists at the +given path, or if the node is not of the expected kind: + +```python exec="true" session="groups" source="above" result="ansi" +print(root.get_group('foo')) +``` + +```python exec="true" session="groups" source="above" result="ansi" +print(root.get_array('foo/bar/baz')) +``` + +```python exec="true" session="groups" source="above" result="ansi" +from zarr.errors import ContainsGroupError + +try: + root.get_array('foo') +except ContainsGroupError as e: + print(e) +``` + The [`zarr.Group.tree`][] method can be used to print a tree representation of the hierarchy, e.g.: diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 0aaf89234e..922eaf1498 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -51,6 +51,7 @@ from zarr.core.metadata.io import save_metadata from zarr.core.sync import SyncMixin, sync from zarr.errors import ( + ArrayNotFoundError, ContainsArrayError, ContainsGroupError, GroupNotFoundError, @@ -820,6 +821,70 @@ async def get[DefaultT]( except KeyError: return default + async def get_array(self, path: str) -> AnyAsyncArray: + """Obtain an array member of this group, raising if it is absent or not an array. + + Parameters + ---------- + path : str + Path of the array relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subarray`. + + Returns + ------- + AsyncArray + The array at the given path. + + Raises + ------ + ArrayNotFoundError + If no node exists at the given path. + ContainsGroupError + If the node at the given path is a group rather than an array. + """ + store_path = self.store_path / path + try: + node = await self.getitem(path) + except KeyError as e: + msg = f"No array found in store {store_path.store!r} at path {store_path.path!r}" + raise ArrayNotFoundError(msg) from e + if isinstance(node, AsyncGroup): + msg = f"A group exists in store {store_path.store!r} at path {store_path.path!r}." + raise ContainsGroupError(msg) + return node + + async def get_group(self, path: str) -> AsyncGroup: + """Obtain a group member of this group, raising if it is absent or not a group. + + Parameters + ---------- + path : str + Path of the group relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subsubgroup`. + + Returns + ------- + AsyncGroup + The group at the given path. + + Raises + ------ + GroupNotFoundError + If no node exists at the given path. + ContainsArrayError + If the node at the given path is an array rather than a group. + """ + store_path = self.store_path / path + try: + node = await self.getitem(path) + except KeyError as e: + msg = f"No group found in store {store_path.store!r} at path {store_path.path!r}" + raise GroupNotFoundError(msg) from e + if isinstance(node, AsyncArray): + msg = f"An array exists in store {store_path.store!r} at path {store_path.path!r}." + raise ContainsArrayError(msg) + return node + async def _save_metadata(self, ensure_parents: bool = False) -> None: await save_metadata(self.store_path, self.metadata, ensure_parents=ensure_parents) @@ -1880,6 +1945,74 @@ def get[DefaultT]( except KeyError: return default + def get_array(self, path: str) -> AnyArray: + """Obtain an array member of this group, raising if it is absent or not an array. + + Parameters + ---------- + path : str + Path of the array relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subarray`. + + Returns + ------- + Array + The array at the given path. + + Raises + ------ + ArrayNotFoundError + If no node exists at the given path. + ContainsGroupError + If the node at the given path is a group rather than an array. + + Examples + -------- + ```python + import zarr + from zarr.core.group import Group + group = Group.from_store(zarr.storage.MemoryStore()) + group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") + group.get_array("subarray") + # + ``` + """ + return Array(self._sync(self._async_group.get_array(path))) + + def get_group(self, path: str) -> Group: + """Obtain a group member of this group, raising if it is absent or not a group. + + Parameters + ---------- + path : str + Path of the group relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subsubgroup`. + + Returns + ------- + Group + The group at the given path. + + Raises + ------ + GroupNotFoundError + If no node exists at the given path. + ContainsArrayError + If the node at the given path is an array rather than a group. + + Examples + -------- + ```python + import zarr + from zarr.core.group import Group + group = Group.from_store(zarr.storage.MemoryStore()) + group.create_group(name="subgroup") + group.get_group("subgroup") + # + ``` + """ + return Group(self._sync(self._async_group.get_group(path))) + def __delitem__(self, key: str) -> None: """Delete a group member. diff --git a/tests/test_group.py b/tests/test_group.py index 1acd5551ca..29377a5392 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -41,8 +41,10 @@ from zarr.core.metadata.v3 import ArrayV3Metadata from zarr.core.sync import _collect_aiterator, sync from zarr.errors import ( + ArrayNotFoundError, ContainsArrayError, ContainsGroupError, + GroupNotFoundError, MetadataValidationError, ZarrUserWarning, ) @@ -101,7 +103,7 @@ async def test_create_creates_parents(store: Store, zarr_format: ZarrFormat) -> root = await zarr.api.asynchronous.open_group( store=store, ) - agroup = await root.getitem("a") + agroup = await root.get_group("a") assert agroup.attrs == {"key": "value"} # create a child node with a couple intermediates @@ -446,6 +448,77 @@ def test_group_get_with_default(store: Store, zarr_format: ZarrFormat) -> None: assert result.attrs["foo"] == "bar" +def test_group_get_array(store: Store, zarr_format: ZarrFormat) -> None: + """ + `Group.get_array` returns the array at the given path, for both direct child names + and nested paths, and the result is statically typed as an Array. + """ + group = Group.from_store(store, zarr_format=zarr_format) + subgroup = group.create_group(name="subgroup") + subarray = group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") + subsubarray = subgroup.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") + + observed = group.get_array("subarray") + assert isinstance(observed, Array) + assert observed == subarray + assert group.get_array("subgroup/subarray") == subsubarray + + +def test_group_get_array_missing(store: Store, zarr_format: ZarrFormat) -> None: + """ + `Group.get_array` raises `ArrayNotFoundError` when no node exists at the given path. + """ + group = Group.from_store(store, zarr_format=zarr_format) + with pytest.raises(ArrayNotFoundError, match="No array found in store"): + group.get_array("missing") + + +def test_group_get_array_wrong_node_type(store: Store, zarr_format: ZarrFormat) -> None: + """ + `Group.get_array` raises `ContainsGroupError` when the node at the given path is a + group rather than an array. + """ + group = Group.from_store(store, zarr_format=zarr_format) + group.create_group(name="subgroup") + with pytest.raises(ContainsGroupError, match="A group exists in store"): + group.get_array("subgroup") + + +def test_group_get_group(store: Store, zarr_format: ZarrFormat) -> None: + """ + `Group.get_group` returns the group at the given path, for both direct child names + and nested paths, and the result is statically typed as a Group. + """ + group = Group.from_store(store, zarr_format=zarr_format) + subgroup = group.create_group(name="subgroup") + subsubgroup = subgroup.create_group(name="subsubgroup") + + observed = group.get_group("subgroup") + assert isinstance(observed, Group) + assert observed == subgroup + assert group.get_group("subgroup/subsubgroup") == subsubgroup + + +def test_group_get_group_missing(store: Store, zarr_format: ZarrFormat) -> None: + """ + `Group.get_group` raises `GroupNotFoundError` when no node exists at the given path. + """ + group = Group.from_store(store, zarr_format=zarr_format) + with pytest.raises(GroupNotFoundError, match="No group found in store"): + group.get_group("missing") + + +def test_group_get_group_wrong_node_type(store: Store, zarr_format: ZarrFormat) -> None: + """ + `Group.get_group` raises `ContainsArrayError` when the node at the given path is an + array rather than a group. + """ + group = Group.from_store(store, zarr_format=zarr_format) + group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") + with pytest.raises(ContainsArrayError, match="An array exists in store"): + group.get_group("subarray") + + @pytest.mark.parametrize("consolidated", [True, False]) def test_group_delitem(store: Store, zarr_format: ZarrFormat, consolidated: bool) -> None: """ @@ -1469,7 +1542,7 @@ async def test_group_getitem_consolidated(self, store: Store) -> None: # On disk, we've consolidated all the metadata in the root zarr.json group = await zarr.api.asynchronous.open(store=store) - rg0 = await group.getitem("g0") + rg0 = await group.get_group("g0") expected = ConsolidatedMetadata( metadata={ @@ -1490,10 +1563,10 @@ async def test_group_getitem_consolidated(self, store: Store) -> None: ) assert rg0.metadata.consolidated_metadata == expected - rg1 = await rg0.getitem("g1") + rg1 = await rg0.get_group("g1") assert rg1.metadata.consolidated_metadata == expected.metadata["g1"].consolidated_metadata - rg2 = await rg1.getitem("g2") + rg2 = await rg1.get_group("g2") assert rg2.metadata.consolidated_metadata == ConsolidatedMetadata(metadata={}) async def test_group_delitem_consolidated(self, store: Store) -> None: diff --git a/tests/test_metadata/test_consolidated.py b/tests/test_metadata/test_consolidated.py index 3596d2bcaa..e6087435fe 100644 --- a/tests/test_metadata/test_consolidated.py +++ b/tests/test_metadata/test_consolidated.py @@ -111,12 +111,10 @@ async def test_getitem_consolidated_empty_leaf_group( group = await zarr.api.asynchronous.open_consolidated( store=memory_store, zarr_format=zarr_format ) - raw = await group.getitem("raw") - assert isinstance(raw, zarr.AsyncGroup) + raw = await group.get_group("raw") assert raw.metadata.consolidated_metadata is not None - varm = await raw.getitem("varm") - assert isinstance(varm, zarr.AsyncGroup) + varm = await raw.get_group("varm") assert varm.metadata.consolidated_metadata == ConsolidatedMetadata(metadata={}) async def test_open_consolidated_false_raises(self) -> None: @@ -770,8 +768,7 @@ async def test_absolute_path_for_subgroup(self, memory_store: zarr.storage.Memor await zarr.api.asynchronous.consolidate_metadata(memory_store) group = await zarr.api.asynchronous.open_group(store=memory_store) - subgroup = await group.getitem("/a") - assert isinstance(subgroup, AsyncGroup) + subgroup = await group.get_group("/a") members = [x async for x in subgroup.keys()] # noqa: SIM118 assert members == ["b"] diff --git a/tests/test_store/test_zip.py b/tests/test_store/test_zip.py index 0d8dadd18a..32b18c5273 100644 --- a/tests/test_store/test_zip.py +++ b/tests/test_store/test_zip.py @@ -22,7 +22,6 @@ import zarr from zarr import create_array from zarr.core.buffer import Buffer, cpu, default_buffer_prototype -from zarr.core.group import Group from zarr.core.sync import sync from zarr.storage import ZipStore from zarr.testing.store import StoreTests @@ -141,13 +140,13 @@ def test_externally_zipped_store(self, tmp_path: Path) -> None: zarr_path = tmp_path / "foo.zarr" root = zarr.open_group(store=zarr_path, mode="w") root.require_group("foo") - assert isinstance(foo := root["foo"], Group) # noqa: RUF018 + foo = root.get_group("foo") foo["bar"] = np.array([1]) shutil.make_archive(str(zarr_path), "zip", zarr_path) zip_path = tmp_path / "foo.zarr.zip" zipped = zarr.open_group(ZipStore(zip_path, mode="r"), mode="r") assert list(zipped.keys()) == list(root.keys()) - assert isinstance(group := zipped["foo"], Group) + group = zipped.get_group("foo") assert list(group.keys()) == list(group.keys()) async def test_list_without_explicit_open(self, tmp_path: Path) -> None: From 8e5635ade5c30aee047e7657332dfdf4a8a4ea8c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 28 Jul 2026 16:57:44 +0200 Subject: [PATCH 15/24] docs(examples): measure gzip alongside zstd in the pipeline example The example only measured zstd level 0, the fast Zarr v3 default, so it never covered the slow-encoder case. Add gzip level 5 as a third compressor: it moves the bottleneck from async scheduling to codec compute, which is a different regime for the two pipelines and worth showing side by side. Also drop the claim in the notes that threading "can pay off" with compression. Measured on a 10-core M4, the worker count barely moves any compressed row -- the pool parallelizes over outer chunks, and adding compression does not change how many of those there are. Assisted-by: ClaudeCode:claude-opus-5 --- examples/codec_pipeline_performance/README.md | 9 ++++--- .../codec_pipeline_performance.py | 26 ++++++++++++++----- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/examples/codec_pipeline_performance/README.md b/examples/codec_pipeline_performance/README.md index e590fe725a..82c5adc6b6 100644 --- a/examples/codec_pipeline_performance/README.md +++ b/examples/codec_pipeline_performance/README.md @@ -47,6 +47,9 @@ Two effects are worth watching for: - **Threading helps only when there is work to spread.** The pool parallelizes over *outer* chunks. Uncompressed data has almost no CPU-bound codec work, so more workers mostly add overhead. -- **Compression can reverse the result.** On layouts where the fused pipeline - cannot overlap compression work, the async pipeline's overlap wins and the - measured speedup drops below 1. +- **The compressor matters as much as the layout.** The script measures three: + no compression, `zstd` level 0 (the Zarr v3 default, fast), and `gzip` level 5 + (much slower to encode). Uncompressed and zstd workloads are dominated by + scheduling overhead, which is what the fused pipeline removes. gzip shifts the + bottleneck to codec compute, a regime it was not built to win, and the + measured speedup can drop below 1. diff --git a/examples/codec_pipeline_performance/codec_pipeline_performance.py b/examples/codec_pipeline_performance/codec_pipeline_performance.py index 9ef24f534f..7d3f6efbd6 100644 --- a/examples/codec_pipeline_performance/codec_pipeline_performance.py +++ b/examples/codec_pipeline_performance/codec_pipeline_performance.py @@ -49,9 +49,15 @@ BATCHED = "zarr.core.codec_pipeline.BatchedCodecPipeline" FUSED = "zarr.core.codec_pipeline.FusedCodecPipeline" -# zstd level 0 is the Zarr v3 default compressor configuration. +# zstd level 0 is the Zarr v3 default compressor configuration: fast enough +# that, on a low-latency store, codec work is not usually the bottleneck. ZSTD = {"name": "zstd", "configuration": {"level": 0, "checksum": False}} +# gzip is much slower to encode than zstd, which makes it a useful contrast: +# it shifts the balance from scheduling overhead toward codec compute, so the +# two pipelines are compared under a genuinely CPU-bound workload. +GZIP = {"name": "gzip", "configuration": {"level": 5}} + @dataclass(frozen=True) class Layout: @@ -157,16 +163,20 @@ def main() -> None: data = make_data(size) print(f"zarr {zarr.__version__} | {n_cpu} CPUs | {data.nbytes / 1e6:.0f} MB uint8\n") - for compressor_label, compressors in (("uncompressed", None), ("zstd", ZSTD)): + for compressor_label, compressors in ( + ("uncompressed", None), + ("zstd level 0 (fast)", ZSTD), + ("gzip level 5 (slow encode)", GZIP), + ): print(f"===== {compressor_label} =====") for layout in layouts: print(f"\n{layout.label} ({layout.detail})") print(f" {'workers':>7} {'op':<6} {'batched':>9} {'fused':>9} {'speedup':>9}") base_w, base_r = measure(BATCHED, layout, data, compressors, None) - # For uncompressed data there is no CPU-bound codec work to spread - # across threads, so a thread pool only adds overhead. With - # compression it can pay off. Measure both rather than asserting. + # Measure at one worker and at cpu_count rather than assuming which + # is better: the thread pool only has something to spread when + # there are several outer chunks *and* real codec work to overlap. for workers in (1, n_cpu): fused_w, fused_r = measure(FUSED, layout, data, compressors, workers) for op, base, fused in (("write", base_w, fused_w), ("read", base_r, fused_r)): @@ -184,7 +194,11 @@ def main() -> None: " shard, so layouts with few outer chunks see little benefit from more\n" " workers no matter how many inner chunks they contain.\n" " * `codec_pipeline.max_workers` is read only by the FusedCodecPipeline;\n" - " the default BatchedCodecPipeline ignores it." + " the default BatchedCodecPipeline ignores it.\n" + " * The compressor matters as much as the layout. A slow encoder like gzip\n" + " makes codec compute, not scheduling, the bottleneck -- which is the\n" + " regime the fused pipeline was not built to win. Compare its rows\n" + " against zstd's before drawing conclusions about your own workload." ) From e345d185f721d978e3d450ab927928156f733062 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Tue, 28 Jul 2026 19:04:02 +0200 Subject: [PATCH 16/24] Update 3.3.0-release.md Co-authored-by: Kyle Barron --- docs/blog/posts/3.3.0-release.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/blog/posts/3.3.0-release.md b/docs/blog/posts/3.3.0-release.md index ef0cf76698..88e46be7e1 100644 --- a/docs/blog/posts/3.3.0-release.md +++ b/docs/blog/posts/3.3.0-release.md @@ -120,7 +120,7 @@ Relevant issues and pull requests: ### How sharding works -Chunks encoded with the `sharding_indexed` codec contain subchunks, or "inner chunks". For example, if the `chunk_grid` field of the array metadata declares an "outer chunk" size of, say `(10, 10)`, a `sharding_indexed` codec in the `codecs` field could declare an "inner chunk" size of `(5, 5)`. Readers accessing such a chunk will observe a stored object (a stream of bytes) that decodes to an array with size `(10, 10)` (the "outer chunk"), which is comprised of four separate, contiguous byte ranges that each decode to a `(5, 5)` inner chunk. Each inner chunk occupies its own byte range in the outer chunk. +Chunks encoded with the `sharding_indexed` codec contain a secondary level of chunking, called subchunks. For example, if the `chunk_grid` field of the array metadata declares an "outer chunk" size of, say `(10, 10)`, a `sharding_indexed` codec in the `codecs` field could declare an "inner chunk" size of `(5, 5)`. Readers accessing such a chunk will observe a stored object (a stream of bytes) that decodes to an array with size `(10, 10)` (the "outer chunk"), which is comprised of four separate, contiguous byte ranges that each decode to a `(5, 5)` inner chunk. Each inner chunk occupies its own byte range in the outer chunk. A reader can satisfy a request for all four inner chunks by issuing four separate byte-range requests, or by making a *single* request for a byte range that spans all four inner chunks. The latter option is nice because it cuts down on the number of requests we need. Historically Zarr-Python used this optimization when reading entire outer chunks; in 3.3.0, we use this optimization in more cases, resulting in more efficient I/O patterns for sharded reads. From 05ce214428b48ae6f4de327804a072b9c9f20792 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Tue, 28 Jul 2026 19:04:14 +0200 Subject: [PATCH 17/24] Update 3.3.0-release.md Co-authored-by: Kyle Barron --- docs/blog/posts/3.3.0-release.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/blog/posts/3.3.0-release.md b/docs/blog/posts/3.3.0-release.md index 88e46be7e1..7a419277f5 100644 --- a/docs/blog/posts/3.3.0-release.md +++ b/docs/blog/posts/3.3.0-release.md @@ -132,7 +132,7 @@ The requested inner chunks are not necessarily contiguous -- there might be a by ### Byte range coalescing -We call this procedure -- rewriting `N` byte range requests into `M` byte range requests, where `M` <= `N` -- "byte range coalescing", and it's a new performance optimization shipping in Zarr-Python 3.3.0. Unlike the `FusedCodecPipeline`, this one is on by default with base settings we think are good, so most users won't need to tune anything. +We call this procedure -- merging adjacent byte ranges -- "byte range coalescing", and it's a new performance optimization shipping in Zarr-Python 3.3.0. Unlike the `FusedCodecPipeline`, this one is on by default with base settings we think are good, so most users won't need to tune anything. Two knobs control it, both documented in the [runtime configuration guide](../../user-guide/config.md). Nearby byte ranges in the same shard are merged into a single request when the gap between them is no larger than `array.sharding_coalesce_max_gap_bytes` (default 1 MiB) and the merged read stays within `array.sharding_coalesce_max_bytes` (default 16 MiB). The gap threshold is what trades wasted bytes against saved requests: raising it reads more data you didn't ask for, in exchange for fewer requests. From 4a4eb43846dd65f5277336b6b3da76e20e04ef77 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Tue, 28 Jul 2026 19:04:26 +0200 Subject: [PATCH 18/24] Update 3.3.0-release.md Co-authored-by: Kyle Barron --- docs/blog/posts/3.3.0-release.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/blog/posts/3.3.0-release.md b/docs/blog/posts/3.3.0-release.md index 7a419277f5..371af6c673 100644 --- a/docs/blog/posts/3.3.0-release.md +++ b/docs/blog/posts/3.3.0-release.md @@ -164,6 +164,6 @@ We hope these new features are helpful, and we would appreciate any feedback tha ## Going faster -The work released in this PR is just the first step of a larger performance-oriented direction for Zarr-Python. Landing these two enhancements taught us a *lot* about the performance-sensitive areas of the library. We can and will invest more time in performance tuning, e.g. by adding or changing abstractions, writing code for special cases, etc. +The updates in this release are just the first step of a larger performance-oriented direction for Zarr-Python. Landing these two enhancements taught us a *lot* about the performance-sensitive areas of the library. We can and will invest more time in performance tuning, e.g. by adding or changing abstractions, writing code for special cases, etc. But we should also look to wrapping lower-level libraries that simply have an easier path to performance by virtue of being written in a compiled language. The [`zarrs`](https://zarrs.dev/) project is an ecosystem of Zarr tools written in Rust, with [extremely high performance](https://book.zarrs.dev/#-zarrs-is-fast-). Is there a `zarrs` binding in Zarr-Python's future? I hope so! We are keenly observing development of [`zarrista`](https://developmentseed.org/zarrista/latest/) as a proof-of-concept for what a Python-`zarrs` binding layer might look like. Stay tuned! From 23502029de2a33e7a10d2c9af05d8379bdb2b002 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Tue, 28 Jul 2026 19:04:37 +0200 Subject: [PATCH 19/24] Update .authors.yml Co-authored-by: Kyle Barron --- docs/blog/.authors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/blog/.authors.yml b/docs/blog/.authors.yml index f756cdcaa9..10ce423cfc 100644 --- a/docs/blog/.authors.yml +++ b/docs/blog/.authors.yml @@ -1,6 +1,6 @@ authors: d-v-b: name: Davis Bennett - description: Zarr-Python core developer + description: Core developer avatar: https://github.com/d-v-b.png url: https://github.com/d-v-b From 3243ac2d823a28fb486ccc641e00c88b4d02e25b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 28 Jul 2026 20:09:43 +0200 Subject: [PATCH 20/24] docs(examples): rebuild the pipeline example on the real benchmark workload The example measured a 1D array with 100k-element chunks and reported that the thread pool never mattered. That contradicted the blog post, and the example was wrong on two counts. Workload: switch to the workload the post's numbers actually came from -- 4096x4096 int32 in 16 shards of 1024x1024, each holding 256 inner chunks of 64x64, over both MemoryStore and LocalStore. The 64x64 chunks are the load-bearing detail: small enough that per-chunk scheduling dominates uncompressed IO, large enough that per-chunk gzip is real work to spread across cores. The previous 100k-element chunks were far too coarse for a thread pool to have anything to do. Each regime also gets the data that exercises it -- arange for uncompressed, noisy integers for gzip, which would otherwise compress away to nothing. Measurement: run the whole operation inside `zarr.config.set`, not just the array construction. The pipeline class is resolved at construction, but `codec_pipeline.max_workers` is read per operation, so timing calls made outside the config block silently used the ambient worker count and made every configuration look identical. With both fixed, the example reproduces the documented behavior: gzip writes at 1 worker are slower than the default and a thread pool recovers 1.4-1.8x, while uncompressed writes are fastest at 1 worker and lose ~0.6x to pool overhead. Assisted-by: ClaudeCode:claude-opus-5 --- examples/codec_pipeline_performance/README.md | 26 +- .../codec_pipeline_performance.py | 268 +++++++++--------- 2 files changed, 153 insertions(+), 141 deletions(-) diff --git a/examples/codec_pipeline_performance/README.md b/examples/codec_pipeline_performance/README.md index 82c5adc6b6..5d85412c29 100644 --- a/examples/codec_pipeline_performance/README.md +++ b/examples/codec_pipeline_performance/README.md @@ -1,7 +1,8 @@ # Codec Pipeline Performance This example compares the default `BatchedCodecPipeline` against the opt-in -`FusedCodecPipeline` across several array layouts, with and without compression. +`FusedCodecPipeline` on a sharded array, across two stores (memory and local) +and two codec regimes (uncompressed and gzip), at one worker and at `cpu_count`. A *codec pipeline* turns chunks of array data into stored bytes and back, running the configured codecs and performing the storage IO. The default @@ -37,19 +38,22 @@ The script has no arguments and writes only to an in-memory store. ## Interpreting the output -The numbers are specific to your CPU, your Python build, and the layouts chosen +The numbers are specific to your CPU, your Python build, and the workload chosen here. They are a measurement of your machine, not a published benchmark -- treat a single run as indicative and re-measure against your own data and store before switching pipelines in production. Two effects are worth watching for: -- **Threading helps only when there is work to spread.** The pool parallelizes - over *outer* chunks. Uncompressed data has almost no CPU-bound codec work, so - more workers mostly add overhead. -- **The compressor matters as much as the layout.** The script measures three: - no compression, `zstd` level 0 (the Zarr v3 default, fast), and `gzip` level 5 - (much slower to encode). Uncompressed and zstd workloads are dominated by - scheduling overhead, which is what the fused pipeline removes. gzip shifts the - bottleneck to codec compute, a regime it was not built to win, and the - measured speedup can drop below 1. +- **The two codec regimes tell opposite stories about `max_workers`.** + Uncompressed IO is dominated by per-chunk *scheduling*, so `Fused (1 worker)` + is already fastest and a thread pool only adds overhead. gzip is genuinely + CPU-bound: a single worker compresses chunk after chunk sequentially and can + be *slower than the default*, while a thread pool spreads that compression + over cores and reclaims the win. That flip is why the fused pipeline is + threaded by default, and why pinning `max_workers=1` is worth it for + memory-backed uncompressed data. +- **Chunk size decides whether threading can help at all.** The 64×64 inner + chunks here are small enough that per-chunk scheduling dominates uncompressed + IO, yet large enough that per-chunk gzip is real work to parallelize. Much + coarser chunks leave the pool with too few items to spread. diff --git a/examples/codec_pipeline_performance/codec_pipeline_performance.py b/examples/codec_pipeline_performance/codec_pipeline_performance.py index 7d3f6efbd6..b821f3e6a7 100644 --- a/examples/codec_pipeline_performance/codec_pipeline_performance.py +++ b/examples/codec_pipeline_performance/codec_pipeline_performance.py @@ -7,18 +7,27 @@ # /// """ -Compare the performance of the `BatchedCodecPipeline` and the `FusedCodecPipeline`. +Compare the `BatchedCodecPipeline` and the `FusedCodecPipeline`. The default `BatchedCodecPipeline` schedules storage IO and codec compute -asynchronously -- roughly one coroutine per chunk operation. That is the right -model for high-latency stores, where there is useful work to do while waiting -for IO. For *low-latency* stores (in-process memory, the local filesystem) the -IO completes too quickly for the overlap to pay for itself, and the async -scheduling is pure overhead. +asynchronously -- roughly one coroutine per chunk operation. For a *sharded* +array that means one coroutine per inner chunk inside every shard. That is the +right model for high-latency stores, where there is useful work to do while +waiting for IO. For low-latency stores (in-process memory, the local +filesystem) the IO completes too quickly for the overlap to pay for itself, and +the scheduling becomes pure overhead. The `FusedCodecPipeline` runs codec compute and synchronous IO synchronously, -removing that overhead. This script measures the difference across a few -layouts so you can see where it helps and where it does not. +removing that overhead. Whether it wins, and whether its thread pool helps, +depends on which resource is actually scarce: + + * Uncompressed IO is dominated by per-chunk *scheduling*, not compute. There + is nothing for a thread pool to parallelize, so a single worker is already + fastest and extra workers only add overhead. + * gzip is genuinely CPU-bound. A single worker compresses every chunk + sequentially and can be *slower than the default*, while a thread pool + spreads that compression across cores and reclaims the win. This is when + `max_workers > 1` earns its keep. Run it with: @@ -33,172 +42,171 @@ import operator import os import statistics +import tempfile import timeit -from dataclasses import dataclass -from functools import partial +from pathlib import Path from typing import TYPE_CHECKING import numpy as np import zarr -from zarr.storage import MemoryStore +from zarr.storage import LocalStore, MemoryStore if TYPE_CHECKING: from collections.abc import Callable + from zarr.abc.store import Store + BATCHED = "zarr.core.codec_pipeline.BatchedCodecPipeline" FUSED = "zarr.core.codec_pipeline.FusedCodecPipeline" -# zstd level 0 is the Zarr v3 default compressor configuration: fast enough -# that, on a low-latency store, codec work is not usually the bottleneck. -ZSTD = {"name": "zstd", "configuration": {"level": 0, "checksum": False}} - -# gzip is much slower to encode than zstd, which makes it a useful contrast: -# it shifts the balance from scheduling overhead toward codec compute, so the -# two pipelines are compared under a genuinely CPU-bound workload. -GZIP = {"name": "gzip", "configuration": {"level": 5}} - - -@dataclass(frozen=True) -class Layout: - """One array geometry to measure.""" - - label: str - shape: tuple[int, ...] - chunks: tuple[int, ...] - shards: tuple[int, ...] | None - - @property - def detail(self) -> str: - if self.shards is None: - return f"{self.shape[0] // self.chunks[0]} chunks, no sharding" - chunks_per_shard = self.shards[0] // self.chunks[0] - n_shards = self.shape[0] // self.shards[0] - return f"{chunks_per_shard} chunks/shard x {n_shards} shards" - +# gzip is CPU-bound to encode, which is exactly the regime where the thread +# pool matters. Level 6 is gzip's own default. +GZIP = {"name": "gzip", "configuration": {"level": 6}} -def make_data(n: int) -> np.ndarray: - """Semi-random but compressible data, so zstd has real work to do. +# 4096x4096 int32 = 64 MiB, split into 16 shards of 1024x1024, each holding +# 16x16 = 256 inner chunks of 64x64 -> 4096 inner chunks in total. The chunks +# are small enough that per-chunk coroutine scheduling dominates uncompressed +# IO, yet large enough that per-chunk gzip is real work to spread over cores. +SHAPE = (4096, 4096) +SHARDS = (1024, 1024) +CHUNKS = (64, 64) +DTYPE = "int32" - Pure random data is incompressible and pure constant data compresses away - to nothing; either would make the compressed measurements misleading. - """ - period = 256 - pattern = (np.sin(np.linspace(0, 2 * np.pi, period)) * 50 + 128).round().astype(np.uint8) - values = np.tile(pattern, int(np.ceil(n / period)))[:n].astype(np.int16) - rng = np.random.default_rng(0) - values += rng.integers(-1, 2, size=n, dtype=np.int16) - return np.clip(values, 0, 255).astype(np.uint8) +CONFIGS: tuple[tuple[str, dict[str, object]], ...] = ( + ("Batched (default)", {"codec_pipeline.path": BATCHED}), + ("Fused (1 worker)", {"codec_pipeline.path": FUSED, "codec_pipeline.max_workers": 1}), + ("Fused (cpu_count)", {"codec_pipeline.path": FUSED, "codec_pipeline.max_workers": None}), +) -def time_call(fn: Callable[[], object], repeat: int = 5) -> float: +def time_call(fn: Callable[[], object], repeat: int = 3) -> float: """Median wall-clock seconds for one call to `fn`. - `timeit.Timer` handles the mechanics: it uses `perf_counter` and disables - the cyclic garbage collector for the duration of each timing run, so a - collection triggered by earlier work cannot land in the middle of a - measurement. - - `number=1` because a single call here already moves tens of megabytes -- - the per-call overhead `timeit` is designed to amortize is irrelevant at - this scale, and looping inside a run would only measure the same thing - repeatedly. - - The median across repeats is deliberate: `timeit` documents that the - minimum is the best estimate of the achievable speed, but for IO-and-codec - work the typical case is more informative than the luckiest run, and a - mean would be dragged around by scheduler noise. + `timeit.Timer` supplies `perf_counter` and disables the cyclic garbage + collector during each run, so a collection triggered by earlier work cannot + land inside a measurement. `number=1` because a single call here already + moves 64 MiB -- the per-call overhead `timeit` amortizes is irrelevant at + this scale. """ return statistics.median(timeit.Timer(fn).repeat(repeat=repeat, number=1)) def measure( - pipeline: str, - layout: Layout, + settings: dict[str, object], + store: Store, data: np.ndarray, compressors: object, - max_workers: int | None, ) -> tuple[float, float]: - """Time one full write and one full read under `pipeline`. + """Time one full write and one full read of `data` under `settings`. - The pipeline class is read from the global config when the array is - constructed, so the array must be *created* inside the config block. Once - built, the array keeps that pipeline -- which is why the timed calls can - sit outside the `with` block. + The whole operation runs inside `zarr.config.set`, not just the array + construction. The pipeline class is resolved when the array is built, but + `codec_pipeline.max_workers` is read *per operation*, so a timed call made + outside the config block would silently use whatever worker count was + globally in effect -- which makes every configuration look identical. """ - settings: dict[str, object] = {"codec_pipeline.path": pipeline} - if pipeline == FUSED: - settings["codec_pipeline.max_workers"] = max_workers - - with zarr.config.set(settings): - array = zarr.create_array( - store=MemoryStore(), - shape=layout.shape, - chunks=layout.chunks, - shards=layout.shards, - dtype="uint8", - compressors=compressors, - ) - everything = slice(None) - write = time_call(partial(operator.setitem, array, everything, data)) - read = time_call(partial(operator.getitem, array, everything)) - # Guard against measuring something that silently produced wrong data. - if not np.array_equal(array[everything], data): - raise AssertionError(f"round trip mismatch for {pipeline}") + def write_once() -> None: + with zarr.config.set(settings): + array = zarr.create_array( + store=store, + shape=SHAPE, + chunks=CHUNKS, + shards=SHARDS, + dtype=DTYPE, + compressors=compressors, + fill_value=0, + overwrite=True, + ) + operator.setitem(array, everything, data) + + write = time_call(write_once) + + # The bytes on disk are identical whichever pipeline wrote them, so reading + # back what we just wrote isolates read performance on the same data. + def read_once() -> object: + with zarr.config.set(settings): + return zarr.open_array(store=store, mode="r")[everything] + + read = time_call(read_once) + + if not np.array_equal(read_once(), data): + raise AssertionError("round trip mismatch") return write, read +def make_store(kind: str, tmp: Path) -> Store: + if kind == "memory": + return MemoryStore() + return LocalStore(tmp / f"demo_{kind}_{os.getpid()}.zarr") + + def main() -> None: n_cpu = os.cpu_count() or 1 - size = 64_000_000 # 64 MB of uint8 - layouts = ( - Layout("flat", (size,), (100_000,), None), - Layout("many shards", (size,), (100_000,), (100_000 * 100,)), - Layout("one big shard", (size,), (100_000,), (size,)), + # Each regime gets the data that actually exercises it. `arange` is + # trivially compressible, which is fine when nothing compresses it, but it + # would make gzip finish almost instantly and hide the CPU-bound behavior + # this example is about. The noisy array keeps gzip genuinely busy. + n = int(np.prod(SHAPE)) + plain_data = np.arange(n, dtype=DTYPE).reshape(SHAPE) + noisy_data = np.random.default_rng(0).integers(0, 2**24, size=SHAPE, dtype=DTYPE) + + n_shards = int(np.prod([s // c for s, c in zip(SHAPE, SHARDS, strict=True)])) + per_shard = int(np.prod([s // c for s, c in zip(SHARDS, CHUNKS, strict=True)])) + print(f"zarr {zarr.__version__} | {n_cpu} CPUs") + print( + f"array {SHAPE} {DTYPE} = {plain_data.nbytes / 2**20:.0f} MiB | " + f"{n_shards} shards x {per_shard} inner chunks = {n_shards * per_shard} chunks\n" ) - data = make_data(size) - print(f"zarr {zarr.__version__} | {n_cpu} CPUs | {data.nbytes / 1e6:.0f} MB uint8\n") - - for compressor_label, compressors in ( - ("uncompressed", None), - ("zstd level 0 (fast)", ZSTD), - ("gzip level 5 (slow encode)", GZIP), - ): - print(f"===== {compressor_label} =====") - for layout in layouts: - print(f"\n{layout.label} ({layout.detail})") - print(f" {'workers':>7} {'op':<6} {'batched':>9} {'fused':>9} {'speedup':>9}") - base_w, base_r = measure(BATCHED, layout, data, compressors, None) - - # Measure at one worker and at cpu_count rather than assuming which - # is better: the thread pool only has something to spread when - # there are several outer chunks *and* real codec work to overlap. - for workers in (1, n_cpu): - fused_w, fused_r = measure(FUSED, layout, data, compressors, workers) - for op, base, fused in (("write", base_w, fused_w), ("read", base_r, fused_r)): + with tempfile.TemporaryDirectory() as tmp: + for store_kind in ("memory", "local"): + for codec_label, compressors, data in ( + ("uncompressed", None, plain_data), + ("gzip-6 (CPU-bound)", GZIP, noisy_data), + ): + print(f"=== {store_kind} store / {codec_label} ===") + print( + f"{'pipeline':<22}{'write (s)':>11}{'vs base':>10}" + f"{'read (s)':>12}{'vs base':>10}" + ) + results: dict[str, tuple[float, float]] = {} + for label, settings in CONFIGS: + store = make_store(store_kind, Path(tmp)) + results[label] = measure(settings, store, data, compressors) + + base_write, base_read = results[CONFIGS[0][0]] + for label, (write, read) in results.items(): print( - f" {workers:>7} {op:<6} {base:>8.3f}s {fused:>8.3f}s {base / fused:>8.2f}x" + f"{label:<22}{write:>10.3f}{base_write / write:>9.1f}x" + f"{read:>11.3f}{base_read / read:>9.1f}x" ) - print() + + # The headline comparison: does the thread pool earn its keep? + single_write, single_read = results["Fused (1 worker)"] + pool_write, pool_read = results["Fused (cpu_count)"] + print( + f" workers (cpu_count vs 1 worker): " + f"write {single_write / pool_write:.1f}x " + f"read {single_read / pool_read:.1f}x" + ) + print() print( - "Notes:\n" - " * A speedup below 1.00x means the fused pipeline was slower for that case.\n" - " * Threading only helps when there is CPU-bound codec work to overlap and\n" - " more than one outer chunk to spread across threads. The thread pool\n" - " parallelizes over outer chunks, never over the inner chunks of a single\n" - " shard, so layouts with few outer chunks see little benefit from more\n" - " workers no matter how many inner chunks they contain.\n" + "Reading it:\n" + " * Uncompressed IO is scheduling-bound, so Fused (1 worker) is already\n" + " fastest -- a thread pool has nothing to parallelize and only adds\n" + " overhead.\n" + " * gzip is CPU-bound, so Fused (1 worker) can be *slower* than the\n" + " default, while Fused (cpu_count) spreads compression across cores\n" + " and reclaims the win. That flip is why the fused pipeline is\n" + " threaded by default, and why pinning max_workers=1 is worth it for\n" + " memory-backed uncompressed data.\n" " * `codec_pipeline.max_workers` is read only by the FusedCodecPipeline;\n" - " the default BatchedCodecPipeline ignores it.\n" - " * The compressor matters as much as the layout. A slow encoder like gzip\n" - " makes codec compute, not scheduling, the bottleneck -- which is the\n" - " regime the fused pipeline was not built to win. Compare its rows\n" - " against zstd's before drawing conclusions about your own workload." + " the default BatchedCodecPipeline ignores it." ) From e0f358f43b96e82804665482a5ff6cb24d9116a6 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 28 Jul 2026 20:16:35 +0200 Subject: [PATCH 21/24] docs(blog): apply Kyle's remaining review suggestions Two suggestions from the review on d-v-b/zarr-python#246 were not yet applied: - Replace the `4 << 20` bit-shift literals with `4 * 1024**2`, which reads more clearly as bytes -> KiB -> MiB. The suggestion wrote `1024 ** 3`, but these values are MiB, so the exponent is 2. - Reword the closing paragraph to state the compiled-code plan directly instead of framing it as something we "should look to". The rest of the review (subchunks phrasing, dropping the `N`/`M` notation, the "first step of a larger direction" opener, and the author description) is already in the branch. Assisted-by: ClaudeCode:claude-opus-5 --- docs/blog/posts/3.3.0-release.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/blog/posts/3.3.0-release.md b/docs/blog/posts/3.3.0-release.md index 371af6c673..74042e0078 100644 --- a/docs/blog/posts/3.3.0-release.md +++ b/docs/blog/posts/3.3.0-release.md @@ -151,8 +151,8 @@ arr = zarr.create_array( shards=(1000, 1000), dtype="float32", config={ - "sharding_coalesce_max_gap_bytes": 4 << 20, # 4 MiB - "sharding_coalesce_max_bytes": 64 << 20, # 64 MiB + "sharding_coalesce_max_gap_bytes": 4 * 1024**2, # 4 MiB + "sharding_coalesce_max_bytes": 64 * 1024**2, # 64 MiB }, ) print(arr.shape) @@ -166,4 +166,4 @@ We hope these new features are helpful, and we would appreciate any feedback tha The updates in this release are just the first step of a larger performance-oriented direction for Zarr-Python. Landing these two enhancements taught us a *lot* about the performance-sensitive areas of the library. We can and will invest more time in performance tuning, e.g. by adding or changing abstractions, writing code for special cases, etc. -But we should also look to wrapping lower-level libraries that simply have an easier path to performance by virtue of being written in a compiled language. The [`zarrs`](https://zarrs.dev/) project is an ecosystem of Zarr tools written in Rust, with [extremely high performance](https://book.zarrs.dev/#-zarrs-is-fast-). Is there a `zarrs` binding in Zarr-Python's future? I hope so! We are keenly observing development of [`zarrista`](https://developmentseed.org/zarrista/latest/) as a proof-of-concept for what a Python-`zarrs` binding layer might look like. Stay tuned! +We plan to consider including compiled code that should enable significant performance improvements. The [`zarrs`](https://zarrs.dev/) project is an ecosystem of Zarr tools written in Rust, with [extremely high performance](https://book.zarrs.dev/#-zarrs-is-fast-). Is there a `zarrs` binding in Zarr-Python's future? I hope so! We are keenly observing development of [`zarrista`](https://developmentseed.org/zarrista/latest/) as a proof-of-concept for what a Python-`zarrs` binding layer might look like. Stay tuned! From 015732f58cc67888f7f638e09ca447c9349c85eb Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Tue, 28 Jul 2026 22:06:03 +0200 Subject: [PATCH 22/24] feat(zarr-metadata): Zarr metadata model layer (#4119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): bump the actions group across 1 directory with 8 updates (#176) Bumps the actions group with 8 updates in the / directory: | Package | From | To | | --- | --- | --- | | [prefix-dev/setup-pixi](https://github.com/prefix-dev/setup-pixi) | `0.9.5` | `0.9.6` | | [codecov/codecov-action](https://github.com/codecov/codecov-action) | `6.0.0` | `6.0.1` | | [github/issue-metrics](https://github.com/github/issue-metrics) | `4.2.2` | `4.2.7` | | [j178/prek-action](https://github.com/j178/prek-action) | `2.0.3` | `2.0.4` | | [actions/upload-artifact](https://github.com/actions/upload-artifact) | `7.0.0` | `7.0.1` | | [actions/download-artifact](https://github.com/actions/download-artifact) | `7.0.0` | `8.0.1` | | [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) | `1.13.0` | `1.14.0` | | [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) | `0.5.3` | `0.5.6` | Updates `prefix-dev/setup-pixi` from 0.9.5 to 0.9.6 - [Release notes](https://github.com/prefix-dev/setup-pixi/releases) - [Commits](https://github.com/prefix-dev/setup-pixi/compare/1b2de7f3351f171c8b4dfeb558c639cb58ed4ec0...5185adfbffb4bd703da3010310260805d89ebb11) Updates `codecov/codecov-action` from 6.0.0 to 6.0.1 - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/57e3a136b779b570ffcdbf80b3bdc90e7fab3de2...e79a6962e0d4c0c17b229090214935d2e33f8354) Updates `github/issue-metrics` from 4.2.2 to 4.2.7 - [Release notes](https://github.com/github/issue-metrics/releases) - [Commits](https://github.com/github/issue-metrics/compare/c9e9838147fd355dace335ba787f01b6641a400a...1e38d5e62363e14db8019ed7d106b9855bdba6cc) Updates `j178/prek-action` from 2.0.3 to 2.0.4 - [Release notes](https://github.com/j178/prek-action/releases) - [Commits](https://github.com/j178/prek-action/compare/6ad80277337ad479fe43bd70701c3f7f8aa74db3...bdca6f102f98e2b4c7029491a53dfd366469e33d) Updates `actions/upload-artifact` from 7.0.0 to 7.0.1 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v7...043fb46d1a93c77aae656e7c1c64a875d1fc6a0a) Updates `actions/download-artifact` from 7.0.0 to 8.0.1 - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v7...3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c) Updates `pypa/gh-action-pypi-publish` from 1.13.0 to 1.14.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.13.0...cef221092ed1bacb1cc03d23a2d87d1d172e277b) Updates `zizmorcore/zizmor-action` from 0.5.3 to 0.5.6 - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/b1d7e1fb5de872772f31590499237e7cce841e8e...5f14fd08f7cf1cb1609c1e344975f152c7ee938d) --- updated-dependencies: - dependency-name: prefix-dev/setup-pixi dependency-version: 0.9.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: codecov/codecov-action dependency-version: 6.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: github/issue-metrics dependency-version: 4.2.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: j178/prek-action dependency-version: 2.0.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/upload-artifact dependency-version: 7.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/download-artifact dependency-version: 8.0.1 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: pypa/gh-action-pypi-publish dependency-version: 1.14.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: zizmorcore/zizmor-action dependency-version: 0.5.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feat(zarr-metadata): add model._validation — structural validators Assisted-by: ClaudeCode:claude-fable-5 * feat(zarr-metadata): add model._array — array metadata models Assisted-by: ClaudeCode:claude-fable-5 * test(zarr-metadata): port array model test suite Assisted-by: ClaudeCode:claude-fable-5 * feat(zarr-metadata): add group metadata models Assisted-by: ClaudeCode:claude-fable-5 * test(zarr-metadata): add consolidated metadata model tests Assisted-by: ClaudeCode:claude-fable-5 * feat(zarr-metadata): export model layer from package front door Assisted-by: ClaudeCode:claude-fable-5 * docs(zarr-metadata): changelog entry for the model layer Assisted-by: ClaudeCode:claude-fable-5 * feat(zarr-metadata): harden model validation and error reporting Findings from an API-ergonomics exercise (a fresh agent consuming defective metadata documents): - ValidationProblem gains a machine-readable kind (missing_key / invalid_type / invalid_value / invalid_json), ending message string-matching in consumers. - The v2 array validator now enforces what its types declare (dtype, order, compressor, filters, dimension_separator), and all four document validators check the fixed zarr_format / node_type literals. - All ingestion failures surface as MetadataValidationError: missing store keys and undecodable bytes in from_key_value (previously KeyError / JSONDecodeError) and constructor invariants (previously bare ValueError). - ZarrMetadataV3 is renamed NamedConfigModelV3: it models a name + configuration pair, and the old name read as a whole-document type. - Discoverability: the validate_*/is_*/parse_* contract is documented on zarr_metadata.model itself; update() documents that it does not re-validate; the v2 to_json/to_key_value attributes split is documented on both. Assisted-by: ClaudeCode:claude-fable-5 * feat(zarr-metadata): annotate metadata fields with role alias MetadataFieldModelV3 Model fields and consumer signatures should convey the logical meaning of the type (a metadata-document field), not the form it takes when JSON-serialized (a named configuration). MetadataFieldModelV3 is today exactly NamedConfigModelV3; if a future spec revision adds a field form that cannot normalize to name + configuration, the alias widens to a union and annotation sites do not move. Mirrors the raw-layer split between NamedConfigV3 (shape) and MetadataV3 (field union). Assisted-by: ClaudeCode:claude-fable-5 * test(zarr-metadata): assert required-key coverage via the typed constant test_v3_to_json_includes_required_fields hand-enumerated keys with chained asserts, restating what ARRAY_METADATA_REQUIRED_KEYS_V3 already defines. Now: one coverage assert driven by the constant (tracks the TypedDict automatically) and one whole-document equality for the values. Assisted-by: ClaudeCode:claude-fable-5 * test(zarr-metadata): single whole-document comparison for v3 to_json The subset assert against ARRAY_METADATA_REQUIRED_KEYS_V3 was redundant: equality with a literal that spells out the full document already covers every required key. One dict, one assert. Assisted-by: ClaudeCode:claude-fable-5 * fix(zarr-metadata): close validation holes found by adversarial review Invalid documents that previously passed validation: - shape/chunks containing JSON booleans (bool is an int subclass in Python but not an integer in a metadata document) or negative values - dimension_names whose length does not match shape - attributes and configuration values that are not JSON-serializable — now checked recursively like fill_value, so an int-keyed dict cannot be silently rewritten by json.dumps on round-trip and a set() cannot escape as a TypeError from to_key_value - consolidated_metadata envelopes: the group validator now deep-validates the envelope and its entries via the shared validate_consolidated_metadata_v3, which ConsolidatedMetadataModelV3 .from_json also uses, so is_group_metadata_v3 never vouches for a document the model constructor would reject Three pre-existing test fixtures paired dimension_names=('x',) with the default scalar shape () and were themselves spec-invalid; they now use a matching 1-d shape. Deliberately unchanged, pending a design decision: unknown extension fields with must_understand: true still pass (which layer owns the spec's refusal duty), and empty v2 dtype records / empty codec names still pass (domain territory). Assisted-by: ClaudeCode:claude-fable-5 * feat(zarr-metadata): expose must_understand_fields on the v3 models The v3 core spec: 'An implementation MUST fail to open Zarr groups or arrays if any metadata fields are present which (a) the implementation does not recognize and (b) are not explicitly set to "must_understand": false' — and fields are implicitly must-understand unless waived. The model layer cannot discharge this itself: recognition is reader-specific (consolidated_metadata is itself an extension field one reader understands and another does not), and a document carrying a must-understand extension is still a valid document. So the models partition by obligation: must_understand_fields is the subset of extra_fields not explicitly waived, and a compliant reader fails to open when must_understand_fields.keys() - recognized is non-empty. The design spec pins that duty on the part-2 resolve layer, matching what zarr-python's parse_extra_fields enforces today. Assisted-by: ClaudeCode:claude-fable-5 * test(zarr-metadata): executable example of pydantic integration Delegate wholesale rather than letting pydantic introspect the dataclass: InstanceOf (is-instance core schema) + BeforeValidator(from_json) + PlainSerializer(to_json, return_type=dict). Field-by-field validation is impossible anyway (the models' annotation-only imports live behind TYPE_CHECKING, so pydantic raises class-not-fully-defined) and would diverge from the library's structural validation via coercion if it weren't. MetadataValidationError subclasses ValueError, so failed parses surface as pydantic ValidationError with the loc-annotated messages. pydantic is already in the package's test dependency group. Assisted-by: ClaudeCode:claude-fable-5 * test(zarr-metadata): document pydantic's native dataclass path and why not Correcting the previous commit's too-strong claim: pydantic CAN introspect the model dataclass — TypeAdapter(...).rebuild() with the TYPE_CHECKING-only names supplied as _types_namespace resolves the schema, and __post_init__ invariants still run. A new test exercises that path and pins why it is not the recommended integration: it validates the model shape, not the document (bare-string data_type rejected — no from_json normalization), and pydantic's lax coercion silently re-opens holes the library validators close (shape=[True, -5] coerces to (1, -5); a wrong dimension_names count passes). Assisted-by: ClaudeCode:claude-fable-5 * test(zarr-metadata): engine-backed pydantic BaseModel example (pydantic-zarr pattern) For consumers that want a first-class BaseModel — JSON schema generation and generics for typed attributes, as in pydantic-zarr's ArraySpec — the example adds a third pattern: pydantic-native fields as the user-facing surface, with the library as the engine. A mode='before' validator canonicalizes every input via from_json(...).to_json(), so structural validation and normalization run before pydantic parses fields (the [True, -5] coercion divergence cannot occur), and to_metadata_model / to_document bridge both ways through the document form. One translation noted at the bridge: the document spells 'no dimension names' as key absence, the pydantic side as None. Assisted-by: ClaudeCode:claude-fable-5 * test(zarr-metadata): pin that a null dimension_names field is invalid Spec: 'If specified, must be an array of strings or null objects... If dimension_names is not specified, all dimensions are unnamed.' The null object is a permitted element (an unnamed dimension), never the field value; key absence is the only spelling of 'not specified'. Pins the validator's existing rejection so it is not later 'fixed' to accept null-as-absence, and documents that in-memory None maps to key absence on serialization. Assisted-by: ClaudeCode:claude-fable-5 * feat(zarr-metadata): optional pydantic integration as zarr_metadata.pydantic Gamed out three shapes with prototypes before choosing: - dunders on the core classes (works, verified pydantic 2.0-2.13, but puts a framework protocol in the dependency-free layer); - pydantic-aware SUBCLASSES in a namespace (rejected on empirical failures: identity split breaks equality, core instances are rejected by subclass-typed fields, and nested construction produces core-class children unless every cross-reference is overridden); - Annotated field types over the CORE classes in an opt-in module (chosen): instances are the core classes so interop is free, pydantic imports eagerly at the module (loud failure when absent), core stays framework-free, and pydantic-protocol risk is quarantined to one clearly-labeled module. The module exports one field type per model. Validation delegates to from_json (structural validation and normalization cannot be bypassed by pydantic coercion), instances pass through, serialization emits the canonical document, and WithJsonSchema describes the accepted document form so model_json_schema works. Tests cover all seven field types, core-instance interop, error quality, JSON schema, roundtrip, and that importing zarr_metadata does not import pydantic. Assisted-by: ClaudeCode:claude-fable-5 * fix(zarr-metadata): create_default derives the chunk grid from shape create_default(shape=(100, 100)) silently kept the scalar default's 0-d chunk grid (chunk_shape: ()), producing a structurally-valid but semantically inconsistent document — a footgun for every test fixture built on it. When shape is overridden and the grid is not, the default is now one regular chunk covering the array (v3 chunk_shape == shape, v2 chunks == shape); an explicit chunk_grid/chunks override still wins. update() stays a dumb dataclasses.replace, per its documented contract. One existing whole-document test literal carried exactly this inconsistency (shape (10,) with chunk_shape ()) and was updated. Assisted-by: ClaudeCode:claude-fable-5 * test(zarr-metadata): pin zero-length-dimension case of the derived chunk grid The spec's constraint is conditional ('non-zero when the corresponding dimensions of the arrays have non-zero length'), so chunk_shape == shape is sound for every shape, including empty dimensions. Assisted-by: ClaudeCode:claude-fable-5 * docs(zarr-metadata): document that create_default's derivation is one-way Overriding shape without a grid derives the grid; the reverse does not hold. A user-supplied chunk_grid is an extension point taken verbatim — deriving shape from it would require interpreting grid configurations, which the model layer never does and cannot do for unrecognized grid names. Pinned by test so the asymmetry reads as a decision, not an oversight; the v2 model documents the same one-way rule for chunks for cross-version consistency. Assisted-by: ClaudeCode:claude-fable-5 * refactor(zarr-metadata): eliminate every type-ignore comment Audited all 24 (15 src, 9 tests); each was either obsolete, replaceable by a sound cast, or avoidable by better-typed code: - Two fill_value arg-type ignores were factually obsolete: their justifying comment said 'fill_value: object in upstream TypedDict', but 0.3.0 narrowed it to JSONValue. - Eight pre-existing call-arg/reportInvalidTypeForm ignores on the PEP 728 TypedDicts and the recursive JSONValue alias were mypy-dialect suppressions that the checker of record (pyright strict with enableExperimentalFeatures) never needed; mypy has never checked this package. - The two extra_fields comprehensions are a genuine checker limitation (a key filter cannot narrow a PEP 728 TypedDict's item-value union), now expressed as casts whose comments state the soundness claim instead of suppressing the diagnostic. - pydantic.py's generic coercer factory takes the parse callable explicitly instead of calling from_json through type[_M]. - NamedConfigModelV3.from_json casts the validated configuration (sound since configuration values are now deep-validated as JSON). - Tests: _build_v2/_build_v3 gained real Unpack[...Partial] signatures; raw-document pydantic inputs go through model_validate (the idiomatic entry point for untyped data) instead of ignoring constructor signatures; the frozen-dataclass test uses setattr for its intentional runtime error. src and tests/model now carry zero type-ignore comments. Assisted-by: ClaudeCode:claude-fable-5 * fix(zarr-metadata): absent v2 dimension_separator means '.', not '/' roborev job 426 (branch review) found that ArrayMetadataModelV2 normalized an ABSENT dimension_separator key to '/', inherited verbatim from the zng prototype. The v2 convention's default is '.': a consumer deriving chunk keys from the model against a real-world v2 array written with the default separator would have looked for '0/0' instead of '0.0'. No test caught it because every fixture started from create_default(), which always carries an explicit separator. Absence is normalized to an explicit '.' -- a semantics-preserving spelling normalization consistent with the model's existing canonical forms (bare-string metadata fields, missing configuration). The field is deliberately NOT modeled as Optional: the document grammar has no null spelling for this key, and a None in the model invites writing 'dimension_separator': null into documents. Pinned by three tests, including explicit-null rejection. Assisted-by: ClaudeCode:claude-fable-5 * feat(zarr-metadata): UNSET sentinel for absent optional document keys Establishes the models' None/absence invariant: None in a model always corresponds to a JSON null in the document (a v2 compressor/filters value, an unnamed dimension inside dimension_names), and UNSET always means the key is absent. The two are never interchangeable. Applied to the two fields that used None as an absence marker: dimension_names (ArrayMetadataModelV3) and consolidated_metadata (GroupMetadataModelV3). For dimension_names this also preserves a semantic distinction d-v-b identified: an absent field ("there are no dimension names") and an explicit all-null array ("every dimension has a name, which is null") are different documents; both spellings now round-trip faithfully and compare unequal. Normalizing absence to the all-null form was considered and rejected: the spellings' interpretations coincide but interpretation-equivalence is the resolve layer's business, and collapsing document-level distinctions on that basis is the layer violation this package exists to avoid. Verified that current zarr-python never writes "consolidated_metadata": null (GroupMetadata.to_dict pops the key), so None there was purely an absence marker, not a document spelling. UnsetType is a single-member enum (identity-checkable, repr "UNSET", deliberately truthy so `if not x` cannot silently treat it as absent); UNSET and UnsetType are exported from zarr_metadata.model and the package front door. Assisted-by: ClaudeCode:claude-fable-5 * fix(zarr-metadata): accept and preserve the wild consolidated_metadata null Historical zarr-python versions wrote "consolidated_metadata": null into group documents for groups without consolidated metadata, so real stores contain the spelling; the validator was rejecting those documents ("expected a mapping"). Per the None/UNSET invariant, the field is now honestly three-state: UNSET (key absent), None (the document's literal null, preserved on round-trip), or a ConsolidatedMetadataModelV3. Interpreting null as absence is the consumer's call, not a document rewrite by this layer. Also records an implementation constraint on the sentinel itself: typing_extensions.Sentinel (PEP 661) is the intended spelling, but pyright 1.1.411 degrades a Sentinel to Unknown in dataclass FIELD annotations (function signatures work), verified by probe both with and without enableExperimentalFeatures. Using it would reintroduce suppressions at every use site under the strict gate, so UNSET stays a single-member enum, with the Sentinel switch documented in _sentinel.py for when pyright catches up. Assisted-by: ClaudeCode:claude-fable-5 * fix(zarr-metadata): repair consolidated_metadata null to absence, not preserve it d-v-b: the bugged spelling should not be preserved or honored. The three-state field reverts to two states (model | UNSET): a document carrying "consolidated_metadata": null — written by a historical zarr-python bug — remains readable (the validator accepts it so real stores open), but the spelling gets no model representation: it is read as absence and never written back. This is the one deliberate exception to faithful round-tripping, pinned as such: from_json(null_doc) equals from_json(absent_doc), and to_json omits the key. Assisted-by: ClaudeCode:claude-fable-5 * docs(zarr-metadata): pin the Sentinel blocker to pyright regression #11115 Investigated: the Unknown-degradation of typing_extensions.Sentinel is a confirmed upstream pyright regression, not by-design. Introduced in 1.1.405 (verified: 1.1.404 is clean on the same probe, 1.1.411 fails), affects reads of any class-body attribute annotation (dataclass or plain class), does not affect function signatures or module variables, and Final on the sentinel does not help. Tracked as microsoft/pyright#11115 (open, bug+regression); #11467 closed as its duplicate. The enum sentinel stays until the fix lands. Assisted-by: ClaudeCode:claude-fable-5 * docs(zarr-metadata): sentinel switch is blocked by mypy too, not just pyright Pinning a working pyright (<= 1.1.404) in CI was considered and does not suffice: the pin controls one of four checker surfaces. Contributor IDEs (Pylance bundles current pyright) and downstream consumers' pyright read the py.typed inline annotations with their own versions, and decisively, mypy 2.1.0 has no PEP 661 support at all — a sentinel in type position is a hard [valid-type] error, which would degrade these fields to Any for mypy consumers, including zarr-python itself. The enum is currently the only spelling with exact types on every surface; switch when pyright#11115 is fixed AND mypy implements PEP 661. Assisted-by: ClaudeCode:claude-fable-5 * docs(zarr-metadata): PEP 661 is Final (Python 3.15), not a draft Corrects the sentinel implementation note: PEP 661 was accepted 2026-04-23 and ships as stdlib sentinel in Python 3.15. The two checker gaps blocking the Sentinel spelling (pyright regression #11115, mypy not yet implementing the PEP) are therefore temporary gaps against a Final standard, and the enum is a stopgap with a defined end state. Assisted-by: ClaudeCode:claude-fable-5 * docs(zarr-metadata): ty fully supports typed sentinels; pyright/mypy are the laggards ty 0.0.56 types the Sentinel spelling perfectly in dataclass fields: exact T | UNSET unions, both-direction is/is-not narrowing, and wrong-typed constructor arguments rejected (verified with reveal_type, so it is real inference, not silent Any). The checker matrix for sentinel-in-type-position is therefore ty full / pyright regressed (#11115) / mypy not implemented — recorded so the switch decision has current calibration. Assisted-by: ClaudeCode:claude-fable-5 * feat(zarr-metadata): adopt the PEP 661 sentinel for UNSET d-v-b's call: PEP 661 is Final, ty already types the sentinel spelling exactly, mypy support is in review (python/mypy#21647) and treated as imminent, and pyright has a known-good version — so use the standard sentinel today rather than carrying the enum stopgap. - UNSET is now typing_extensions.Sentinel("UNSET"), used directly in type expressions (tuple[str | None, ...] | UNSET); the UnsetType companion enum is gone from the API. - typing_extensions floor bumped to 4.14 (where Sentinel arrived). - CI pins pyright==1.1.404, the last version before the class-attribute sentinel regression (microsoft/pyright#11115); pyproject documents the same pin for local runs. 0 errors on the pin; ty checks the sentinel fields clean (its 2 remaining diagnostics are its incomplete PEP 728 extra_items write support, unrelated). - Known short-term cost, accepted deliberately: mypy-checked consumers need cast/type-ignore at narrowing sites until mypy#21647 merges, and contributors' Pylance may show phantom Unknowns until the pyright fix ships. Recorded in _sentinel.py and the changelog. - The pydantic native-introspection test reverts to documenting that introspection is unsupported (pydantic 2.13 cannot schema a Sentinel); the delegation patterns are unaffected. Assisted-by: ClaudeCode:claude-fable-5 * fix(zarr-metadata): .zattrs presence is part of the store, not an artifact Resolves the last flagged round-trip question from the initial port: to_key_value on the v2 models always emitted a .zattrs key, so a store that never had one gained a file on round-trip. Per d-v-b's ruling, attributes on ArrayMetadataModelV2/GroupMetadataModelV2 is now `dict[str, JSONValue] | UNSET`: UNSET means no .zattrs file (and no attributes key in the merged document form) and emits nothing, while any dict — including an explicit empty {} — means the file exists and is emitted. The two spellings stay distinct through round-trips, per the None/UNSET invariant; create_default defaults to UNSET (a fresh minimal node has no .zattrs). Assisted-by: ClaudeCode:claude-fable-5 * fix(zarr-metadata): require typing_extensions>=4.16 so UNSET pickles by reference Models hold the UNSET sentinel as field values (dimension_names, attributes), so any object graph containing a model must survive pickling and deep-copying. A sentinel's contract is identity — state-based pickling would produce impostor objects that fail every `is UNSET` check — which is why typing_extensions <= 4.15 refused to pickle sentinels at all. typing_extensions 4.16 implements Sentinel.__reduce__ as pickling by reference (a lookup of the sentinel's name on its defining module), the same mechanism enum members use, so the singleton identity survives the round trip. Bump the floor and pin the behavior with tests: identity across pickle/copy/deepcopy, models holding UNSET round-tripping, and a guard that a non-importable sentinel still fails loudly rather than pickling by state. Co-Authored-By: Claude Fable 5 * refactor(zarr-metadata): format-version-first naming; dataclasses primary, JSON suffix for documents Applies the naming decisions from the PR discussion: ZarrV2/ZarrV3 moves to the front of every type name so a format version cannot be misread as a class revision, and the model dataclasses take the bare entity names (ZarrV3ArrayMetadata, ZarrV3GroupMetadata, ZarrV3ConsolidatedMetadata, ZarrV3NamedConfig, role alias ZarrV3MetadataField) while the TypedDict document forms carry a JSON suffix (ZarrV3ArrayMetadataJSON, ..., ZarrV3MetadataFieldJSON, ZarrV3NamedConfigJSON). The zarr_metadata.pydantic field types take the bare entity names, matching the model classes they validate into; the module now references the model module qualified to keep those names free. Raw-layer names released in 0.3.0 are renamed without aliases (pre-1.0), documented in changes/4119.removal.md. Validation problem messages name documents in plain English instead of type names. snake_case function names (validate_array_metadata_v3, ...) and SCREAMING_SNAKE constants are deliberately untouched: the revision ambiguity the rename fixes does not arise for them, and renaming them is a separate decision. Co-Authored-By: Claude Fable 5 * fix(zarr-metadata): harden model validation Assisted-by: Codex:gpt-5 * docs(zarr-metadata): define v3 conformance boundary Assisted-by: Codex:gpt-5 * fix(zarr-metadata): preserve v3 extension obligations Assisted-by: Codex:gpt-5 * fix(zarr-metadata): enforce v3 core extension rules Assisted-by: Codex:gpt-5 * fix(zarr-metadata): align v3 additional field types Assisted-by: Codex:gpt-5 * chore(zarr-metadata): finalize v3 model conformance Assisted-by: Codex:gpt-5 * docs: remove llm docs * chore(zarr-metadata): correct changelog PR number Assisted-by: Codex:gpt-5 * fix(metadata): enforce canonical document boundaries Assisted-by: Codex:GPT-5 * docs(metadata): record review fix design Assisted-by: Codex:gpt-5 * fix(metadata): align validation and schemas Assisted-by: Codex:gpt-5 * fix(metadata): tighten validation boundaries Assisted-by: Codex:gpt-5 * fix(metadata): repair package CI Assisted-by: Codex:gpt-5 * chore: remove stray design notes from docs Assisted-by: ClaudeCode:claude-fable-5 * fix(zarr-metadata): deep-copy mutable state in to_json output to_json previously returned documents holding direct references to the model's internal dicts (attributes, named-config configurations, extra fields, v2 compressor/filters, consolidated entries), so mutating a serialized document silently mutated the frozen model. Every value that can hold a mutable container is now deep-copied on the way out, with parametrized tests proving mutation independence for all seven models. Assisted-by: ClaudeCode:claude-fable-5 * docs(zarr-metadata): describe the widened package scope The README tagline, intro, and scope section (and the PyPI description) still presented the package as type definitions only. Restructure them around the two layers plus optional integration, extend the contribution scope to models and structural validation, and state the runtime-behavior boundary explicitly. Assisted-by: ClaudeCode:claude-fable-5 * style(zarr-metadata): move None to the end of the JSONValue union The unpinned ruff in CI now enforces RUF036. Assisted-by: ClaudeCode:claude-fable-5 * feat(zarr-metadata): export store-key aliases and type to_key_value with them The six store-key Literal aliases were private and unused: only reachable via underscore modules, and absent from every signature. Export them from zarr_metadata.model beside their constants (matching the package's name/constant pairing everywhere else), and key each to_key_value return mapping by them so the store keys a model can emit are visible in its signature. from_key_value keeps Mapping[str, bytes] input on purpose — it accepts whole store mappings. A pair test guards export and value drift, and the removal note now states the version-placement and JSON-suffix conventions explicitly. Assisted-by: ClaudeCode:claude-fable-5 * refactor(zarr-metadata): make every public type name parse against a naming grammar Three grammars now cover the public surface, enforced by a conformance test that walks every public module's __all__: - core document/model names: ZarrV{2,3} + entity + optional role suffix (JSON / JSONPartial / Partial / StoreKey) - extension-entity names: registered entity + exactly one role suffix (CodecMetadata, DataTypeName, FillValue, ...) - a closed standalone-vocabulary allowlist for role-less scalar and diagnostic types, with a staleness guard The three .z-file document types were the only names that fit no grammar and are renamed: ZArrayMetadata -> ZarrV2ZArrayJSON, ZGroupMetadata -> ZarrV2ZGroupJSON, ZAttrsMetadata -> ZarrV2ZAttrsJSON. The leading V2 of V2ChunkKeyEncodingMetadata is that encoding's registered entity name, not a format version; its module docstring now says so. Assisted-by: ClaudeCode:claude-fable-5 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- .github/workflows/zarr-metadata.yml | 5 +- packages/zarr-metadata/README.md | 81 +- .../zarr-metadata/changes/4119.feature.md | 92 + .../zarr-metadata/changes/4119.removal.md | 41 + packages/zarr-metadata/pyproject.toml | 19 +- .../src/zarr_metadata/__init__.py | 114 +- .../src/zarr_metadata/_common.py | 16 +- .../src/zarr_metadata/_pydantic_schema.py | 114 ++ .../src/zarr_metadata/model/__init__.py | 132 ++ .../src/zarr_metadata/model/_array.py | 498 +++++ .../src/zarr_metadata/model/_group.py | 442 +++++ .../src/zarr_metadata/model/_sentinel.py | 37 + .../src/zarr_metadata/model/_validation.py | 875 +++++++++ .../src/zarr_metadata/pydantic.py | 176 ++ .../src/zarr_metadata/v2/__init__.py | 38 +- .../src/zarr_metadata/v2/array.py | 83 +- .../src/zarr_metadata/v2/attributes.py | 6 +- .../src/zarr_metadata/v2/codec.py | 4 +- .../src/zarr_metadata/v2/consolidated.py | 18 +- .../src/zarr_metadata/v2/group.py | 26 +- .../src/zarr_metadata/v3/__init__.py | 18 +- .../src/zarr_metadata/v3/_common.py | 10 +- .../src/zarr_metadata/v3/array.py | 80 +- .../zarr_metadata/v3/chunk_key_encoding/v2.py | 6 + .../src/zarr_metadata/v3/codec/__init__.py | 2 +- .../src/zarr_metadata/v3/codec/cast_value.py | 4 +- .../src/zarr_metadata/v3/codec/crc32c.py | 2 +- .../v3/codec/sharding_indexed.py | 6 +- .../src/zarr_metadata/v3/consolidated.py | 28 +- .../src/zarr_metadata/v3/data_type/struct.py | 4 +- .../src/zarr_metadata/v3/group.py | 20 +- .../zarr-metadata/tests/model/__init__.py | 0 packages/zarr-metadata/tests/model/_cases.py | 65 + .../zarr-metadata/tests/model/test_array.py | 1738 +++++++++++++++++ .../zarr-metadata/tests/model/test_group.py | 572 ++++++ .../tests/model/test_pydantic.py | 302 +++ .../tests/model/test_pydantic_module.py | 264 +++ .../tests/model/test_sentinel.py | 89 + .../tests/test_partial_equivalence.py | 16 +- .../zarr-metadata/tests/test_public_api.py | 164 +- .../tests/v2/array/test_fixtures.py | 6 +- .../tests/v2/consolidated/test_fixtures.py | 4 +- .../tests/v2/group/test_fixtures.py | 6 +- .../tests/v3/array/test_fixtures.py | 6 +- .../tests/v3/array/with_extra_field.json | 2 +- .../tests/v3/consolidated/test_fixtures.py | 4 +- .../tests/v3/group/test_fixtures.py | 4 +- 47 files changed, 5940 insertions(+), 299 deletions(-) create mode 100644 packages/zarr-metadata/changes/4119.feature.md create mode 100644 packages/zarr-metadata/changes/4119.removal.md create mode 100644 packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/__init__.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/_array.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/_group.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/_validation.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/pydantic.py create mode 100644 packages/zarr-metadata/tests/model/__init__.py create mode 100644 packages/zarr-metadata/tests/model/_cases.py create mode 100644 packages/zarr-metadata/tests/model/test_array.py create mode 100644 packages/zarr-metadata/tests/model/test_group.py create mode 100644 packages/zarr-metadata/tests/model/test_pydantic.py create mode 100644 packages/zarr-metadata/tests/model/test_pydantic_module.py create mode 100644 packages/zarr-metadata/tests/model/test_sentinel.py diff --git a/.github/workflows/zarr-metadata.yml b/.github/workflows/zarr-metadata.yml index 4e5bb0fb1a..df7d96cc1c 100644 --- a/.github/workflows/zarr-metadata.yml +++ b/.github/workflows/zarr-metadata.yml @@ -82,7 +82,10 @@ jobs: - name: Sync test dependency group run: uv sync --group test --python 3.11 - name: Run pyright - run: uv run --group test --with pyright pyright src + # Pinned to the last version that types PEP 661 sentinels in class + # attributes correctly; 1.1.405+ regressed (microsoft/pyright#11115). + # Unpin when the fix lands. + run: uv run --group test --with 'pyright==1.1.404' pyright src zarr-metadata-complete: name: zarr-metadata complete diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index a842e07886..69b80d7332 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -1,51 +1,86 @@ # zarr-metadata -Python type definitions for Zarr v2 and v3 metadata. +Python types, models, and validators for Zarr v2 and v3 metadata. ## What this is -A typed-data package: `TypedDict` definitions and `Literal` aliases for the -JSON shapes specified by the [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html) -and [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) -specifications, plus types for [`zarr-extensions`](https://github.com/zarr-developers/zarr-extensions/) -and a few widely-used-but-unspecified entities (e.g. consolidated metadata). +Two layers and an optional integration: + +- **Typed JSON shapes**: `TypedDict` definitions and `Literal` aliases for the + JSON documents specified by the [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html) + and [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) + specifications, plus types for [`zarr-extensions`](https://github.com/zarr-developers/zarr-extensions/) + and a few widely-used-but-unspecified entities (e.g. consolidated metadata). +- **Document models** (`zarr_metadata.model`): canonical frozen-dataclass + models of whole metadata documents, with structural validators, loc-aware + parsers, and store-key (de)serialization. A document produced by `to_json` + shares no mutable state with the model that produced it. +- **Optional Pydantic integration** (`zarr_metadata.pydantic`, requires + Pydantic 2.13 or newer): each model as a Pydantic field type that validates + raw documents through the same strict parser. ## What this is for -These types describe the JSON shape of Zarr metadata. They are -intended for libraries that **read, write, validate, or transform** -Zarr metadata. Pair them with a runtime validator like -[pydantic](https://docs.pydantic.dev/) to check JSON loaded from disk: +The public `TypedDict` definitions describe the static JSON shape of Zarr +metadata. For strict, loc-aware validation of JSON loaded from disk, use the +model parser: ```python import json -from pydantic import TypeAdapter -from zarr_metadata.v3.array import ArrayMetadataV3 +from zarr_metadata.model import ZarrV3ArrayMetadata with open("zarr.json", "rb") as f: raw = json.load(f) -metadata = TypeAdapter(ArrayMetadataV3).validate_python(raw) +metadata = ZarrV3ArrayMetadata.from_json(raw) +``` + +The optional Pydantic integration delegates raw input to the same strict +parser and returns the same normalized model class: + +```python +from pydantic import TypeAdapter +import zarr_metadata.pydantic as zmp + +metadata = TypeAdapter(zmp.ZarrV3ArrayMetadata).validate_python(raw) +encoded = metadata.to_key_value()["zarr.json"] ``` -## What this is *not* +A bare `TypeAdapter` over a public document `TypedDict` is a coercive shape +adapter, not a Zarr conformance validator; it may coerce values or discard +members that the strict model parser rejects. + +## Validation boundary -- Not a parser or builder. There are no `make_array_metadata(...)` factories — - that surface belongs to consumer libraries. -- Not a runtime validator on its own. Pair with `pydantic`, `msgspec`, or - similar to enforce shapes at decode time. +The model validators enforce the declared document structure and a small set +of context-free consistency rules, including fixed format literals, finite +JSON numbers, non-negative dimensions, non-empty v3 codec pipelines, and one +`dimension_names` entry per array dimension. They do not interpret extension +names or configurations, resolve codec pipelines, or decide whether a data +type, chunk grid, codec, or storage transformer is supported. Those decisions +belong to consumer implementations. -Even with a runtime validator, these types only describe **structural** -shape — they will not flag *semantically* invalid metadata, like a 3D v3 -array whose `dimension_names` has 4 entries instead of 3. That's a job -for downstream validator routines. +The Pydantic integration's generated JSON Schemas express independently +checkable document structure and field constraints, but they are not a +replacement for runtime model validation. Standard JSON Schema treats a +mathematically integral number such as `1.0` as an integer, while the runtime +boundary requires Python `int` values, and it cannot express arbitrary +same-length relations such as `dimension_names` versus `shape` or v2 `chunks` +versus `shape`. Consumers should run the model parser after schema validation. ## Scope At minimum, this library supports what Zarr-Python needs: the complete Zarr v2 and v3 specs, consolidated metadata, and a subset of the metadata defined in `zarr-extensions`. We are generally open to contributions that -add types for Zarr metadata with a published spec. +add types, models, or structural validation for Zarr metadata with a +published spec. + +Runtime array behavior is out of scope: nothing here encodes or decodes +chunks, resolves codec or data type names to implementations, or performs +store I/O. The models begin and end at the metadata documents themselves — +`from_key_value` / `to_key_value` map documents to store keys and bytes, +and everything past that belongs to consumer libraries. ## Releasing diff --git a/packages/zarr-metadata/changes/4119.feature.md b/packages/zarr-metadata/changes/4119.feature.md new file mode 100644 index 0000000000..b9d0bb508c --- /dev/null +++ b/packages/zarr-metadata/changes/4119.feature.md @@ -0,0 +1,92 @@ +Added `zarr_metadata.model`: frozen-dataclass models (`ZarrV2ArrayMetadata`, +`ZarrV3ArrayMetadata`, `ZarrV2GroupMetadata`, `ZarrV3GroupMetadata`, +`ZarrV2ConsolidatedMetadata`, `ZarrV3ConsolidatedMetadata`, `ZarrV3NamedConfig`) +that are canonical, semantically lossless representations of Zarr metadata +documents, plus structural validators (`validate_*` / `is_*` / `parse_*`). +Every v3 extension point (data type, chunk grid, chunk key encoding, codecs, +storage transformers) is held as `ZarrV3NamedConfig`: a name, configuration, +and `must_understand` obligation; nothing is interpreted. On the wire, an +empty configuration with the default obligation uses the spec's plain-string +shorthand. Model fields are annotated with the role alias +`ZarrV3MetadataField` (today exactly `ZarrV3NamedConfig`), so annotations +convey the logical meaning and stay put if the spec adds another field form. + +Validation is strict about what the types declare: v2 `dtype` / `order` / +`compressor` / `filters` / `dimension_separator` shapes and the fixed +`zarr_format` / `node_type` literals are all enforced. Every +`ValidationProblem` carries a machine-readable `kind` +(`missing_key` / `invalid_type` / `invalid_value` / `invalid_json`) so +consumers can dispatch on the failure mode without matching message strings, +and every ingestion failure — including missing store keys and undecodable +bytes in `from_key_value` — surfaces as `MetadataValidationError`. An +adversarial review added further structural checks: JSON booleans are not +accepted as dimension lengths, dimensions are non-negative, +`dimension_names` must have one entry per dimension of `shape`, `attributes` +and `configuration` values are JSON-checked recursively (like `fill_value`), +non-finite floats and non-standard JSON constants are rejected, abstract +mappings and sequences normalize to encoder-safe canonical containers, +v2 `shape` and `chunks` must have the same rank, non-null v2 filter pipelines +contain at least one filter, document `TypeIs` guards only narrow values that +already use the declared canonical containers, +and the inline consolidated-metadata envelope and entries are deep-validated +so the group validator's verdict always agrees with the model constructor. + +The v3 models expose `must_understand_fields`: the subset of `extra_fields` +not explicitly waived with `must_understand: false` (fields are implicitly +must-understand per the spec). Readers discharge the spec's fail-to-open +duty by subtracting the extension names they recognize; the model only +partitions by obligation, since recognition is reader-specific. + +Optional pydantic integration ships as `zarr_metadata.pydantic` (importing it +requires pydantic 2.13 or newer; the core package does not depend on it): one +`Annotated` +field type per model, validating raw documents through `from_json`, passing +core-model instances through unchanged, serializing via `to_json`, and +publishing JSON Schemas derived from private constrained document types that +mirror the independently expressible runtime rules. Cross-field cardinality +relations still require runtime validation. The instances are the core model +classes, so values interoperate freely with non-pydantic code. + +`create_default` keeps its output self-consistent: overriding `shape` without +a chunk grid derives one regular chunk covering the array (v3 +`chunk_shape == shape`; v2 `chunks == shape`) instead of silently keeping the +scalar default's 0-d grid. + +A v2 `.zarray` that omits `dimension_separator` is interpreted with the v2 +convention's default `"."` (the model previously normalized absence to `"/"`, +which would misaddress the chunks of real-world default-separator arrays). +The value is never null: absent, `"."`, or `"/"` are the only spellings. + +Optional document keys use `UNSET` — a PEP 661 sentinel +(`typing_extensions.Sentinel`), usable directly in type expressions — never +`None`: in a model, `None` always corresponds to a JSON `null` in the +document (a v2 `compressor`, an unnamed dimension inside `dimension_names`), +and `UNSET` always means the key is absent. Checker note: ty types the +sentinel exactly; pyright needs `<= 1.1.404` until microsoft/pyright#11115 +is fixed (this package's CI pins it); mypy users need a `cast` or +`type: ignore` at narrowing sites until python/mypy#21647 merges. This keeps semantically distinct spellings +distinct — an absent `dimension_names` ("there are no dimension names") and +an explicit `[null, null]` ("every dimension has a name, which is null") are +different documents and round-trip as such. The `consolidated_metadata: null` +written by a historical zarr-python bug is the one deliberate exception to +faithful round-tripping: those stores remain readable, but the bug spelling +is repaired to absence on read and never written back. + +The v2 models treat the `.zattrs` file's presence as part of the store: +`attributes` is `UNSET` when no `.zattrs` file exists (and `to_key_value` +emits none), while an explicit empty `.zattrs` is `{}` and round-trips as a +file. Previously `to_key_value` always emitted `.zattrs`, silently adding a +file to stores that never had one. + +The store-key `Literal` aliases (`ZarrV2ArrayMetadataStoreKey`, +`ZarrV2AttributesStoreKey`, ...) are exported from `zarr_metadata.model` +alongside their constants, and each `to_key_value` return type is keyed by +them, so the set of store keys a model can emit is visible in its signature. +`from_key_value` deliberately keeps `Mapping[str, bytes]` input: it accepts +any string-keyed store mapping and ignores unrelated keys. + +`to_json` returns a document that shares no mutable state with the model: +every value that can hold a mutable container (attributes, configurations, +extra fields, v2 codec configurations, fill values, consolidated entries) is +deep-copied on the way out, so editing a serialized document can never +silently mutate the frozen model that produced it. diff --git a/packages/zarr-metadata/changes/4119.removal.md b/packages/zarr-metadata/changes/4119.removal.md new file mode 100644 index 0000000000..2a9a6f84c4 --- /dev/null +++ b/packages/zarr-metadata/changes/4119.removal.md @@ -0,0 +1,41 @@ +The document (TypedDict) types are renamed to put the format version at the +front of the name and to mark the JSON-document form with a `JSON` suffix, +so a format version can never be misread as a class revision and the bare +entity names are reserved for the `zarr_metadata.model` dataclasses: + +- `ArrayMetadataV2` → `ZarrV2ArrayMetadataJSON` (and `...Partial` accordingly) +- `ArrayMetadataV3` → `ZarrV3ArrayMetadataJSON` (and `...Partial` accordingly) +- `GroupMetadataV2` → `ZarrV2GroupMetadataJSON` (and `...Partial` accordingly) +- `GroupMetadataV3` → `ZarrV3GroupMetadataJSON` (and `...Partial` accordingly) +- `ConsolidatedMetadataV2` → `ZarrV2ConsolidatedMetadataJSON` +- `ConsolidatedMetadataV3` → `ZarrV3ConsolidatedMetadataJSON` +- `NamedConfigV3` → `ZarrV3NamedConfigJSON` +- `MetadataV3` → `ZarrV3MetadataFieldJSON` (the union of the bare-name and + named-configuration spellings of one metadata field) +- `ExtensionFieldV3` → `ZarrV3ExtensionField` +- `CodecMetadataV2` → `ZarrV2CodecMetadata` +- `DataTypeMetadataV2` → `ZarrV2DataTypeMetadata` +- `ArrayOrderV2` → `ZarrV2ArrayOrder` +- `ArrayDimensionSeparatorV2` → `ZarrV2ArrayDimensionSeparator` +- `ZArrayMetadata` → `ZarrV2ZArrayJSON` (the strict on-disk `.zarray` document) +- `ZGroupMetadata` → `ZarrV2ZGroupJSON` (the strict on-disk `.zgroup` document) +- `ZAttrsMetadata` → `ZarrV2ZAttrsJSON` (the `.zattrs` document) + +The old names are removed, not aliased. The `zarr_metadata.pydantic` field +types take the bare entity names (`ZarrV3ArrayMetadata`, ...), matching the +model classes they validate into. + +The conventions, stated once for future additions: CamelCase type names put +the format version first (`ZarrV2ArrayMetadataJSON`, +`ZarrV3ArrayMetadataStoreKey`), while SCREAMING_SNAKE constants and +snake_case functions put it last (`ARRAY_METADATA_STORE_KEY_V2`, +`validate_array_metadata_v3`). The `JSON` suffix marks a raw-document type +whose bare name is taken by (or reserved for) a `zarr_metadata.model` +dataclass; raw field-level types the models hold verbatim +(`ZarrV2CodecMetadata`, `ZarrV3ExtensionField`) keep their bare names. +Extension-entity types put the registered entity name first and end in +exactly one role suffix (`BloscCodecMetadata`, `Uint8DataTypeName`) — the +`V2` in `V2ChunkKeyEncodingMetadata` is that encoding's entity name, not a +format version, which is always spelled `ZarrV2`/`ZarrV3`. Every public +type name is checked against this grammar by +`tests/test_public_api.py::test_public_type_names_comply_with_naming_grammar`. diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 05667d59e3..edc4b696a6 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "zarr-metadata" dynamic = ["version"] -description = "Spec-defined metadata types for Zarr v2 and v3." +description = "Spec-defined metadata types, models, and validators for Zarr v2 and v3." readme = "README.md" requires-python = ">=3.11" license = "MIT" @@ -32,7 +32,11 @@ classifiers = [ ] keywords = ["zarr"] dependencies = [ - "typing_extensions>=4.13", + # >=4.16: first release where `Sentinel` pickles by reference + # (`__reduce__` returns the sentinel's name), so `UNSET` — and any model + # holding it — can cross process boundaries and be deep-copied with its + # singleton identity intact. 4.15 and earlier refuse to pickle sentinels. + "typing_extensions>=4.16", ] [project.urls] @@ -43,7 +47,7 @@ Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/z Documentation = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-metadata/README.md" [dependency-groups] -test = ["pytest", "pydantic>=2"] +test = ["pytest", "pydantic>=2.13", "jsonschema"] [tool.hatch.version] source = "vcs" @@ -67,9 +71,9 @@ xfail_strict = true addopts = ["-ra", "--strict-config", "--strict-markers"] filterwarnings = [ "error", - # pydantic warns about ReadOnly TypedDict items not being enforced at runtime. - # That's expected here — we rely on type-checker enforcement, not pydantic mutation guards. - "ignore::UserWarning:pydantic._internal._generate_schema", + # Pydantic validates these public immutable-shape TypedDicts correctly but + # cannot enforce the type checker's ReadOnly mutation restriction. + "ignore:Items? .* using the `ReadOnly` qualifier.*:UserWarning:pydantic._internal._generate_schema", ] [tool.numpydoc_validation] @@ -82,6 +86,9 @@ checks = [ "PR06", ] +# CI pins pyright==1.1.404: later versions regress PEP 661 sentinel typing in +# class attributes (microsoft/pyright#11115), which zarr_metadata.model._sentinel +# relies on. Use the same pin locally; unpin when the fix lands. [tool.pyright] include = ["src"] enableExperimentalFeatures = true diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index 46949570a2..b5e52e976d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -1,22 +1,48 @@ from importlib.metadata import version -from zarr_metadata._common import JSONValue, NamedConfigV3 +from zarr_metadata._common import JSONValue, ZarrV3NamedConfigJSON +from zarr_metadata.model import ( + UNSET, + MetadataValidationError, + ProblemKind, + ValidationProblem, + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadataPartial, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV2GroupMetadataPartial, + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadataPartial, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, + ZarrV3GroupMetadataPartial, + ZarrV3MetadataField, + ZarrV3NamedConfig, +) from zarr_metadata.v2.array import ( ARRAY_DIMENSION_SEPARATOR_V2, ARRAY_ORDER_V2, - ArrayDimensionSeparatorV2, - ArrayMetadataV2, - ArrayMetadataV2Partial, - ArrayOrderV2, - DataTypeMetadataV2, - ZArrayMetadata, -) -from zarr_metadata.v2.attributes import ZAttrsMetadata -from zarr_metadata.v2.codec import CodecMetadataV2 -from zarr_metadata.v2.consolidated import ConsolidatedMetadataV2 -from zarr_metadata.v2.group import GroupMetadataV2, GroupMetadataV2Partial, ZGroupMetadata -from zarr_metadata.v3._common import MetadataV3 -from zarr_metadata.v3.array import ArrayMetadataV3, ArrayMetadataV3Partial, ExtensionFieldV3 + ZarrV2ArrayDimensionSeparator, + ZarrV2ArrayMetadataJSON, + ZarrV2ArrayMetadataJSONPartial, + ZarrV2ArrayOrder, + ZarrV2DataTypeMetadata, + ZarrV2ZArrayJSON, +) +from zarr_metadata.v2.attributes import ZarrV2ZAttrsJSON +from zarr_metadata.v2.codec import ZarrV2CodecMetadata +from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON +from zarr_metadata.v2.group import ( + ZarrV2GroupMetadataJSON, + ZarrV2GroupMetadataJSONPartial, + ZarrV2ZGroupJSON, +) +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +from zarr_metadata.v3.array import ( + ZarrV3ArrayMetadataJSON, + ZarrV3ArrayMetadataJSONPartial, + ZarrV3ExtensionField, +) from zarr_metadata.v3.chunk_grid.rectilinear import ( RECTILINEAR_CHUNK_GRID_NAME, RectilinearChunkGridMetadata, @@ -86,7 +112,7 @@ TransposeCodecName, ) from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME, ZstdCodecMetadata, ZstdCodecName -from zarr_metadata.v3.consolidated import ConsolidatedMetadataV3 +from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON from zarr_metadata.v3.data_type.bool import ( BOOL_DATA_TYPE_NAME, BoolDataTypeName, @@ -185,7 +211,7 @@ Uint64DataTypeName, Uint64FillValue, ) -from zarr_metadata.v3.group import GroupMetadataV3, GroupMetadataV3Partial +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON, ZarrV3GroupMetadataJSONPartial __version__ = version("zarr-metadata") @@ -231,15 +257,10 @@ "UINT16_DATA_TYPE_NAME", "UINT32_DATA_TYPE_NAME", "UINT64_DATA_TYPE_NAME", + "UNSET", "V2_CHUNK_KEY_ENCODING_NAME", "V2_CHUNK_KEY_ENCODING_SEPARATOR", "ZSTD_CODEC_NAME", - "ArrayDimensionSeparatorV2", - "ArrayMetadataV2", - "ArrayMetadataV2Partial", - "ArrayMetadataV3", - "ArrayMetadataV3Partial", - "ArrayOrderV2", "BloscCName", "BloscCodecMetadata", "BloscCodecName", @@ -254,31 +275,22 @@ "CastRoundingMode", "CastValueCodecMetadata", "CastValueCodecName", - "CodecMetadataV2", "Complex64DataTypeName", "Complex64FillValue", "Complex128DataTypeName", "Complex128FillValue", - "ConsolidatedMetadataV2", - "ConsolidatedMetadataV3", "Crc32cCodecMetadata", "Crc32cCodecName", - "DataTypeMetadataV2", "DefaultChunkKeyEncodingMetadata", "DefaultChunkKeyEncodingName", "DefaultChunkKeyEncodingSeparator", "Endianness", - "ExtensionFieldV3", "Float16DataTypeName", "Float16FillValue", "Float32DataTypeName", "Float32FillValue", "Float64DataTypeName", "Float64FillValue", - "GroupMetadataV2", - "GroupMetadataV2Partial", - "GroupMetadataV3", - "GroupMetadataV3Partial", "GzipCodecMetadata", "GzipCodecName", "Int8DataTypeName", @@ -290,13 +302,13 @@ "Int64DataTypeName", "Int64FillValue", "JSONValue", - "MetadataV3", - "NamedConfigV3", + "MetadataValidationError", "NumpyDatetime64DataTypeName", "NumpyDatetime64FillValue", "NumpyTimeUnit", "NumpyTimedelta64DataTypeName", "NumpyTimedelta64FillValue", + "ProblemKind", "RawBytesDataTypeName", "RawBytesFillValue", "RectilinearChunkGridMetadata", @@ -325,9 +337,39 @@ "V2ChunkKeyEncodingMetadata", "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingSeparator", - "ZArrayMetadata", - "ZAttrsMetadata", - "ZGroupMetadata", + "ValidationProblem", + "ZarrV2ArrayDimensionSeparator", + "ZarrV2ArrayMetadata", + "ZarrV2ArrayMetadataJSON", + "ZarrV2ArrayMetadataJSONPartial", + "ZarrV2ArrayMetadataPartial", + "ZarrV2ArrayOrder", + "ZarrV2CodecMetadata", + "ZarrV2ConsolidatedMetadata", + "ZarrV2ConsolidatedMetadataJSON", + "ZarrV2DataTypeMetadata", + "ZarrV2GroupMetadata", + "ZarrV2GroupMetadataJSON", + "ZarrV2GroupMetadataJSONPartial", + "ZarrV2GroupMetadataPartial", + "ZarrV2ZArrayJSON", + "ZarrV2ZAttrsJSON", + "ZarrV2ZGroupJSON", + "ZarrV3ArrayMetadata", + "ZarrV3ArrayMetadataJSON", + "ZarrV3ArrayMetadataJSONPartial", + "ZarrV3ArrayMetadataPartial", + "ZarrV3ConsolidatedMetadata", + "ZarrV3ConsolidatedMetadataJSON", + "ZarrV3ExtensionField", + "ZarrV3GroupMetadata", + "ZarrV3GroupMetadataJSON", + "ZarrV3GroupMetadataJSONPartial", + "ZarrV3GroupMetadataPartial", + "ZarrV3MetadataField", + "ZarrV3MetadataFieldJSON", + "ZarrV3NamedConfig", + "ZarrV3NamedConfigJSON", "ZstdCodecMetadata", "ZstdCodecName", "__version__", diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index 598a12e80c..f3259f7b73 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -13,7 +13,14 @@ JSONValue = TypeAliasType( "JSONValue", - "int | float | bool | None | str | list[JSONValue] | tuple[JSONValue, ...] | Mapping[str, JSONValue]", # type: ignore[reportInvalidTypeForm] + int + | float + | bool + | str + | list["JSONValue"] + | tuple["JSONValue", ...] + | Mapping[str, "JSONValue"] + | None, ) """A recursive type alias for JSON-encodable values. @@ -24,13 +31,14 @@ """ -class NamedConfigV3(TypedDict): +class ZarrV3NamedConfigJSON(TypedDict): """ Externally-tagged union member for a metadata field. - The `configuration` mapping holds arbitrary JSON-encodable values; - it is typed as `Mapping[str, JSONValue]`. + The optional `configuration` mapping holds arbitrary JSON-encodable + values. `must_understand` is implicitly true when absent. """ name: str configuration: NotRequired[Mapping[str, JSONValue]] + must_understand: NotRequired[bool] diff --git a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py new file mode 100644 index 0000000000..e9792d6931 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py @@ -0,0 +1,114 @@ +"""Private input types used only to generate accurate Pydantic JSON schemas.""" + +from __future__ import annotations + +from collections.abc import Mapping # noqa: TC003 # resolved by Pydantic at runtime +from typing import Annotated, Literal, NotRequired + +from pydantic import Field +from typing_extensions import TypedDict + +from zarr_metadata._common import JSONValue +from zarr_metadata.v2.array import ( # noqa: TC001 # resolved by Pydantic at runtime + ZarrV2DataTypeMetadata, +) +from zarr_metadata.v2.codec import ( # resolved by Pydantic at runtime + ZarrV2CodecMetadata, +) + +NonNegativeInt = Annotated[int, Field(ge=0)] + + +class ZarrV3NamedConfigJSON(TypedDict, closed=True): + """Closed v3 named configuration accepted at optional extension points.""" + + name: str + configuration: NotRequired[Mapping[str, JSONValue]] + must_understand: NotRequired[bool] + + +class ZarrV3MandatoryNamedConfigJSON(TypedDict, closed=True): + """Closed named configuration accepted where understanding is mandatory.""" + + name: str + configuration: NotRequired[Mapping[str, JSONValue]] + must_understand: NotRequired[Literal[True]] + + +ZarrV3MetadataFieldJSON = str | ZarrV3NamedConfigJSON +ZarrV3MandatoryMetadataFieldJSON = str | ZarrV3MandatoryNamedConfigJSON +ZarrV3CodecPipelineJSON = Annotated[tuple[ZarrV3MetadataFieldJSON, ...], Field(min_length=1)] +ZarrV2FilterPipelineJSON = Annotated[tuple[ZarrV2CodecMetadata, ...], Field(min_length=1)] + + +class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=JSONValue): + """Schema input for a v3 array document, including arbitrary extensions.""" + + zarr_format: Literal[3] + node_type: Literal["array"] + data_type: ZarrV3MandatoryMetadataFieldJSON + shape: tuple[NonNegativeInt, ...] + chunk_grid: ZarrV3MandatoryMetadataFieldJSON + chunk_key_encoding: ZarrV3MandatoryMetadataFieldJSON + fill_value: JSONValue + codecs: ZarrV3CodecPipelineJSON + attributes: NotRequired[Mapping[str, JSONValue]] + storage_transformers: NotRequired[tuple[ZarrV3MetadataFieldJSON, ...]] + dimension_names: NotRequired[tuple[str | None, ...]] + + +class ZarrV3ConsolidatedMetadataJSON(TypedDict, closed=True): + """Schema input for the closed inline consolidated-metadata envelope.""" + + kind: Literal["inline"] + must_understand: Literal[False] + metadata: Mapping[str, ZarrV3ArrayMetadataJSON | ZarrV3GroupMetadataJSON] + + +class ZarrV3GroupMetadataJSON(TypedDict, extra_items=JSONValue): + """Schema input for a v3 group document, including arbitrary extensions.""" + + zarr_format: Literal[3] + node_type: Literal["group"] + attributes: NotRequired[Mapping[str, JSONValue]] + consolidated_metadata: NotRequired[ZarrV3ConsolidatedMetadataJSON | None] + + +class ZarrV2ArrayMetadataJSON(TypedDict, closed=True): + """Schema input for the closed, merged v2 array representation.""" + + zarr_format: Literal[2] + shape: tuple[NonNegativeInt, ...] + chunks: tuple[NonNegativeInt, ...] + dtype: ZarrV2DataTypeMetadata + compressor: ZarrV2CodecMetadata | None + fill_value: JSONValue + order: Literal["C", "F"] + filters: ZarrV2FilterPipelineJSON | None + dimension_separator: NotRequired[Literal[".", "/"]] + attributes: NotRequired[Mapping[str, JSONValue]] + + +class ZarrV2GroupMetadataJSON(TypedDict, closed=True): + """Schema input for the closed, merged v2 group representation.""" + + zarr_format: Literal[2] + attributes: NotRequired[Mapping[str, JSONValue]] + + +class ZarrV2ConsolidatedMetadataJSON(TypedDict, closed=True): + """Schema input matching the v2 consolidated model's structural parser.""" + + zarr_consolidated_format: Literal[1] + metadata: Mapping[str, JSONValue] + + +__all__ = [ + "ZarrV2ArrayMetadataJSON", + "ZarrV2ConsolidatedMetadataJSON", + "ZarrV2GroupMetadataJSON", + "ZarrV3ArrayMetadataJSON", + "ZarrV3ConsolidatedMetadataJSON", + "ZarrV3GroupMetadataJSON", + "ZarrV3MetadataFieldJSON", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py new file mode 100644 index 0000000000..e726c54d3e --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -0,0 +1,132 @@ +"""In-memory models for Zarr metadata documents. + +Models are frozen dataclasses that hold a canonical, semantically lossless +representation of the JSON documents; they never interpret extension points +(codecs, chunk grids, data types). Validators check JSON structure, not domain validity. +Each document concept gets a `validate_*` function returning every problem +found (a `list[ValidationProblem]`, each with a machine-readable `kind`), an +`is_*` type guard, and a `parse_*` function that narrows or raises +`MetadataValidationError`. Model `from_json` / `from_key_value` constructors +raise `MetadataValidationError` for every ingestion failure, including +missing store keys and undecodable bytes. +""" + +from zarr_metadata.model._array import ( + ARRAY_METADATA_STORE_KEY_V2, + ARRAY_METADATA_STORE_KEY_V3, + ATTRIBUTES_STORE_KEY_V2, + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadataPartial, + ZarrV2ArrayMetadataStoreKey, + ZarrV2AttributesStoreKey, + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadataPartial, + ZarrV3ArrayMetadataStoreKey, + ZarrV3MetadataField, + ZarrV3NamedConfig, +) +from zarr_metadata.model._group import ( + CONSOLIDATED_METADATA_KEY_V3, + CONSOLIDATED_METADATA_STORE_KEY_V2, + GROUP_METADATA_STORE_KEY_V2, + GROUP_METADATA_STORE_KEY_V3, + ZarrV2ConsolidatedMetadata, + ZarrV2ConsolidatedMetadataStoreKey, + ZarrV2GroupMetadata, + ZarrV2GroupMetadataPartial, + ZarrV2GroupMetadataStoreKey, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, + ZarrV3GroupMetadataPartial, + ZarrV3GroupMetadataStoreKey, +) +from zarr_metadata.model._sentinel import UNSET +from zarr_metadata.model._validation import ( + ARRAY_METADATA_OPTIONAL_KEYS_V3, + ARRAY_METADATA_REQUIRED_KEYS_V2, + ARRAY_METADATA_REQUIRED_KEYS_V3, + ARRAY_METADATA_STANDARD_KEYS_V3, + GROUP_METADATA_OPTIONAL_KEYS_V3, + GROUP_METADATA_REQUIRED_KEYS_V2, + GROUP_METADATA_REQUIRED_KEYS_V3, + GROUP_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ProblemKind, + ValidationProblem, + is_array_metadata_v2, + is_array_metadata_v3, + is_group_metadata_v2, + is_group_metadata_v3, + is_json, + is_metadata_field_v3, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_group_metadata_v2, + parse_group_metadata_v3, + parse_json, + parse_metadata_field_v3, + validate_array_metadata_v2, + validate_array_metadata_v3, + validate_group_metadata_v2, + validate_group_metadata_v3, + validate_json, + validate_metadata_field_v3, +) + +__all__ = [ + "ARRAY_METADATA_OPTIONAL_KEYS_V3", + "ARRAY_METADATA_REQUIRED_KEYS_V2", + "ARRAY_METADATA_REQUIRED_KEYS_V3", + "ARRAY_METADATA_STANDARD_KEYS_V3", + "ARRAY_METADATA_STORE_KEY_V2", + "ARRAY_METADATA_STORE_KEY_V3", + "ATTRIBUTES_STORE_KEY_V2", + "CONSOLIDATED_METADATA_KEY_V3", + "CONSOLIDATED_METADATA_STORE_KEY_V2", + "GROUP_METADATA_OPTIONAL_KEYS_V3", + "GROUP_METADATA_REQUIRED_KEYS_V2", + "GROUP_METADATA_REQUIRED_KEYS_V3", + "GROUP_METADATA_STANDARD_KEYS_V3", + "GROUP_METADATA_STORE_KEY_V2", + "GROUP_METADATA_STORE_KEY_V3", + "UNSET", + "MetadataValidationError", + "ProblemKind", + "ValidationProblem", + "ZarrV2ArrayMetadata", + "ZarrV2ArrayMetadataPartial", + "ZarrV2ArrayMetadataStoreKey", + "ZarrV2AttributesStoreKey", + "ZarrV2ConsolidatedMetadata", + "ZarrV2ConsolidatedMetadataStoreKey", + "ZarrV2GroupMetadata", + "ZarrV2GroupMetadataPartial", + "ZarrV2GroupMetadataStoreKey", + "ZarrV3ArrayMetadata", + "ZarrV3ArrayMetadataPartial", + "ZarrV3ArrayMetadataStoreKey", + "ZarrV3ConsolidatedMetadata", + "ZarrV3GroupMetadata", + "ZarrV3GroupMetadataPartial", + "ZarrV3GroupMetadataStoreKey", + "ZarrV3MetadataField", + "ZarrV3NamedConfig", + "is_array_metadata_v2", + "is_array_metadata_v3", + "is_group_metadata_v2", + "is_group_metadata_v3", + "is_json", + "is_metadata_field_v3", + "parse_array_metadata_v2", + "parse_array_metadata_v3", + "parse_group_metadata_v2", + "parse_group_metadata_v3", + "parse_json", + "parse_metadata_field_v3", + "validate_array_metadata_v2", + "validate_array_metadata_v3", + "validate_group_metadata_v2", + "validate_group_metadata_v3", + "validate_json", + "validate_metadata_field_v3", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py new file mode 100644 index 0000000000..c4c967f891 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -0,0 +1,498 @@ +"""In-memory models for Zarr array metadata documents.""" + +from __future__ import annotations + +import copy +import dataclasses +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Final, Literal, TypeAlias, cast + +from typing_extensions import TypedDict, Unpack + +from zarr_metadata.model._sentinel import UNSET +from zarr_metadata.model._validation import ( + ARRAY_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ValidationProblem, + arrays_to_tuples, + dump_store_json, + load_store_json, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_metadata_field_v3, +) + +if TYPE_CHECKING: + from zarr_metadata._common import JSONValue, ZarrV3NamedConfigJSON + from zarr_metadata.v2.array import ( + ZarrV2ArrayDimensionSeparator, + ZarrV2ArrayMetadataJSON, + ZarrV2ArrayOrder, + ZarrV2DataTypeMetadata, + ) + from zarr_metadata.v2.codec import ZarrV2CodecMetadata + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON, ZarrV3ExtensionField + +ZarrV3ArrayMetadataStoreKey = Literal["zarr.json"] +ARRAY_METADATA_STORE_KEY_V3: Final[ZarrV3ArrayMetadataStoreKey] = "zarr.json" + +ZarrV2ArrayMetadataStoreKey = Literal[".zarray"] +ARRAY_METADATA_STORE_KEY_V2: Final[ZarrV2ArrayMetadataStoreKey] = ".zarray" + +ZarrV2AttributesStoreKey = Literal[".zattrs"] +ATTRIBUTES_STORE_KEY_V2: Final[ZarrV2AttributesStoreKey] = ".zattrs" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ZarrV3NamedConfig: + """A normalized v3 metadata field with its reader obligation. + + Bare names and missing configurations normalize to an empty configuration. + Bare names and missing `must_understand` members normalize to the spec's + implicit `True` value. + """ + + name: str + configuration: dict[str, JSONValue] + must_understand: bool = True + + def to_json(self) -> ZarrV3MetadataFieldJSON: + if not self.configuration and self.must_understand: + return self.name + out: ZarrV3NamedConfigJSON = {"name": self.name} + if self.configuration: + # to_json output shares no mutable state with the model. + out["configuration"] = copy.deepcopy(self.configuration) + if not self.must_understand: + out["must_understand"] = False + return out + + @classmethod + def from_json(cls, data: object) -> ZarrV3NamedConfig: + field = parse_metadata_field_v3(data) + if isinstance(field, str): + return cls(name=field, configuration={}, must_understand=True) + # Sound cast: parse_metadata_field_v3 checked the configuration is a + # string-keyed mapping of JSON values; arrays_to_tuples only converts + # lists to tuples within that shape. + configuration = cast( + "dict[str, JSONValue]", arrays_to_tuples(dict(field.get("configuration", {}))) + ) + return cls( + name=field["name"], + configuration=configuration, + must_understand=field.get("must_understand", True), + ) + + +ZarrV3MetadataField: TypeAlias = ZarrV3NamedConfig +"""The in-memory model of one field of a v3 metadata document. + +This is the role-named alias for annotation positions: model fields and +consumer signatures should say `ZarrV3MetadataField` (the logical meaning) +rather than `ZarrV3NamedConfig` (the serialized form the field currently +takes). Today every metadata field normalizes to a named configuration plus +its reader obligation, so the alias is exactly `ZarrV3NamedConfig`; if a future +spec revision adds a field form that cannot be normalized to those values, +this alias widens to a union and annotation sites do not change. Mirrors the +raw-layer split between `ZarrV3NamedConfigJSON` (shape) and +`ZarrV3MetadataFieldJSON` (field union). +""" + + +def must_understand_subset( + extra_fields: Mapping[str, ZarrV3ExtensionField], +) -> dict[str, ZarrV3ExtensionField]: + """The subset of `extra_fields` the reader is obligated to understand. + + Per the v3 spec, an extension field is implicitly `must_understand: True` + unless it explicitly says otherwise, and an implementation MUST fail to + open a group or array carrying fields it does not recognize that are not + explicitly `must_understand: false`. A non-mapping field value cannot + carry the explicit waiver, so it always requires understanding (the + runtime isinstance check defends against values looser than the declared + `ZarrV3ExtensionField`). + """ + fields = cast("Mapping[str, object]", extra_fields) + return cast( + "dict[str, ZarrV3ExtensionField]", + { + name: value + for name, value in fields.items() + if not ( + isinstance(value, Mapping) + and cast("Mapping[str, object]", value).get("must_understand") is False + ) + }, + ) + + +class ZarrV3ArrayMetadataPartial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `ZarrV3ArrayMetadata`. + + Every key is optional and typed with the model's own (not serialized) + value types, so it describes valid keyword arguments to + `ZarrV3ArrayMetadata.update`. The `init=False` fields `zarr_format` and + `node_type` are intentionally excluded, since they cannot be passed to + `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_array.py::test_partial_keys_match_settable_model_fields`. + """ + + shape: tuple[int, ...] + fill_value: JSONValue + data_type: ZarrV3MetadataField + chunk_grid: ZarrV3MetadataField + codecs: tuple[ZarrV3MetadataField, ...] + chunk_key_encoding: ZarrV3MetadataField + dimension_names: tuple[str | None, ...] | UNSET + attributes: dict[str, JSONValue] + storage_transformers: tuple[ZarrV3MetadataField, ...] + extra_fields: dict[str, ZarrV3ExtensionField] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ZarrV3ArrayMetadata: + """In-memory model of a v3 array metadata document. + + A canonical, semantically lossless representation of the `zarr.json` + content for an array. Extension points (`data_type`, `chunk_grid`, + `chunk_key_encoding`, `codecs`, `storage_transformers`) are held as + `ZarrV3MetadataField` values (currently `ZarrV3NamedConfig` name, + configuration, and obligation records) and are never interpreted; + `fill_value` is held verbatim in its JSON form. Equivalent extension + spellings normalize to shorthand strings when configuration is empty and + understanding is required. + """ + + zarr_format: Literal[3] = field(default=3, init=False) + node_type: Literal["array"] = field(default="array", init=False) + shape: tuple[int, ...] + fill_value: JSONValue + data_type: ZarrV3MetadataField + chunk_grid: ZarrV3MetadataField + codecs: tuple[ZarrV3MetadataField, ...] + chunk_key_encoding: ZarrV3MetadataField + dimension_names: tuple[str | None, ...] | UNSET + attributes: dict[str, JSONValue] + storage_transformers: tuple[ZarrV3MetadataField, ...] + extra_fields: dict[str, ZarrV3ExtensionField] + + @classmethod + def create_default(cls, **overrides: Unpack[ZarrV3ArrayMetadataPartial]) -> ZarrV3ArrayMetadata: + """ + Create a default (empty) v3 array metadata model, with optional overrides. + + The default is a structurally-valid scalar `uint8` array — the array + analog of `list()` returning `[]`. Any field can be overridden by keyword + (the same fields accepted by `update`). Overriding `shape` without + `chunk_grid` derives a consistent default grid: one regular chunk + covering the array (`chunk_shape` equal to `shape`). + + The derivation is deliberately one-way. A user-supplied `chunk_grid` + is an extension point and is taken verbatim — deriving `shape` from + it would require interpreting the grid's configuration, which this + layer never does (and cannot do for unrecognized grid names). So + overriding `chunk_grid` without `shape` keeps the scalar default + `shape=()`, and consistency between the two is the caller's + responsibility. + """ + if "shape" in overrides and "chunk_grid" not in overrides: + overrides["chunk_grid"] = ZarrV3NamedConfig( + name="regular", configuration={"chunk_shape": tuple(overrides["shape"])} + ) + default = cls( + shape=(), + fill_value=0, + data_type=ZarrV3NamedConfig(name="uint8", configuration={}), + chunk_grid=ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": ()}), + codecs=(ZarrV3NamedConfig(name="bytes", configuration={}),), + chunk_key_encoding=ZarrV3NamedConfig(name="default", configuration={}), + dimension_names=UNSET, + attributes={}, + storage_transformers=(), + extra_fields={}, + ) + return default.update(**overrides) + + def update(self, **kwargs: Unpack[ZarrV3ArrayMetadataPartial]) -> ZarrV3ArrayMetadata: + """ + Return a new `ZarrV3ArrayMetadata` with the given fields updated. + + Only the constructor-settable fields listed in + `ZarrV3ArrayMetadataPartial` can be updated; any attempt to update + other fields (including the fixed `zarr_format` / `node_type`) is + rejected at the type level. Each given field fully replaces its + previous value, including `extra_fields`. + + This is useful for test fixtures that want to override a few fields of a + base template without having to re-specify the entire document. + + No re-validation is performed (`update` is `dataclasses.replace`), so + a repair or edit can produce an invalid document; validity is checked + on `from_json`, not on field replacement. + """ + return dataclasses.replace(self, **kwargs) + + def __post_init__(self) -> None: + overlap = set(self.extra_fields.keys()).intersection(ARRAY_METADATA_STANDARD_KEYS_V3) + if overlap: + raise MetadataValidationError( + [ + ValidationProblem( + ("extra_fields",), + "Extra fields cannot overlap with standard Zarr V3 array metadata fields", + "invalid_value", + ) + ] + ) + + def to_json(self) -> ZarrV3ArrayMetadataJSON: + # to_json output shares no mutable state with the model: every value + # that can hold a mutable container is deep-copied. + out: ZarrV3ArrayMetadataJSON = { + "zarr_format": self.zarr_format, + "node_type": self.node_type, + "shape": self.shape, + "fill_value": copy.deepcopy(self.fill_value), + "data_type": self.data_type.to_json(), + "chunk_grid": self.chunk_grid.to_json(), + "codecs": tuple(codec.to_json() for codec in self.codecs), + "chunk_key_encoding": self.chunk_key_encoding.to_json(), + } + if self.dimension_names is not UNSET: + out["dimension_names"] = self.dimension_names + if len(self.attributes) > 0: + out["attributes"] = copy.deepcopy(self.attributes) + if len(self.storage_transformers) > 0: + out["storage_transformers"] = tuple( + transformer.to_json() for transformer in self.storage_transformers + ) + # Extra fields are the TypedDict's `extra_items` (PEP 728). Assign them + # by key rather than `out.update(**...)`: type checkers understand the + # indexed-write path against `extra_items`, but not the `update(**...)` + # overload. + for key, value in self.extra_fields.items(): + out[key] = copy.deepcopy(value) + return out + + @classmethod + def from_json(cls, data: object) -> ZarrV3ArrayMetadata: + parsed = parse_array_metadata_v3(arrays_to_tuples(data)) + # Sound cast: the TypedDict types all non-standard keys as its + # `extra_items` (`ZarrV3ExtensionField`); the comprehension's inferred value + # type is the union over ALL keys because the key filter cannot narrow it. + extra_fields = cast( + "dict[str, ZarrV3ExtensionField]", + {k: v for k, v in parsed.items() if k not in ARRAY_METADATA_STANDARD_KEYS_V3}, + ) + return cls( + shape=parsed["shape"], + fill_value=parsed["fill_value"], + data_type=ZarrV3NamedConfig.from_json(parsed["data_type"]), + chunk_grid=ZarrV3NamedConfig.from_json(parsed["chunk_grid"]), + codecs=tuple(ZarrV3NamedConfig.from_json(c) for c in parsed["codecs"]), + chunk_key_encoding=ZarrV3NamedConfig.from_json(parsed["chunk_key_encoding"]), + dimension_names=parsed.get("dimension_names", UNSET), + attributes=dict(parsed.get("attributes", {})), + storage_transformers=tuple( + ZarrV3NamedConfig.from_json(t) for t in parsed.get("storage_transformers", ()) + ), + extra_fields=extra_fields, + ) + + @property + def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: + """Extra fields the reader is obligated to understand. + + Everything in `extra_fields` not explicitly waived with + `must_understand: false` (the spec's implicit-true rule). A compliant + reader MUST fail to open the array if this contains any field it does + not recognize; the model layer only partitions by obligation, since + recognition is reader-specific. + """ + return must_understand_subset(self.extra_fields) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3ArrayMetadata: + return cls.from_json(load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V3)) + + def to_key_value( + self, *, indent: int | str | None = None + ) -> Mapping[ZarrV3ArrayMetadataStoreKey, bytes]: + return {ARRAY_METADATA_STORE_KEY_V3: dump_store_json(self.to_json(), indent=indent)} + + +class ZarrV2ArrayMetadataPartial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `ZarrV2ArrayMetadata`. + + Every key is optional and typed with the model's own value types, so it + describes valid keyword arguments to `ZarrV2ArrayMetadata.update` and + `create_default`. The `init=False` field `zarr_format` is intentionally + excluded, since it cannot be passed to `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_array.py::test_v2_partial_keys_match_settable_model_fields`. + """ + + shape: tuple[int, ...] + dtype: ZarrV2DataTypeMetadata + chunks: tuple[int, ...] + fill_value: JSONValue + order: ZarrV2ArrayOrder + compressor: ZarrV2CodecMetadata | None + filters: tuple[ZarrV2CodecMetadata, ...] | None + dimension_separator: ZarrV2ArrayDimensionSeparator + attributes: dict[str, JSONValue] | UNSET + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ZarrV2ArrayMetadata: + """In-memory model of a v2 array metadata document. + + A canonical, lossless representation of the `.zarray` content plus the + sibling `.zattrs` attributes. `dtype`, `compressor`, and `filters` are + held in their raw JSON forms and are never interpreted; `fill_value` is + held verbatim in its JSON form. `attributes` is `UNSET` when no + `.zattrs` file (or merged `attributes` key) exists — distinct from an + explicit empty `.zattrs`, which is `{}` and round-trips as a file. One + spelling normalization: a `.zarray` that omits `dimension_separator` + means `"."` by the v2 convention, and the model holds and re-emits that + value explicitly. + """ + + zarr_format: Literal[2] = field(default=2, init=False) + shape: tuple[int, ...] + dtype: ZarrV2DataTypeMetadata + chunks: tuple[int, ...] + fill_value: JSONValue + order: ZarrV2ArrayOrder + compressor: ZarrV2CodecMetadata | None + filters: tuple[ZarrV2CodecMetadata, ...] | None + # "." is the v2 convention's default for an ABSENT dimension_separator key; + # from_json normalizes absence to it (a semantics-preserving spelling + # normalization, like the v3 bare-string metadata-field form). The value + # is never None: the document grammar has no null spelling for this field. + dimension_separator: ZarrV2ArrayDimensionSeparator = field(default=".") + attributes: dict[str, JSONValue] | UNSET + + def update(self, **kwargs: Unpack[ZarrV2ArrayMetadataPartial]) -> ZarrV2ArrayMetadata: + """ + Return a new `ZarrV2ArrayMetadata` with the given fields updated. + + Only the constructor-settable fields listed in + `ZarrV2ArrayMetadataPartial` can be updated; the fixed `zarr_format` is + rejected at the type level. Each given field fully replaces its previous + value. + """ + return dataclasses.replace(self, **kwargs) + + @classmethod + def create_default(cls, **overrides: Unpack[ZarrV2ArrayMetadataPartial]) -> ZarrV2ArrayMetadata: + """ + Create a default (empty) v2 array metadata model, with optional overrides. + + The default is a structurally-valid scalar `uint8` (`"|u1"`) array — the + array analog of `list()` returning `[]`. Any field can be overridden by + keyword (the same fields accepted by `update`). Overriding `shape` + without `chunks` derives `chunks` equal to `shape` (one chunk covering + the array). + + The derivation is deliberately one-way, matching the v3 model: + overriding `chunks` without `shape` keeps the scalar default + `shape=()`, and consistency between the two is the caller's + responsibility. + """ + if "shape" in overrides and "chunks" not in overrides: + overrides["chunks"] = tuple(overrides["shape"]) + default = cls( + shape=(), + dtype="|u1", + chunks=(), + fill_value=0, + order="C", + compressor=None, + filters=None, + attributes=UNSET, + ) + return default.update(**overrides) + + def to_json(self) -> ZarrV2ArrayMetadataJSON: + """Return the merged in-memory document form. + + `attributes` is included when set (even empty). This is not the + on-disk `.zarray` content: a conforming `.zarray` must exclude + `attributes` (they live in the sibling `.zattrs` file). Use + `to_key_value` to produce the spec-conforming split for storage. + """ + # to_json output shares no mutable state with the model: every value + # that can hold a mutable container is deep-copied. + out: ZarrV2ArrayMetadataJSON = { + "zarr_format": self.zarr_format, + "shape": self.shape, + "dtype": self.dtype, + "order": self.order, + "chunks": self.chunks, + "fill_value": copy.deepcopy(self.fill_value), + "dimension_separator": self.dimension_separator, + "compressor": copy.deepcopy(self.compressor), + "filters": copy.deepcopy(self.filters), + } + if self.attributes is not UNSET: + out["attributes"] = copy.deepcopy(self.attributes) + return out + + @classmethod + def from_json(cls, data: object) -> ZarrV2ArrayMetadata: + parsed = parse_array_metadata_v2(arrays_to_tuples(data)) + return cls( + shape=parsed["shape"], + dtype=parsed["dtype"], + chunks=parsed["chunks"], + fill_value=parsed["fill_value"], + order=parsed["order"], + compressor=parsed["compressor"], + filters=parsed["filters"], + dimension_separator=parsed.get("dimension_separator", "."), + attributes=(dict(parsed["attributes"]) if "attributes" in parsed else UNSET), + ) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ArrayMetadata: + zarray_raw = cast("object", load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V2)) + if not isinstance(zarray_raw, Mapping): + return cls.from_json(zarray_raw) + zarray = cast("Mapping[str, object]", zarray_raw) + if "attributes" in zarray: + raise MetadataValidationError( + [ + ValidationProblem( + ("attributes",), + "unexpected document member", + "invalid_value", + ) + ] + ) + if ATTRIBUTES_STORE_KEY_V2 in mapping: + zattrs = cast("object", load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2)) + return cls.from_json({**zarray, "attributes": zattrs}) + return cls.from_json(zarray) + + def to_key_value( + self, *, indent: int | str | None = None + ) -> Mapping[ZarrV2ArrayMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]: + # Attributes live only in the sibling `.zattrs` file; the `.zarray` + # document must exclude them. The `.zattrs` key is present exactly + # when attributes are set (even empty) — UNSET emits no file. + zarray = {k: v for k, v in self.to_json().items() if k != "attributes"} + out: dict[ZarrV2ArrayMetadataStoreKey | ZarrV2AttributesStoreKey, bytes] = { + ARRAY_METADATA_STORE_KEY_V2: dump_store_json(zarray, indent=indent) + } + if self.attributes is not UNSET: + out[ATTRIBUTES_STORE_KEY_V2] = dump_store_json(self.attributes, indent=indent) + return out diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py new file mode 100644 index 0000000000..d576833c26 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -0,0 +1,442 @@ +"""In-memory models for Zarr group and consolidated metadata documents.""" + +from __future__ import annotations + +import copy +import dataclasses +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Final, Literal, cast + +from typing_extensions import TypedDict, Unpack + +from zarr_metadata.model._array import ( + ATTRIBUTES_STORE_KEY_V2, + ZarrV3ArrayMetadata, + must_understand_subset, +) +from zarr_metadata.model._sentinel import UNSET +from zarr_metadata.model._validation import ( + GROUP_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ValidationProblem, + arrays_to_tuples, + dump_store_json, + load_store_json, + parse_group_metadata_v2, + parse_group_metadata_v3, + validate_consolidated_metadata_v3, + validate_json, +) + +if TYPE_CHECKING: + from zarr_metadata._common import JSONValue + from zarr_metadata.model._array import ZarrV2AttributesStoreKey + from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON + from zarr_metadata.v3.array import ZarrV3ExtensionField + from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON + from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON + +ZarrV3GroupMetadataStoreKey = Literal["zarr.json"] +GROUP_METADATA_STORE_KEY_V3: Final[ZarrV3GroupMetadataStoreKey] = "zarr.json" + +ZarrV2GroupMetadataStoreKey = Literal[".zgroup"] +GROUP_METADATA_STORE_KEY_V2: Final[ZarrV2GroupMetadataStoreKey] = ".zgroup" + +ZarrV2ConsolidatedMetadataStoreKey = Literal[".zmetadata"] +CONSOLIDATED_METADATA_STORE_KEY_V2: Final[ZarrV2ConsolidatedMetadataStoreKey] = ".zmetadata" + +# The key under which consolidated metadata is embedded in a v3 group document. +# This is a reference-implementation convention (not a spec artifact), stored +# as an extension field on the group's `zarr.json`. +CONSOLIDATED_METADATA_KEY_V3: Final = "consolidated_metadata" + + +class ZarrV3GroupMetadataPartial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `ZarrV3GroupMetadata`. + + Every key is optional and typed with the model's own value types, so it + describes valid keyword arguments to `ZarrV3GroupMetadata.update` and + `create_default`. The `init=False` fields `zarr_format` and `node_type` + are intentionally excluded, since they cannot be passed to + `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_group.py::test_group_partial_keys_match_settable_model_fields`. + """ + + attributes: dict[str, JSONValue] + consolidated_metadata: ZarrV3ConsolidatedMetadata | UNSET + extra_fields: dict[str, ZarrV3ExtensionField] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ZarrV3GroupMetadata: + """In-memory model of a v3 group metadata document. + + A canonical, semantically lossless representation of the `zarr.json` + content for a group. The `consolidated_metadata` reference-implementation + convention is modeled as a typed field holding thin child models; every + other unknown top-level key lands in `extra_fields` verbatim. + """ + + zarr_format: Literal[3] = field(default=3, init=False) + node_type: Literal["group"] = field(default="group", init=False) + attributes: dict[str, JSONValue] + consolidated_metadata: ZarrV3ConsolidatedMetadata | UNSET + extra_fields: dict[str, ZarrV3ExtensionField] + + def __post_init__(self) -> None: + reserved = GROUP_METADATA_STANDARD_KEYS_V3 | {CONSOLIDATED_METADATA_KEY_V3} + if set(self.extra_fields.keys()).intersection(reserved): + raise MetadataValidationError( + [ + ValidationProblem( + ("extra_fields",), + "Extra fields cannot overlap with standard Zarr V3 group metadata fields", + "invalid_value", + ) + ] + ) + + @classmethod + def create_default(cls, **overrides: Unpack[ZarrV3GroupMetadataPartial]) -> ZarrV3GroupMetadata: + """ + Create a default (empty) v3 group metadata model, with optional overrides. + + The default is a structurally-valid group with no attributes — the group + analog of `list()` returning `[]`. Any field can be overridden by keyword + (the same fields accepted by `update`). + """ + default = cls(attributes={}, consolidated_metadata=UNSET, extra_fields={}) + return default.update(**overrides) + + def update(self, **kwargs: Unpack[ZarrV3GroupMetadataPartial]) -> ZarrV3GroupMetadata: + """ + Return a new `ZarrV3GroupMetadata` with the given fields updated. + + Only the constructor-settable fields listed in + `ZarrV3GroupMetadataPartial` can be updated; the fixed `zarr_format` / + `node_type` are rejected at the type level. Each given field fully + replaces its previous value, including `extra_fields`. + """ + return dataclasses.replace(self, **kwargs) + + def to_json(self) -> ZarrV3GroupMetadataJSON: + # to_json output shares no mutable state with the model: every value + # that can hold a mutable container is deep-copied. + out: ZarrV3GroupMetadataJSON = { + "zarr_format": self.zarr_format, + "node_type": self.node_type, + } + if len(self.attributes) > 0: + out["attributes"] = copy.deepcopy(self.attributes) + if self.consolidated_metadata is not UNSET: + # Consolidated metadata is a known non-core top-level JSON field. + out[CONSOLIDATED_METADATA_KEY_V3] = cast( + "ZarrV3ExtensionField", self.consolidated_metadata.to_json() + ) + for key, value in self.extra_fields.items(): + out[key] = copy.deepcopy(value) + return out + + @classmethod + def from_json(cls, data: object) -> ZarrV3GroupMetadata: + parsed = parse_group_metadata_v3(arrays_to_tuples(data)) + # Cast for narrowing across standard and arbitrary extra TypedDict items. + consolidated_raw = cast("object", parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET)) + consolidated: ZarrV3ConsolidatedMetadata | UNSET + if consolidated_raw is UNSET or consolidated_raw is None: + # consolidated_metadata: null was written by a historical + # zarr-python bug; it gets no model representation. It is read as + # absence and never written back — repaired, not preserved. + consolidated = UNSET + else: + consolidated = ZarrV3ConsolidatedMetadata.from_json(consolidated_raw) + # Sound cast: the TypedDict types all non-standard keys as its + # `extra_items` (`ZarrV3ExtensionField`); the comprehension's inferred value + # type is the union over ALL keys because the key filter cannot narrow it. + extra_fields = cast( + "dict[str, ZarrV3ExtensionField]", + { + k: v + for k, v in parsed.items() + if k not in GROUP_METADATA_STANDARD_KEYS_V3 and k != CONSOLIDATED_METADATA_KEY_V3 + }, + ) + return cls( + attributes=dict(parsed.get("attributes", {})), + consolidated_metadata=consolidated, + extra_fields=extra_fields, + ) + + @property + def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: + """Extra fields the reader is obligated to understand. + + Everything in `extra_fields` not explicitly waived with + `must_understand: false` (the spec's implicit-true rule). A compliant + reader MUST fail to open the group if this contains any field it does + not recognize; the model layer only partitions by obligation, since + recognition is reader-specific. + """ + return must_understand_subset(self.extra_fields) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3GroupMetadata: + return cls.from_json(load_store_json(mapping, GROUP_METADATA_STORE_KEY_V3)) + + def to_key_value( + self, *, indent: int | str | None = None + ) -> Mapping[ZarrV3GroupMetadataStoreKey, bytes]: + return {GROUP_METADATA_STORE_KEY_V3: dump_store_json(self.to_json(), indent=indent)} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ZarrV3ConsolidatedMetadata: + """In-memory model of v3 inline consolidated metadata. + + Models the reference-implementation convention where consolidated metadata + is embedded as an extension field on a group's `zarr.json`. Each entry in + `metadata` is a complete child document, held as a thin array or group + model. `must_understand` is typed permissively as `bool` to mirror the + document shape, but only `False` is valid; this is enforced at runtime. + """ + + kind: Literal["inline"] = field(default="inline", init=False) + must_understand: bool = False + metadata: dict[str, ZarrV3ArrayMetadata | ZarrV3GroupMetadata] + + def __post_init__(self) -> None: + if self.must_understand is not False: + raise MetadataValidationError( + [ + ValidationProblem( + ("must_understand",), + f"Invalid value for 'must_understand'. Expected False. " + f"Got {self.must_understand!r}.", + "invalid_value", + ) + ] + ) + + def to_json(self) -> ZarrV3ConsolidatedMetadataJSON: + # `must_understand` is emitted as the literal False: the field is typed + # permissively as `bool`, but `__post_init__` guarantees the value. + return { + "kind": self.kind, + "must_understand": False, + "metadata": {key: node.to_json() for key, node in self.metadata.items()}, + } + + @classmethod + def from_json(cls, data: object) -> ZarrV3ConsolidatedMetadata: + normalized = arrays_to_tuples(data) + problems = validate_consolidated_metadata_v3(normalized) + if problems: + raise MetadataValidationError(problems) + env = cast("Mapping[str, object]", normalized) + entries: dict[str, ZarrV3ArrayMetadata | ZarrV3GroupMetadata] = {} + for key, entry in cast("Mapping[str, object]", env["metadata"]).items(): + node_type = cast("Mapping[str, object]", entry).get("node_type") + if node_type == "array": + entries[key] = ZarrV3ArrayMetadata.from_json(entry) + else: + entries[key] = ZarrV3GroupMetadata.from_json(entry) + return cls(metadata=entries) + + +class ZarrV2GroupMetadataPartial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `ZarrV2GroupMetadata`. + + Every key is optional and typed with the model's own value types, so it + describes valid keyword arguments to `ZarrV2GroupMetadata.update` and + `create_default`. The `init=False` field `zarr_format` is intentionally + excluded, since it cannot be passed to `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_group.py::test_group_partial_keys_match_settable_model_fields`. + """ + + attributes: dict[str, JSONValue] | UNSET + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ZarrV2GroupMetadata: + """In-memory model of a v2 group metadata document. + + A canonical, lossless representation of the `.zgroup` content plus the + sibling `.zattrs` attributes, folded into a single in-memory value + (mirroring the merged `ZarrV2GroupMetadataJSON` document form). `attributes` is + `UNSET` when no `.zattrs` file (or merged `attributes` key) exists — + distinct from an explicit empty `.zattrs`, which is `{}` and round-trips + as a file. + """ + + zarr_format: Literal[2] = field(default=2, init=False) + attributes: dict[str, JSONValue] | UNSET + + @classmethod + def create_default(cls, **overrides: Unpack[ZarrV2GroupMetadataPartial]) -> ZarrV2GroupMetadata: + """ + Create a default (empty) v2 group metadata model, with optional overrides. + + The default is a structurally-valid group with no attributes — the group + analog of `list()` returning `[]`. Any field can be overridden by keyword + (the same fields accepted by `update`). + """ + default = cls(attributes=UNSET) + return default.update(**overrides) + + def update(self, **kwargs: Unpack[ZarrV2GroupMetadataPartial]) -> ZarrV2GroupMetadata: + """ + Return a new `ZarrV2GroupMetadata` with the given fields updated. + + Only the constructor-settable fields listed in + `ZarrV2GroupMetadataPartial` can be updated; the fixed `zarr_format` + is rejected at the type level. Each given field fully replaces its + previous value. + """ + return dataclasses.replace(self, **kwargs) + + def to_json(self) -> ZarrV2GroupMetadataJSON: + """Return the merged in-memory document form. + + `attributes` is included when set (even empty). This is not the + on-disk `.zgroup` content: a conforming `.zgroup` must exclude + `attributes` (they live in the sibling `.zattrs` file). Use + `to_key_value` to produce the spec-conforming split for storage. + """ + # to_json output shares no mutable state with the model. + out: ZarrV2GroupMetadataJSON = {"zarr_format": self.zarr_format} + if self.attributes is not UNSET: + out["attributes"] = copy.deepcopy(self.attributes) + return out + + @classmethod + def from_json(cls, data: object) -> ZarrV2GroupMetadata: + parsed = parse_group_metadata_v2(arrays_to_tuples(data)) + return cls(attributes=(dict(parsed["attributes"]) if "attributes" in parsed else UNSET)) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2GroupMetadata: + zgroup_raw = cast("object", load_store_json(mapping, GROUP_METADATA_STORE_KEY_V2)) + if not isinstance(zgroup_raw, Mapping): + return cls.from_json(zgroup_raw) + zgroup = cast("Mapping[str, object]", zgroup_raw) + if "attributes" in zgroup: + raise MetadataValidationError( + [ + ValidationProblem( + ("attributes",), + "unexpected document member", + "invalid_value", + ) + ] + ) + if ATTRIBUTES_STORE_KEY_V2 in mapping: + zattrs = cast("object", load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2)) + return cls.from_json({**zgroup, "attributes": zattrs}) + return cls.from_json(zgroup) + + def to_key_value( + self, *, indent: int | str | None = None + ) -> Mapping[ZarrV2GroupMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]: + # Attributes live only in the sibling `.zattrs` file; the `.zgroup` + # document must exclude them. The `.zattrs` key is present exactly + # when attributes are set (even empty) — UNSET emits no file. + zgroup = {k: v for k, v in self.to_json().items() if k != "attributes"} + out: dict[ZarrV2GroupMetadataStoreKey | ZarrV2AttributesStoreKey, bytes] = { + GROUP_METADATA_STORE_KEY_V2: dump_store_json(zgroup, indent=indent) + } + if self.attributes is not UNSET: + out[ATTRIBUTES_STORE_KEY_V2] = dump_store_json(self.attributes, indent=indent) + return out + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ZarrV2ConsolidatedMetadata: + """In-memory model of a v2 `.zmetadata` document. + + The `metadata` map holds the flat file-keyed entries (`"path/.zarray"`, + `"path/.zattrs"`, ...) verbatim, preserving the normalized JSON tree. + Entries are deliberately NOT merged into per-node models: which nodes had + a `.zattrs` file at all is information the canonical representation must + keep. Interpreting entries into node models is consumer work. + """ + + zarr_consolidated_format: Literal[1] = field(default=1, init=False) + metadata: dict[str, JSONValue] + + def to_json(self) -> dict[str, JSONValue]: + # to_json output shares no mutable state with the model. + return { + "zarr_consolidated_format": self.zarr_consolidated_format, + "metadata": copy.deepcopy(self.metadata), + } + + @classmethod + def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata: + normalized = arrays_to_tuples(data) + if not isinstance(normalized, Mapping): + raise MetadataValidationError( + [ValidationProblem((), "expected a mapping", "invalid_type")] + ) + doc = cast("Mapping[str, object]", normalized) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key", "missing_key") + for key in ("zarr_consolidated_format", "metadata") + if key not in doc + ] + problems.extend( + ValidationProblem((key,), "unexpected document member", "invalid_value") + for key in doc.keys() - {"zarr_consolidated_format", "metadata"} + ) + if "zarr_consolidated_format" in doc and ( + not isinstance(doc["zarr_consolidated_format"], int) + or isinstance(doc["zarr_consolidated_format"], bool) + or doc["zarr_consolidated_format"] != 1 + ): + problems.append( + ValidationProblem( + ("zarr_consolidated_format",), + f"expected 1, got {doc['zarr_consolidated_format']!r}", + "invalid_value", + ) + ) + if "metadata" in doc: + entries = doc["metadata"] + if not isinstance(entries, Mapping) or not all( + isinstance(k, str) for k in cast("Mapping[object, object]", entries) + ): + problems.append( + ValidationProblem( + ("metadata",), "expected a mapping with string keys", "invalid_type" + ) + ) + else: + for key, value in cast("Mapping[str, object]", entries).items(): + problems.extend( + ValidationProblem( + ("metadata", key, *problem.loc), problem.message, problem.kind + ) + for problem in validate_json(value) + ) + if problems: + raise MetadataValidationError(problems) + entries_tupled = cast( + "dict[str, JSONValue]", + arrays_to_tuples(dict(cast("Mapping[str, object]", doc["metadata"]))), + ) + return cls(metadata=entries_tupled) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ConsolidatedMetadata: + return cls.from_json(load_store_json(mapping, CONSOLIDATED_METADATA_STORE_KEY_V2)) + + def to_key_value( + self, *, indent: int | str | None = None + ) -> Mapping[ZarrV2ConsolidatedMetadataStoreKey, bytes]: + return {CONSOLIDATED_METADATA_STORE_KEY_V2: dump_store_json(self.to_json(), indent=indent)} diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py new file mode 100644 index 0000000000..ad71e216fa --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -0,0 +1,37 @@ +"""The absence sentinel for optional metadata-document keys. + +The models observe one invariant: `None` in a model always corresponds to a +JSON `null` in the document (a v2 `compressor`/`filters` value, an unnamed +dimension inside `dimension_names`), and `UNSET` always means the document +key is absent. The two are never interchangeable, so a model value can never +leak into a document as a spelling the writer did not intend. + +Check with identity: `if model.dimension_names is UNSET: ...`. + +Because the contract is identity, the sentinel must never be reconstructed +from state: pickling and copying work by *reference* (typing_extensions >= +4.16 implements `Sentinel.__reduce__` as a lookup of the sentinel's name on +its defining module), so `pickle.loads(pickle.dumps(UNSET)) is UNSET` holds +across process boundaries, and models holding `UNSET` pickle and deep-copy +freely. Earlier typing_extensions releases refused to pickle sentinels +outright — hence the `>=4.16` floor in this package's dependencies. + +Checker support (PEP 661 is Final; stdlib `sentinel` arrives in Python +3.15): ty types this spelling exactly, including `is`/`is not` narrowing. +Pyright supports it but a regression (1.1.405+, tracked as +https://github.com/microsoft/pyright/issues/11115) degrades class-attribute +reads to `Unknown`, so this package pins pyright to the last good version +until the fix lands. Mypy support is in review +(https://github.com/python/mypy/pull/21647); until it merges, mypy-checked +consumers of these fields need a `cast` or `type: ignore` at narrowing +sites. This is a deliberate short-term cost: the sentinel is the standard, +and the checkers are converging on it. +""" + +from __future__ import annotations + +from typing_extensions import Sentinel + +UNSET = Sentinel("UNSET") +"""Marks a metadata-document key as absent (PEP 661 sentinel; usable directly +in type expressions, e.g. `tuple[str, ...] | UNSET`). Test with `is UNSET`.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py new file mode 100644 index 0000000000..a12e1911b1 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -0,0 +1,875 @@ +"""Structural validation for Zarr metadata documents. + +Validators check JSON structure (key presence, value shapes, and fixed +literals like `zarr_format`), not domain validity. Each concept gets a +`validate_*` function returning every problem found, an `is_*` type guard, +and a `parse_*` function that narrows or raises `MetadataValidationError`. + +Every `ValidationProblem` carries a machine-readable `kind` alongside its +human-readable `message`, so consumers can dispatch on the failure mode +(`missing_key`, `invalid_type`, `invalid_value`, `invalid_json`) without +string-matching messages. +""" + +from __future__ import annotations + +import json +import math +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any, Final, Literal, NoReturn, cast + +from typing_extensions import TypeIs + +from zarr_metadata._common import JSONValue +from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON + +ProblemKind = Literal["missing_key", "invalid_type", "invalid_value", "invalid_json"] +"""Machine-readable classification of a `ValidationProblem`. + +- `missing_key`: a required key (document key or store key) is absent. +- `invalid_type`: a value has the wrong structural type (e.g. a string where + a mapping is required, a non-JSON-serializable object). +- `invalid_value`: a value has an acceptable type but an invalid content + (e.g. `zarr_format: 2` in a v3 document, `order: "Q"`). +- `invalid_json`: bytes that do not decode as JSON. +""" + + +@dataclass(frozen=True, slots=True) +class ValidationProblem: + """A single structural problem found while validating a metadata document. + + `loc` is the path from the document root to the offending value, e.g. + `("codecs", 0, "name")`. An empty `loc` refers to the document as a whole. + `kind` classifies the failure mode for programmatic dispatch; `message` + is the human-readable description. + """ + + loc: tuple[str | int, ...] + message: str + kind: ProblemKind + + def __str__(self) -> str: + location = ".".join(str(part) for part in self.loc) if self.loc else "" + return f"{location}: {self.message}" + + +class MetadataValidationError(ValueError): + """Raised when a value fails structural metadata validation. + + Carries every problem found (not just the first) in `.problems`. + """ + + def __init__(self, problems: list[ValidationProblem]) -> None: + self.problems = problems + super().__init__("\n".join(str(problem) for problem in problems)) + + +def _prefix(loc_head: str | int, problems: list[ValidationProblem]) -> list[ValidationProblem]: + """Prepend `loc_head` to the `loc` of every problem (for nested validators).""" + return [ValidationProblem((loc_head, *p.loc), p.message, p.kind) for p in problems] + + +def validate_json(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not JSON-serializable (recursively).""" + if isinstance(value, float): + if math.isfinite(value): + return [] + return [ValidationProblem((), f"non-finite float {value!r} is not JSON", "invalid_value")] + if isinstance(value, (str, int, bool)) or value is None: + return [] + problems: list[ValidationProblem] = [] + if isinstance(value, Mapping): + for key, item in cast("Mapping[object, object]", value).items(): + if not isinstance(key, str): + problems.append( + ValidationProblem((), f"non-string key {key!r} in JSON object", "invalid_type") + ) + continue + problems.extend(_prefix(key, validate_json(item))) + return problems + if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): + for index, item in enumerate(cast("Sequence[object]", value)): + problems.extend(_prefix(index, validate_json(item))) + return problems + return [ValidationProblem((), f"not a JSON-serializable value: {value!r}", "invalid_type")] + + +def _is_canonical_json(value: object) -> TypeIs[JSONValue]: + """Whether `value` already uses the concrete containers in `JSONValue`.""" + if isinstance(value, float): + return math.isfinite(value) + if isinstance(value, (str, int, bool)) or value is None: + return True + if isinstance(value, (list, tuple)): + sequence = cast("list[object] | tuple[object, ...]", value) + return all(_is_canonical_json(item) for item in sequence) + if isinstance(value, dict): + mapping = cast("dict[object, object]", value) + return all( + isinstance(key, str) and _is_canonical_json(item) for key, item in mapping.items() + ) + return False + + +def is_json(value: object) -> TypeIs[JSONValue]: + """Whether `value` is a canonical JSON structure (recursively).""" + return _is_canonical_json(value) + + +def parse_json(value: object) -> JSONValue: + """Return a canonical `JSONValue`, or raise `MetadataValidationError`.""" + normalized = arrays_to_tuples(value) + problems = validate_json(normalized) + if problems: + raise MetadataValidationError(problems) + return cast(JSONValue, normalized) + + +# The standard top-level keys of a v3 array metadata document. Anything outside +# this set is an extension field. Built from the TypedDict's required/optional +# key sets (which resolve inherited keys, unlike `__annotations__`). +ARRAY_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( + ZarrV3ArrayMetadataJSON.__required_keys__ +) +ARRAY_METADATA_OPTIONAL_KEYS_V3: Final[frozenset[str]] = frozenset( + ZarrV3ArrayMetadataJSON.__optional_keys__ +) +ARRAY_METADATA_STANDARD_KEYS_V3: Final[frozenset[str]] = ( + ARRAY_METADATA_REQUIRED_KEYS_V3 | ARRAY_METADATA_OPTIONAL_KEYS_V3 +) + +ARRAY_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( + ZarrV2ArrayMetadataJSON.__required_keys__ +) +ARRAY_METADATA_OPTIONAL_KEYS_V2: Final[frozenset[str]] = frozenset( + ZarrV2ArrayMetadataJSON.__optional_keys__ +) +ARRAY_METADATA_STANDARD_KEYS_V2: Final[frozenset[str]] = ( + ARRAY_METADATA_REQUIRED_KEYS_V2 | ARRAY_METADATA_OPTIONAL_KEYS_V2 +) + +# The standard top-level keys of a v3 group metadata document. Anything outside +# this set is an extension field. +GROUP_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( + ZarrV3GroupMetadataJSON.__required_keys__ +) +GROUP_METADATA_OPTIONAL_KEYS_V3: Final[frozenset[str]] = frozenset( + ZarrV3GroupMetadataJSON.__optional_keys__ +) +GROUP_METADATA_STANDARD_KEYS_V3: Final[frozenset[str]] = ( + GROUP_METADATA_REQUIRED_KEYS_V3 | GROUP_METADATA_OPTIONAL_KEYS_V3 +) + +GROUP_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( + ZarrV2GroupMetadataJSON.__required_keys__ +) +GROUP_METADATA_OPTIONAL_KEYS_V2: Final[frozenset[str]] = frozenset( + ZarrV2GroupMetadataJSON.__optional_keys__ +) +GROUP_METADATA_STANDARD_KEYS_V2: Final[frozenset[str]] = ( + GROUP_METADATA_REQUIRED_KEYS_V2 | GROUP_METADATA_OPTIONAL_KEYS_V2 +) + + +def _missing_keys(required: frozenset[str], doc: Mapping[str, object]) -> list[ValidationProblem]: + """One `missing_key` problem per required key absent from `doc`.""" + return [ + ValidationProblem((key,), "missing required key", "missing_key") + for key in sorted(required - doc.keys()) + ] + + +def _unexpected_keys( + allowed: frozenset[str], doc: Mapping[object, object] +) -> list[ValidationProblem]: + """One problem per member outside a closed document's declared shape.""" + problems: list[ValidationProblem] = [] + for key in doc: + if not isinstance(key, str): + problems.append( + ValidationProblem((), f"non-string document key {key!r}", "invalid_type") + ) + elif key not in allowed: + problems.append( + ValidationProblem((key,), "unexpected document member", "invalid_value") + ) + return problems + + +def _check_literal( + doc: Mapping[str, object], key: str, expected: object +) -> list[ValidationProblem]: + """One `invalid_value` problem if `doc[key]` is present but not `expected`.""" + if key in doc and (type(doc[key]) is not type(expected) or doc[key] != expected): + return [ + ValidationProblem((key,), f"expected {expected!r}, got {doc[key]!r}", "invalid_value") + ] + return [] + + +def _validate_extension_fields_v3( + doc: Mapping[object, object], + standard_keys: frozenset[str], + *, + additional_reserved_keys: frozenset[str] = frozenset(), +) -> list[ValidationProblem]: + """Validate v3 top-level key types and unknown-field JSON payloads.""" + problems: list[ValidationProblem] = [] + reserved_keys = standard_keys | additional_reserved_keys + for key, value in doc.items(): + if not isinstance(key, str): + problems.append( + ValidationProblem((), f"non-string top-level key {key!r}", "invalid_type") + ) + continue + if key in reserved_keys: + continue + problems.extend(_prefix(key, validate_json(value))) + return problems + + +def validate_metadata_field_v3( + value: object, *, allow_must_understand_false: bool = True +) -> list[ValidationProblem]: + """Return every reason `value` is not a v3 metadata field. + + A metadata field is a bare name string or a mapping containing `name` and + optional `configuration` and `must_understand` members. + """ + if isinstance(value, str): + return [] + if not isinstance(value, Mapping): + return [ + ValidationProblem( + (), + "expected a metadata field (string or extension object)", + "invalid_type", + ) + ] + field = cast("Mapping[object, object]", value) + problems: list[ValidationProblem] = [] + allowed_keys = frozenset({"name", "configuration", "must_understand"}) + for key in field: + if not isinstance(key, str): + problems.append( + ValidationProblem((), f"non-string metadata field key {key!r}", "invalid_type") + ) + elif key not in allowed_keys: + problems.append( + ValidationProblem((key,), "unexpected metadata field member", "invalid_value") + ) + if not isinstance(field.get("name"), str): + problems.append(ValidationProblem(("name",), "expected a string name", "invalid_type")) + if "configuration" in field: + configuration = field["configuration"] + if not isinstance(configuration, Mapping): + problems.append( + ValidationProblem(("configuration",), "expected a mapping", "invalid_type") + ) + elif not all(isinstance(k, str) for k in cast("Mapping[object, object]", configuration)): + problems.append( + ValidationProblem(("configuration",), "expected string keys", "invalid_type") + ) + else: + for key, item in cast("Mapping[str, object]", configuration).items(): + problems.extend(_prefix("configuration", _prefix(key, validate_json(item)))) + if "must_understand" in field: + must_understand = field["must_understand"] + if not isinstance(must_understand, bool): + problems.append( + ValidationProblem(("must_understand",), "expected a boolean", "invalid_type") + ) + elif not allow_must_understand_false and not must_understand: + problems.append( + ValidationProblem( + ("must_understand",), + "false is not supported at this extension point", + "invalid_value", + ) + ) + return problems + + +def is_metadata_field_v3(value: object) -> TypeIs[ZarrV3MetadataFieldJSON]: + """Whether `value` is a v3 metadata field: a bare name or a named config.""" + if isinstance(value, str): + return True + if not isinstance(value, dict): + return False + field = cast("dict[object, object]", value) + return _is_canonical_json(field) and not validate_metadata_field_v3(field) + + +def parse_metadata_field_v3(value: object) -> ZarrV3MetadataFieldJSON: + """Return `value` narrowed to `ZarrV3MetadataFieldJSON`, or raise `MetadataValidationError`.""" + normalized = arrays_to_tuples(value) + problems = validate_metadata_field_v3(normalized) + if problems: + raise MetadataValidationError(problems) + return cast(ZarrV3MetadataFieldJSON, normalized) + + +def _is_int_sequence(value: object) -> bool: + """Whether `value` is a non-string sequence of integers. + + JSON booleans decode to `bool`, which is an `int` subclass in Python but + is not an integer in a metadata document, so booleans are excluded. + """ + return ( + not isinstance(value, (str, bytes, bytearray)) + and isinstance(value, Sequence) + and all( + isinstance(item, int) and not isinstance(item, bool) + for item in cast("Sequence[object]", value) + ) + ) + + +def _validate_dim_sequence(doc: Mapping[str, object], key: str) -> list[ValidationProblem]: + """Validate a dimension sequence (`shape` / `chunks`) if present in `doc`. + + Dimension lengths are non-negative integers. + """ + if key not in doc: + return [] + value = doc[key] + if not _is_int_sequence(value): + return [ValidationProblem((key,), "expected a sequence of int", "invalid_type")] + if any(item < 0 for item in cast("Sequence[int]", value)): + return [ValidationProblem((key,), "expected non-negative integers", "invalid_value")] + return [] + + +def _is_dtype_v2(value: object) -> bool: + """Whether `value` is shaped like a v2 dtype: a string or field records. + + A field record is a `(name, dtype)` or `(name, dtype, shape)` sequence, + where `dtype` is itself a string or nested field records and `shape` is a + sequence of int. The string content is NOT interpreted — whether the + string names a real dtype is domain validity, not structure. + """ + if isinstance(value, str): + return True + if not isinstance(value, Sequence): + return False + for record in cast("Sequence[object]", value): + if isinstance(record, str) or not isinstance(record, Sequence): + return False + fields = cast("Sequence[object]", record) + if len(fields) not in (2, 3): + return False + if not isinstance(fields[0], str): + return False + if not _is_dtype_v2(fields[1]): + return False + if len(fields) == 3 and not _is_int_sequence(fields[2]): + return False + return True + + +def _is_canonical_dtype_v2(value: object) -> bool: + """Whether a validated v2 dtype uses the tuple-backed public representation.""" + if isinstance(value, str): + return True + if not isinstance(value, tuple): + return False + for record in cast("tuple[object, ...]", value): + if not isinstance(record, tuple): + return False + fields = cast("tuple[object, ...]", record) + if not _is_canonical_dtype_v2(fields[1]): + return False + if len(fields) == 3 and not isinstance(fields[2], tuple): + return False + return True + + +def _is_canonical_metadata_field_v3(value: object) -> bool: + """Whether a validated v3 metadata field has its declared runtime container type.""" + return isinstance(value, (str, dict)) + + +def _is_canonical_array_metadata_v3(value: object) -> bool: + """Whether a validated v3 array document matches `ZarrV3ArrayMetadataJSON` at runtime.""" + if not isinstance(value, dict): + return False + doc = cast("dict[str, object]", value) + if not isinstance(doc["shape"], tuple) or not isinstance(doc["codecs"], tuple): + return False + if "storage_transformers" in doc and not isinstance(doc["storage_transformers"], tuple): + return False + if "dimension_names" in doc and not isinstance(doc["dimension_names"], tuple): + return False + if not all( + _is_canonical_metadata_field_v3(doc[key]) + for key in ("data_type", "chunk_grid", "chunk_key_encoding") + ): + return False + if not all( + _is_canonical_metadata_field_v3(item) for item in cast("tuple[object, ...]", doc["codecs"]) + ): + return False + return "storage_transformers" not in doc or all( + _is_canonical_metadata_field_v3(item) + for item in cast("tuple[object, ...]", doc["storage_transformers"]) + ) + + +def _is_canonical_array_metadata_v2(value: object) -> bool: + """Whether a validated v2 array document matches `ZarrV2ArrayMetadataJSON` at runtime.""" + if not isinstance(value, dict): + return False + doc = cast("dict[str, object]", value) + if not isinstance(doc["shape"], tuple) or not isinstance(doc["chunks"], tuple): + return False + if not _is_canonical_dtype_v2(doc["dtype"]): + return False + compressor = doc["compressor"] + if compressor is not None and not isinstance(compressor, dict): + return False + filters = doc["filters"] + return filters is None or ( + isinstance(filters, tuple) + and all(isinstance(item, dict) for item in cast("tuple[object, ...]", filters)) + ) + + +def _is_codec_v2(value: object) -> bool: + """Whether `value` is shaped like a v2 codec config: a mapping with a string `id`.""" + return isinstance(value, Mapping) and isinstance( + cast("Mapping[object, object]", value).get("id"), str + ) + + +def _validate_codec_v2(value: object) -> list[ValidationProblem]: + """Validate a v2 codec's required shape and JSON-valued configuration.""" + if not _is_codec_v2(value): + return [ + ValidationProblem( + (), "expected a codec configuration with a string 'id'", "invalid_type" + ) + ] + return validate_json(value) + + +def _validate_attributes(value: object) -> list[ValidationProblem]: + """Validate an `attributes` value: a mapping with string keys. + + Returns a problem at `("attributes",)` if it is not, else `[]`. Shared by the + v2 and v3 validators. Unlike the other `validate_*` functions (which + return value-relative locs for the caller to `_prefix`), this emits the + already-parent-relative `("attributes",)` loc, since it is only ever called + with a document's `attributes` value. + """ + if not isinstance(value, Mapping) or not all( + isinstance(k, str) for k in cast("Mapping[object, object]", value) + ): + return [ + ValidationProblem( + ("attributes",), "expected a mapping with string keys", "invalid_type" + ) + ] + problems: list[ValidationProblem] = [] + for key, item in cast("Mapping[str, object]", value).items(): + problems.extend(_prefix("attributes", _prefix(key, validate_json(item)))) + return problems + + +def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v3 array doc. + + Checks structure, not domain validity. Unknown top-level keys are allowed + (they map to `extra_fields`). + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping", "invalid_type")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V3, doc) + problems.extend( + _validate_extension_fields_v3( + cast("Mapping[object, object]", value), ARRAY_METADATA_STANDARD_KEYS_V3 + ) + ) + problems.extend(_check_literal(doc, "zarr_format", 3)) + problems.extend(_check_literal(doc, "node_type", "array")) + problems.extend(_validate_dim_sequence(doc, "shape")) + if "fill_value" in doc: + problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) + for key in ("data_type", "chunk_grid", "chunk_key_encoding"): + if key in doc: + problems.extend( + _prefix( + key, + validate_metadata_field_v3(doc[key], allow_must_understand_false=False), + ) + ) + for key in ("codecs", "storage_transformers"): + if key in doc: + entries = doc[key] + if isinstance(entries, str) or not isinstance(entries, Sequence): + problems.append(ValidationProblem((key,), "expected a sequence", "invalid_type")) + else: + if key == "codecs" and len(cast("Sequence[object]", entries)) == 0: + problems.append( + ValidationProblem( + ("codecs",), "expected at least one codec", "invalid_value" + ) + ) + for index, entry in enumerate(cast("Sequence[object]", entries)): + problems.extend(_prefix(key, _prefix(index, validate_metadata_field_v3(entry)))) + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + if "dimension_names" in doc: + # Simple typed sequences (dimension_names, shape, chunks) report a single + # field-level loc, not per-bad-item locs; per-index locs are reserved for + # the metadata-field lists (codecs, storage_transformers). + names = doc["dimension_names"] + if isinstance(names, str) or not isinstance(names, Sequence): + problems.append( + ValidationProblem(("dimension_names",), "expected a sequence", "invalid_type") + ) + elif not all( + item is None or isinstance(item, str) for item in cast("Sequence[object]", names) + ): + problems.append( + ValidationProblem( + ("dimension_names",), "expected items of str or None", "invalid_type" + ) + ) + elif _is_int_sequence(doc.get("shape")) and len(cast("Sequence[object]", names)) != len( + cast("Sequence[int]", doc["shape"]) + ): + problems.append( + ValidationProblem( + ("dimension_names",), + "expected one name per dimension of shape", + "invalid_value", + ) + ) + return problems + + +def is_array_metadata_v3(value: object) -> TypeIs[ZarrV3ArrayMetadataJSON]: + """Whether `value` is a structurally-valid v3 array metadata document.""" + return ( + _is_canonical_json(value) + and not validate_array_metadata_v3(value) + and _is_canonical_array_metadata_v3(value) + ) + + +def parse_array_metadata_v3(value: object) -> ZarrV3ArrayMetadataJSON: + """Return `value` as `ZarrV3ArrayMetadataJSON`, or raise `MetadataValidationError`.""" + normalized = arrays_to_tuples(value) + problems = validate_array_metadata_v3(normalized) + if problems: + raise MetadataValidationError(problems) + return cast("ZarrV3ArrayMetadataJSON", normalized) + + +def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v2 array doc. + + Checks structure, not domain validity: `dtype` must be a string or field + records, but the string content is not interpreted; `compressor` and + `filters` are required keys that may be `None`, and otherwise must be + codec configurations (mappings with a string `id`). + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping", "invalid_type")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V2, doc) + problems.extend( + _unexpected_keys(ARRAY_METADATA_STANDARD_KEYS_V2, cast("Mapping[object, object]", value)) + ) + problems.extend(_check_literal(doc, "zarr_format", 2)) + shape_problems = _validate_dim_sequence(doc, "shape") + chunks_problems = _validate_dim_sequence(doc, "chunks") + problems.extend(shape_problems) + problems.extend(chunks_problems) + if ( + not shape_problems + and not chunks_problems + and _is_int_sequence(doc.get("shape")) + and _is_int_sequence(doc.get("chunks")) + ): + shape = cast("Sequence[int]", doc["shape"]) + chunks = cast("Sequence[int]", doc["chunks"]) + if len(shape) != len(chunks): + problems.append( + ValidationProblem( + ("chunks",), + "expected the same number of dimensions as shape", + "invalid_value", + ) + ) + if "dtype" in doc and not _is_dtype_v2(doc["dtype"]): + problems.append( + ValidationProblem( + ("dtype",), + "expected a v2 dtype string or a sequence of field records", + "invalid_type", + ) + ) + if "order" in doc and doc["order"] not in ("C", "F"): + problems.append( + ValidationProblem( + ("order",), f"expected 'C' or 'F', got {doc['order']!r}", "invalid_value" + ) + ) + if "compressor" in doc: + compressor = doc["compressor"] + if compressor is not None: + problems.extend(_prefix("compressor", _validate_codec_v2(compressor))) + if "filters" in doc: + filters = doc["filters"] + if filters is not None and ( + isinstance(filters, str) + or not isinstance(filters, Sequence) + or not all(_is_codec_v2(item) for item in cast("Sequence[object]", filters)) + ): + problems.append( + ValidationProblem( + ("filters",), + "expected null or a sequence of codec configurations with string 'id's", + "invalid_type", + ) + ) + elif filters is not None: + if len(cast("Sequence[object]", filters)) == 0: + problems.append( + ValidationProblem(("filters",), "expected at least one filter", "invalid_value") + ) + for index, item in enumerate(cast("Sequence[object]", filters)): + problems.extend(_prefix("filters", _prefix(index, validate_json(item)))) + if "dimension_separator" in doc and doc["dimension_separator"] not in (".", "/"): + problems.append( + ValidationProblem( + ("dimension_separator",), + f"expected '.' or '/', got {doc['dimension_separator']!r}", + "invalid_value", + ) + ) + if "fill_value" in doc: + problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + return problems + + +def is_array_metadata_v2(value: object) -> TypeIs[ZarrV2ArrayMetadataJSON]: + """Whether `value` is a structurally-valid v2 array metadata document.""" + return ( + _is_canonical_json(value) + and not validate_array_metadata_v2(value) + and _is_canonical_array_metadata_v2(value) + ) + + +def parse_array_metadata_v2(value: object) -> ZarrV2ArrayMetadataJSON: + """Return `value` as `ZarrV2ArrayMetadataJSON`, or raise `MetadataValidationError`.""" + normalized = arrays_to_tuples(value) + problems = validate_array_metadata_v2(normalized) + if problems: + raise MetadataValidationError(problems) + return cast("ZarrV2ArrayMetadataJSON", normalized) + + +def validate_consolidated_metadata_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a valid inline consolidated envelope. + + Locs are value-relative (the caller prefixes with `consolidated_metadata` + where appropriate). Entries recurse into the array and group document + validators, so a validator verdict always agrees with what + `ZarrV3ConsolidatedMetadata.from_json` accepts. + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping", "invalid_type")] + env = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key", "missing_key") + for key in ("kind", "must_understand", "metadata") + if key not in env + ] + problems.extend( + _unexpected_keys( + frozenset({"kind", "must_understand", "metadata"}), + cast("Mapping[object, object]", value), + ) + ) + problems.extend(_check_literal(env, "kind", "inline")) + if "must_understand" in env and env["must_understand"] is not False: + problems.append(ValidationProblem(("must_understand",), "expected False", "invalid_value")) + if "metadata" in env: + entries = env["metadata"] + if not isinstance(entries, Mapping): + problems.append(ValidationProblem(("metadata",), "expected a mapping", "invalid_type")) + else: + for key, entry in cast("Mapping[object, object]", entries).items(): + if not isinstance(key, str): + problems.append( + ValidationProblem(("metadata",), f"non-string key {key!r}", "invalid_type") + ) + continue + entry_obj: object = entry + node_type: object = None + if isinstance(entry, Mapping): + node_type = cast("Mapping[str, object]", entry).get("node_type") + if node_type == "array": + problems.extend( + _prefix("metadata", _prefix(key, validate_array_metadata_v3(entry_obj))) + ) + elif node_type == "group": + problems.extend( + _prefix("metadata", _prefix(key, validate_group_metadata_v3(entry_obj))) + ) + else: + problems.append( + ValidationProblem( + ("metadata", key, "node_type"), + "expected 'array' or 'group'", + "invalid_value", + ) + ) + return problems + + +def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v3 group doc. + + Checks structure, not domain validity. Unknown top-level keys are allowed + (they map to `extra_fields`); a `consolidated_metadata` key, if present, + is deep-validated (envelope and entries) via + `validate_consolidated_metadata_v3`. + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping", "invalid_type")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = _missing_keys(GROUP_METADATA_REQUIRED_KEYS_V3, doc) + problems.extend( + _validate_extension_fields_v3( + cast("Mapping[object, object]", value), + GROUP_METADATA_STANDARD_KEYS_V3, + additional_reserved_keys=frozenset({"consolidated_metadata"}), + ) + ) + problems.extend(_check_literal(doc, "zarr_format", 3)) + problems.extend(_check_literal(doc, "node_type", "group")) + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + if "consolidated_metadata" in doc and doc["consolidated_metadata"] is not None: + # consolidated_metadata: null (a historical zarr-python bug) is + # structurally accepted so those stores remain readable, but the model + # repairs it to absence on read and never writes it back. + problems.extend( + _prefix( + "consolidated_metadata", + validate_consolidated_metadata_v3(doc["consolidated_metadata"]), + ) + ) + return problems + + +def is_group_metadata_v3(value: object) -> TypeIs[ZarrV3GroupMetadataJSON]: + """Whether `value` is a structurally-valid v3 group metadata document.""" + return _is_canonical_json(value) and not validate_group_metadata_v3(value) + + +def parse_group_metadata_v3(value: object) -> ZarrV3GroupMetadataJSON: + """Return `value` narrowed to `ZarrV3GroupMetadataJSON`, or raise `MetadataValidationError`.""" + normalized = arrays_to_tuples(value) + problems = validate_group_metadata_v3(normalized) + if problems: + raise MetadataValidationError(problems) + return cast(ZarrV3GroupMetadataJSON, normalized) + + +def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v2 group doc. + + Validates the in-memory merged form: the `.zgroup` fields plus an + optional `attributes` mapping folded in from `.zattrs`. + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping", "invalid_type")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = _missing_keys(GROUP_METADATA_REQUIRED_KEYS_V2, doc) + problems.extend( + _unexpected_keys(GROUP_METADATA_STANDARD_KEYS_V2, cast("Mapping[object, object]", value)) + ) + problems.extend(_check_literal(doc, "zarr_format", 2)) + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + return problems + + +def is_group_metadata_v2(value: object) -> TypeIs[ZarrV2GroupMetadataJSON]: + """Whether `value` is a structurally-valid v2 group metadata document.""" + return _is_canonical_json(value) and not validate_group_metadata_v2(value) + + +def parse_group_metadata_v2(value: object) -> ZarrV2GroupMetadataJSON: + """Return `value` narrowed to `ZarrV2GroupMetadataJSON`, or raise `MetadataValidationError`.""" + normalized = arrays_to_tuples(value) + problems = validate_group_metadata_v2(normalized) + if problems: + raise MetadataValidationError(problems) + return cast(ZarrV2GroupMetadataJSON, normalized) + + +def _reject_json_constant(constant: str) -> NoReturn: + """Reject the JavaScript constants accepted by Python's JSON decoder.""" + raise ValueError(f"non-standard JSON constant {constant!r}") + + +def load_store_json(mapping: Mapping[str, bytes], key: str) -> Any: + """Decode the JSON document stored at `key` in `mapping`. + + Every ingestion failure surfaces as `MetadataValidationError`: a missing + store key is a `missing_key` problem and undecodable bytes are an + `invalid_json` problem, rather than leaking `KeyError` / + `json.JSONDecodeError` to callers. + """ + if key not in mapping: + raise MetadataValidationError( + [ValidationProblem((key,), "missing store key", "missing_key")] + ) + try: + return json.loads(mapping[key], parse_constant=_reject_json_constant) + except (UnicodeDecodeError, ValueError) as exc: + raise MetadataValidationError( + [ValidationProblem((key,), f"invalid JSON: {exc}", "invalid_json")] + ) from exc + + +def dump_store_json(value: object, *, indent: int | str | None = None) -> bytes: + """Encode a metadata document as strict RFC 8259 JSON bytes.""" + return json.dumps(value, indent=indent, allow_nan=False).encode("utf-8") + + +def arrays_to_tuples(obj: object) -> object: + """Recursively materialize mappings and convert array-like values to tuples.""" + if isinstance(obj, Sequence) and not isinstance(obj, (str, bytes, bytearray)): + sequence = cast("Sequence[object]", obj) + converted_sequence = tuple(arrays_to_tuples(item) for item in sequence) + if isinstance(obj, tuple) and all( + converted is original + for converted, original in zip(converted_sequence, sequence, strict=True) + ): + return cast("tuple[object, ...]", obj) + return converted_sequence + if isinstance(obj, Mapping): + mapping = cast("Mapping[object, object]", obj) + converted: dict[object, object] = { + key: arrays_to_tuples(value) for key, value in mapping.items() + } + if isinstance(obj, dict) and all(converted[key] is value for key, value in mapping.items()): + return cast("object", obj) + return converted + return obj diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py new file mode 100644 index 0000000000..8584efa570 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -0,0 +1,176 @@ +"""Optional pydantic (v2) integration: field types over the core models. + +Importing this module requires pydantic; the core package deliberately does +not depend on it, so this module is never imported by `zarr_metadata` itself. + +Each exported name is an `Annotated` field type over the corresponding core +model class — the instances ARE the core classes, so values interoperate +freely with non-pydantic code (equality, isinstance, nesting). Validation +delegates to the library: a raw document routes through `from_json` (the +single source of truth for structural validation and normalization, so +pydantic's field-level coercion can never bypass it), an existing model +instance passes through unchanged, and serialization emits the canonical +document via `to_json`. `MetadataValidationError` subclasses `ValueError`, +so a failed parse surfaces as a pydantic `ValidationError` carrying the +loc-annotated problem messages. + +Usage: + + import zarr_metadata.pydantic as zmp + + class ArrayManifest(BaseModel): + path: str + metadata: zmp.ZarrV3ArrayMetadata + +Static type checkers see each field type as its core model class, so +`manifest.metadata` is a `zarr_metadata.model.ZarrV3ArrayMetadata`. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Annotated, TypeVar + +from pydantic import BeforeValidator, InstanceOf, PlainSerializer + +from zarr_metadata import model as _model +from zarr_metadata._pydantic_schema import ( + ZarrV2ArrayMetadataJSON as _ZarrV2ArrayMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV2ConsolidatedMetadataJSON as _ZarrV2ConsolidatedMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV2GroupMetadataJSON as _ZarrV2GroupMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV3ArrayMetadataJSON as _ZarrV3ArrayMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV3ConsolidatedMetadataJSON as _ZarrV3ConsolidatedMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV3GroupMetadataJSON as _ZarrV3GroupMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV3MetadataFieldJSON as _ZarrV3MetadataFieldSchema, +) +from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON as _ZarrV2ArrayMetadataJSON +from zarr_metadata.v2.consolidated import ( + ZarrV2ConsolidatedMetadataJSON as _ZarrV2ConsolidatedMetadataJSON, +) +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON as _ZarrV2GroupMetadataJSON +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON as _ZarrV3MetadataFieldJSON +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON as _ZarrV3ArrayMetadataJSON +from zarr_metadata.v3.consolidated import ( + ZarrV3ConsolidatedMetadataJSON as _ZarrV3ConsolidatedMetadataJSON, +) +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON as _ZarrV3GroupMetadataJSON + +if TYPE_CHECKING: + from collections.abc import Callable + +_M = TypeVar("_M") + + +def _coerce_to(cls: type[_M], parse: Callable[[object], _M]) -> Callable[[object], _M]: + """A validator that passes instances of `cls` through and parses anything else.""" + + def coerce(value: object) -> _M: + if isinstance(value, cls): + return value + return parse(value) + + return coerce + + +ZarrV3ArrayMetadata = Annotated[ + InstanceOf[_model.ZarrV3ArrayMetadata], + BeforeValidator( + _coerce_to(_model.ZarrV3ArrayMetadata, _model.ZarrV3ArrayMetadata.from_json), + json_schema_input_type=_ZarrV3ArrayMetadataSchema, + ), + PlainSerializer(_model.ZarrV3ArrayMetadata.to_json, return_type=_ZarrV3ArrayMetadataJSON), +] +"""Field type for a v3 array metadata document (`zarr.json` content).""" + +ZarrV2ArrayMetadata = Annotated[ + InstanceOf[_model.ZarrV2ArrayMetadata], + BeforeValidator( + _coerce_to(_model.ZarrV2ArrayMetadata, _model.ZarrV2ArrayMetadata.from_json), + json_schema_input_type=_ZarrV2ArrayMetadataSchema, + ), + PlainSerializer(_model.ZarrV2ArrayMetadata.to_json, return_type=_ZarrV2ArrayMetadataJSON), +] +"""Field type for a v2 array metadata document (merged `.zarray` + `.zattrs` form).""" + +ZarrV3GroupMetadata = Annotated[ + InstanceOf[_model.ZarrV3GroupMetadata], + BeforeValidator( + _coerce_to(_model.ZarrV3GroupMetadata, _model.ZarrV3GroupMetadata.from_json), + json_schema_input_type=_ZarrV3GroupMetadataSchema, + ), + PlainSerializer(_model.ZarrV3GroupMetadata.to_json, return_type=_ZarrV3GroupMetadataJSON), +] +"""Field type for a v3 group metadata document (`zarr.json` content).""" + +ZarrV2GroupMetadata = Annotated[ + InstanceOf[_model.ZarrV2GroupMetadata], + BeforeValidator( + _coerce_to(_model.ZarrV2GroupMetadata, _model.ZarrV2GroupMetadata.from_json), + json_schema_input_type=_ZarrV2GroupMetadataSchema, + ), + PlainSerializer(_model.ZarrV2GroupMetadata.to_json, return_type=_ZarrV2GroupMetadataJSON), +] +"""Field type for a v2 group metadata document (merged `.zgroup` + `.zattrs` form).""" + +ZarrV3ConsolidatedMetadata = Annotated[ + InstanceOf[_model.ZarrV3ConsolidatedMetadata], + BeforeValidator( + _coerce_to( + _model.ZarrV3ConsolidatedMetadata, + _model.ZarrV3ConsolidatedMetadata.from_json, + ), + json_schema_input_type=_ZarrV3ConsolidatedMetadataSchema, + ), + PlainSerializer( + _model.ZarrV3ConsolidatedMetadata.to_json, + return_type=_ZarrV3ConsolidatedMetadataJSON, + ), +] +"""Field type for v3 inline consolidated metadata.""" + +ZarrV2ConsolidatedMetadata = Annotated[ + InstanceOf[_model.ZarrV2ConsolidatedMetadata], + BeforeValidator( + _coerce_to( + _model.ZarrV2ConsolidatedMetadata, + _model.ZarrV2ConsolidatedMetadata.from_json, + ), + json_schema_input_type=_ZarrV2ConsolidatedMetadataSchema, + ), + PlainSerializer( + _model.ZarrV2ConsolidatedMetadata.to_json, + return_type=_ZarrV2ConsolidatedMetadataJSON, + ), +] +"""Field type for a v2 `.zmetadata` document.""" + +ZarrV3MetadataField = Annotated[ + InstanceOf[_model.ZarrV3NamedConfig], + BeforeValidator( + _coerce_to(_model.ZarrV3NamedConfig, _model.ZarrV3NamedConfig.from_json), + json_schema_input_type=_ZarrV3MetadataFieldSchema, + ), + PlainSerializer(_model.ZarrV3NamedConfig.to_json, return_type=_ZarrV3MetadataFieldJSON), +] +"""Field type for one normalized v3 metadata extension envelope.""" + +__all__ = [ + "ZarrV2ArrayMetadata", + "ZarrV2ConsolidatedMetadata", + "ZarrV2GroupMetadata", + "ZarrV3ArrayMetadata", + "ZarrV3ConsolidatedMetadata", + "ZarrV3GroupMetadata", + "ZarrV3MetadataField", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py index 4e9a76125b..b9001d168e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py @@ -1,26 +1,26 @@ """Zarr v2 metadata types.""" from zarr_metadata.v2.array import ( - ArrayDimensionSeparatorV2, - ArrayMetadataV2, - ArrayOrderV2, - DataTypeMetadataV2, - ZArrayMetadata, + ZarrV2ArrayDimensionSeparator, + ZarrV2ArrayMetadataJSON, + ZarrV2ArrayOrder, + ZarrV2DataTypeMetadata, + ZarrV2ZArrayJSON, ) -from zarr_metadata.v2.attributes import ZAttrsMetadata -from zarr_metadata.v2.codec import CodecMetadataV2 -from zarr_metadata.v2.consolidated import ConsolidatedMetadataV2 -from zarr_metadata.v2.group import GroupMetadataV2, ZGroupMetadata +from zarr_metadata.v2.attributes import ZarrV2ZAttrsJSON +from zarr_metadata.v2.codec import ZarrV2CodecMetadata +from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON, ZarrV2ZGroupJSON __all__ = [ - "ArrayDimensionSeparatorV2", - "ArrayMetadataV2", - "ArrayOrderV2", - "CodecMetadataV2", - "ConsolidatedMetadataV2", - "DataTypeMetadataV2", - "GroupMetadataV2", - "ZArrayMetadata", - "ZAttrsMetadata", - "ZGroupMetadata", + "ZarrV2ArrayDimensionSeparator", + "ZarrV2ArrayMetadataJSON", + "ZarrV2ArrayOrder", + "ZarrV2CodecMetadata", + "ZarrV2ConsolidatedMetadataJSON", + "ZarrV2DataTypeMetadata", + "ZarrV2GroupMetadataJSON", + "ZarrV2ZArrayJSON", + "ZarrV2ZAttrsJSON", + "ZarrV2ZGroupJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/array.py b/packages/zarr-metadata/src/zarr_metadata/v2/array.py index 999c341dc7..84b6446bcb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/array.py @@ -3,18 +3,27 @@ from collections.abc import Mapping from typing import Final, Literal, NotRequired -from typing_extensions import TypedDict +from typing_extensions import TypeAliasType, TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v2.codec import CodecMetadataV2 - -DataTypeMetadataV2 = str | tuple[tuple[str, str] | tuple[str, str, tuple[int, ...]], ...] +from zarr_metadata.v2.codec import ZarrV2CodecMetadata + +ZarrV2DataTypeMetadata = TypeAliasType( + "ZarrV2DataTypeMetadata", + str + | tuple[ + tuple[str, "ZarrV2DataTypeMetadata"] + | tuple[str, "ZarrV2DataTypeMetadata", tuple[int, ...]], + ..., + ], +) """The v2 dtype representation. Either a numpy-style dtype string (e.g. `"/.zarray` for a v2 array. User attributes live in a sibling `.zattrs` file and are - NOT part of this type; see `ZAttrsMetadata`. + NOT part of this type; see `ZarrV2ZAttrsJSON`. See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ @@ -60,15 +69,15 @@ class ZArrayMetadata(TypedDict): zarr_format: Literal[2] shape: tuple[int, ...] chunks: tuple[int, ...] - dtype: DataTypeMetadataV2 - compressor: CodecMetadataV2 | None + dtype: ZarrV2DataTypeMetadata + compressor: ZarrV2CodecMetadata | None fill_value: JSONValue - order: ArrayOrderV2 - filters: tuple[CodecMetadataV2, ...] | None - dimension_separator: NotRequired[ArrayDimensionSeparatorV2] + order: ZarrV2ArrayOrder + filters: tuple[ZarrV2CodecMetadata, ...] | None + dimension_separator: NotRequired[ZarrV2ArrayDimensionSeparator] -class ArrayMetadataV2(TypedDict): +class ZarrV2ArrayMetadataJSON(TypedDict): """ Zarr v2 array metadata document, in-memory merged form. @@ -78,7 +87,7 @@ class ArrayMetadataV2(TypedDict): `attributes` field so a single TypedDict represents the complete in-memory state of a v2 array node. Consumers that read or write a real `.zarray` file should split / merge `attributes` accordingly, - or use `ZArrayMetadata` (strict on-disk) plus `ZAttrsMetadata` directly. + or use `ZarrV2ZArrayJSON` (strict on-disk) plus `ZarrV2ZAttrsJSON` directly. See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ @@ -86,12 +95,12 @@ class ArrayMetadataV2(TypedDict): zarr_format: Literal[2] shape: tuple[int, ...] chunks: tuple[int, ...] - dtype: DataTypeMetadataV2 - compressor: CodecMetadataV2 | None + dtype: ZarrV2DataTypeMetadata + compressor: ZarrV2CodecMetadata | None fill_value: JSONValue - order: ArrayOrderV2 - filters: tuple[CodecMetadataV2, ...] | None - dimension_separator: NotRequired[ArrayDimensionSeparatorV2] + order: ZarrV2ArrayOrder + filters: tuple[ZarrV2CodecMetadata, ...] | None + dimension_separator: NotRequired[ZarrV2ArrayDimensionSeparator] attributes: NotRequired[Mapping[str, JSONValue]] """User attributes from the sibling `.zattrs` file (not part of `.zarray`). @@ -99,11 +108,11 @@ class ArrayMetadataV2(TypedDict): """ -class ArrayMetadataV2Partial(TypedDict, total=False): +class ZarrV2ArrayMetadataJSONPartial(TypedDict, total=False): """ - Partial form of `ArrayMetadataV2`: every field is `NotRequired`. + Partial form of `ZarrV2ArrayMetadataJSON`: every field is `NotRequired`. - Field annotations mirror `ArrayMetadataV2` exactly. The only difference is + Field annotations mirror `ZarrV2ArrayMetadataJSON` exactly. The only difference is `total=False`, which makes every key optional at the type level. Use this when typing dicts that intentionally hold a subset of a complete @@ -113,26 +122,26 @@ class ArrayMetadataV2Partial(TypedDict, total=False): The `NotRequired[...]` wrappers on `dimension_separator` and `attributes` are intentional: keeping them preserves byte-identical `__annotations__` - with `ArrayMetadataV2` so the `==` check in + with `ZarrV2ArrayMetadataJSON` so the `==` check in `tests/test_partial_equivalence.py` passes without special-casing those fields (PEP 655 explicitly permits `NotRequired` inside `total=False`). Note: v2 array metadata has no `extra_items` setting (the v2 spec has no extension-field concept), so this partial inherits the same closed shape. - Drift between this type and `ArrayMetadataV2` is prevented by + Drift between this type and `ZarrV2ArrayMetadataJSON` is prevented by `tests/test_partial_equivalence.py`. """ zarr_format: Literal[2] shape: tuple[int, ...] chunks: tuple[int, ...] - dtype: DataTypeMetadataV2 - compressor: CodecMetadataV2 | None + dtype: ZarrV2DataTypeMetadata + compressor: ZarrV2CodecMetadata | None fill_value: JSONValue - order: ArrayOrderV2 - filters: tuple[CodecMetadataV2, ...] | None - dimension_separator: NotRequired[ArrayDimensionSeparatorV2] + order: ZarrV2ArrayOrder + filters: tuple[ZarrV2CodecMetadata, ...] | None + dimension_separator: NotRequired[ZarrV2ArrayDimensionSeparator] attributes: NotRequired[Mapping[str, JSONValue]] """User attributes from the sibling `.zattrs` file (not part of `.zarray`). @@ -143,10 +152,10 @@ class ArrayMetadataV2Partial(TypedDict, total=False): __all__ = [ "ARRAY_DIMENSION_SEPARATOR_V2", "ARRAY_ORDER_V2", - "ArrayDimensionSeparatorV2", - "ArrayMetadataV2", - "ArrayMetadataV2Partial", - "ArrayOrderV2", - "DataTypeMetadataV2", - "ZArrayMetadata", + "ZarrV2ArrayDimensionSeparator", + "ZarrV2ArrayMetadataJSON", + "ZarrV2ArrayMetadataJSONPartial", + "ZarrV2ArrayOrder", + "ZarrV2DataTypeMetadata", + "ZarrV2ZArrayJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py b/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py index 18b8ded9da..f7cc31babe 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py @@ -7,16 +7,16 @@ from zarr_metadata._common import JSONValue -ZAttrsMetadata = Mapping[str, JSONValue] +ZarrV2ZAttrsJSON = Mapping[str, JSONValue] """On-disk `.zattrs` file content. A JSON object holding user-defined attributes for a v2 array or group. Spec-defined keys for arrays / groups live in sibling `.zarray` / `.zgroup` -files (modeled by `ZArrayMetadata` / `ZGroupMetadata`). This type does not +files (modeled by `ZarrV2ZArrayJSON` / `ZarrV2ZGroupJSON`). This type does not constrain the keys or values of the attributes mapping. """ __all__ = [ - "ZAttrsMetadata", + "ZarrV2ZAttrsJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/codec.py b/packages/zarr-metadata/src/zarr_metadata/v2/codec.py index 6d194b7e29..69125544e6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/codec.py @@ -10,7 +10,7 @@ from zarr_metadata._common import JSONValue -class CodecMetadataV2(TypedDict, extra_items=JSONValue): # type: ignore[call-arg] +class ZarrV2CodecMetadata(TypedDict, extra_items=JSONValue): """ A numcodecs configuration dict, used as a v2 compressor or filter. @@ -25,5 +25,5 @@ class CodecMetadataV2(TypedDict, extra_items=JSONValue): # type: ignore[call-ar __all__ = [ - "CodecMetadataV2", + "ZarrV2CodecMetadata", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py index 61a5527085..6b586bb92e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py @@ -10,12 +10,12 @@ from typing_extensions import TypedDict -from zarr_metadata.v2.array import ZArrayMetadata -from zarr_metadata.v2.attributes import ZAttrsMetadata -from zarr_metadata.v2.group import ZGroupMetadata +from zarr_metadata.v2.array import ZarrV2ZArrayJSON +from zarr_metadata.v2.attributes import ZarrV2ZAttrsJSON +from zarr_metadata.v2.group import ZarrV2ZGroupJSON -class ConsolidatedMetadataV2(TypedDict): +class ZarrV2ConsolidatedMetadataJSON(TypedDict): """ `.zmetadata` file contents. @@ -24,9 +24,9 @@ class ConsolidatedMetadataV2(TypedDict): that path. The keys include the filename suffix, not just the node path; the value's shape is determined by which file the key points at: - - `/.zarray` -> `ZArrayMetadata` - - `/.zgroup` -> `ZGroupMetadata` - - `/.zattrs` -> `ZAttrsMetadata` + - `/.zarray` -> `ZarrV2ZArrayJSON` + - `/.zgroup` -> `ZarrV2ZGroupJSON` + - `/.zattrs` -> `ZarrV2ZAttrsJSON` The TypedDict cannot discriminate the value shape on the key suffix at the type level; consumers should narrow at runtime by inspecting @@ -34,9 +34,9 @@ class ConsolidatedMetadataV2(TypedDict): """ zarr_consolidated_format: int - metadata: Mapping[str, ZArrayMetadata | ZGroupMetadata | ZAttrsMetadata] + metadata: Mapping[str, ZarrV2ZArrayJSON | ZarrV2ZGroupJSON | ZarrV2ZAttrsJSON] __all__ = [ - "ConsolidatedMetadataV2", + "ZarrV2ConsolidatedMetadataJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/group.py b/packages/zarr-metadata/src/zarr_metadata/v2/group.py index 5f456fe8d3..50f2482e6f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/group.py @@ -11,14 +11,14 @@ from zarr_metadata._common import JSONValue -class ZGroupMetadata(TypedDict): +class ZarrV2ZGroupJSON(TypedDict): """ On-disk `.zgroup` file content. Strict shape of the JSON document persisted at `/.zgroup` for a v2 group. The spec defines exactly one field. User attributes live in a sibling `.zattrs` file and are NOT part of this type; see - `ZAttrsMetadata`. + `ZarrV2ZAttrsJSON`. See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ @@ -26,7 +26,7 @@ class ZGroupMetadata(TypedDict): zarr_format: Literal[2] -class GroupMetadataV2(TypedDict): +class ZarrV2GroupMetadataJSON(TypedDict): """ Zarr v2 group metadata document, in-memory merged form. @@ -34,8 +34,8 @@ class GroupMetadataV2(TypedDict): and `.zattrs` (user attributes). On disk these are persisted as two separate files; this type folds them so a single TypedDict represents the complete in-memory state of a v2 group node. Consumers that read - or write the real on-disk files should use `ZGroupMetadata` (strict - `.zgroup`) plus `ZAttrsMetadata` directly. + or write the real on-disk files should use `ZarrV2ZGroupJSON` (strict + `.zgroup`) plus `ZarrV2ZAttrsJSON` directly. See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ @@ -44,11 +44,11 @@ class GroupMetadataV2(TypedDict): attributes: NotRequired[Mapping[str, JSONValue]] -class GroupMetadataV2Partial(TypedDict, total=False): +class ZarrV2GroupMetadataJSONPartial(TypedDict, total=False): """ - Partial form of `GroupMetadataV2`: every field is `NotRequired`. + Partial form of `ZarrV2GroupMetadataJSON`: every field is `NotRequired`. - Field annotations mirror `GroupMetadataV2` exactly. The only difference is + Field annotations mirror `ZarrV2GroupMetadataJSON` exactly. The only difference is `total=False`, which makes every key optional at the type level. Use this when typing dicts that intentionally hold a subset of a complete @@ -58,7 +58,7 @@ class GroupMetadataV2Partial(TypedDict, total=False): `*Partial` types; the practical effect is that `zarr_format` becomes optional. The `NotRequired[...]` wrapper on `attributes` is intentional: keeping it - preserves byte-identical `__annotations__` with `GroupMetadataV2` so the + preserves byte-identical `__annotations__` with `ZarrV2GroupMetadataJSON` so the `==` check in `tests/test_partial_equivalence.py` passes without special-casing that field (PEP 655 explicitly permits `NotRequired` inside `total=False`). @@ -66,7 +66,7 @@ class GroupMetadataV2Partial(TypedDict, total=False): Note: v2 group metadata has no `extra_items` setting (the v2 spec has no extension-field concept), so this partial inherits the same closed shape. - Drift between this type and `GroupMetadataV2` is prevented by + Drift between this type and `ZarrV2GroupMetadataJSON` is prevented by `tests/test_partial_equivalence.py`. """ @@ -75,7 +75,7 @@ class GroupMetadataV2Partial(TypedDict, total=False): __all__ = [ - "GroupMetadataV2", - "GroupMetadataV2Partial", - "ZGroupMetadata", + "ZarrV2GroupMetadataJSON", + "ZarrV2GroupMetadataJSONPartial", + "ZarrV2ZGroupJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v3/__init__.py index c897f20d52..4e335f9573 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/__init__.py @@ -1,14 +1,14 @@ """Zarr v3 metadata types.""" -from zarr_metadata.v3._common import MetadataV3 -from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 -from zarr_metadata.v3.consolidated import ConsolidatedMetadataV3 -from zarr_metadata.v3.group import GroupMetadataV3 +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON, ZarrV3ExtensionField +from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON __all__ = [ - "ArrayMetadataV3", - "ConsolidatedMetadataV3", - "ExtensionFieldV3", - "GroupMetadataV3", - "MetadataV3", + "ZarrV3ArrayMetadataJSON", + "ZarrV3ConsolidatedMetadataJSON", + "ZarrV3ExtensionField", + "ZarrV3GroupMetadataJSON", + "ZarrV3MetadataFieldJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_common.py b/packages/zarr-metadata/src/zarr_metadata/v3/_common.py index 3424587a43..406b76b723 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_common.py @@ -2,14 +2,14 @@ This module is private (underscore-prefixed) and exists to avoid circular imports between leaf modules and sub-package `__init__.py` re-exports. -Public consumers should import `MetadataV3` from `zarr_metadata.v3`. +Public consumers should import `ZarrV3MetadataFieldJSON` from `zarr_metadata.v3`. """ -from zarr_metadata._common import NamedConfigV3 +from zarr_metadata._common import ZarrV3NamedConfigJSON -MetadataV3 = str | NamedConfigV3 +ZarrV3MetadataFieldJSON = str | ZarrV3NamedConfigJSON """The JSON shape of any v3 metadata extension-point entry: either a bare -short-hand name string or a `{name, configuration}` envelope. +short-hand name string or a `{name, configuration, must_understand}` envelope. Used for `data_type`, `chunk_grid`, `chunk_key_encoding`, individual codec entries, and `storage_transformers` in v3 array metadata, and for @@ -19,5 +19,5 @@ __all__ = [ - "MetadataV3", + "ZarrV3MetadataFieldJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/array.py b/packages/zarr-metadata/src/zarr_metadata/v3/array.py index a8b0fa3358..96341f73ca 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/array.py @@ -1,73 +1,49 @@ """Zarr v3 array metadata types.""" from collections.abc import Mapping -from typing import Literal, NotRequired +from typing import Literal, NotRequired, TypeAlias from typing_extensions import TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v3._common import MetadataV3 +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +ZarrV3ExtensionField: TypeAlias = JSONValue +"""The JSON value of an unknown top-level v3 metadata field. -class ExtensionFieldV3(TypedDict, extra_items=JSONValue): # type: ignore[call-arg] - """ - Required shape of any extension field on a v3 metadata document. - - The Zarr v3 spec permits extra keys on array and group metadata - documents, provided each value is an object with a `must_understand` - boolean key. This TypedDict captures that constraint and is used as - the `extra_items=` parameter on `ArrayMetadataV3` and `GroupMetadataV3`. - - `must_understand` is typed as `bool` rather than `Literal[False]` so - that applications which understand a particular extension can produce - or consume it with `must_understand: true` (signalling that readers - that don't recognize the extension MUST refuse to open the document). - The common case is still `false`, signalling that unknown readers may - safely ignore the field. - - Spec interpretation: this type follows the original Zarr v3.0 reading - of the spec, under which any object with a `must_understand` key is a - valid extension field. The v3.1 spec rewrite added language requiring - extension fields to also include a `name: str` key (the "Extension - definition" form). Under the strict v3.1 reading, real-world extension - fields written by zarr-python and zarrs (notably `consolidated_metadata`, - which has no `name` field) are out of spec. The community consensus at - the time of writing is that this is a regression to be reverted; this - package models the v3.0 / pre-revert interpretation. See - https://github.com/zarr-developers/zarr-specs/issues/371 for the - ongoing discussion. - """ - - must_understand: bool +An object carrying the literal member `must_understand: false` may be ignored. +Every other JSON shape implicitly requires understanding; recognition itself +belongs to the reader rather than this structural type. +""" -class ArrayMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=ZarrV3ExtensionField): """ Zarr v3 array metadata document (the `zarr.json` content for an array). - Extra keys are permitted if they conform to `ExtensionFieldV3`. + Extra keys may contain arbitrary JSON values. See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#array-metadata """ zarr_format: Literal[3] node_type: Literal["array"] - data_type: MetadataV3 + data_type: ZarrV3MetadataFieldJSON shape: tuple[int, ...] - chunk_grid: MetadataV3 - chunk_key_encoding: MetadataV3 + chunk_grid: ZarrV3MetadataFieldJSON + chunk_key_encoding: ZarrV3MetadataFieldJSON fill_value: JSONValue - codecs: tuple[MetadataV3, ...] + codecs: tuple[ZarrV3MetadataFieldJSON, ...] attributes: NotRequired[Mapping[str, JSONValue]] - storage_transformers: NotRequired[tuple[MetadataV3, ...]] + storage_transformers: NotRequired[tuple[ZarrV3MetadataFieldJSON, ...]] dimension_names: NotRequired[tuple[str | None, ...]] -class ArrayMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class ZarrV3ArrayMetadataJSONPartial(TypedDict, total=False, extra_items=ZarrV3ExtensionField): """ - Partial form of `ArrayMetadataV3`: every field is `NotRequired`. + Partial form of `ZarrV3ArrayMetadataJSON`: every field is `NotRequired`. - Field annotations and `extra_items=` mirror `ArrayMetadataV3` exactly. + Field annotations and `extra_items=` mirror `ZarrV3ArrayMetadataJSON` exactly. The only difference is `total=False`, which makes every key optional at the type level. @@ -78,29 +54,29 @@ class ArrayMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV The `NotRequired[...]` wrappers on `attributes`, `storage_transformers`, and `dimension_names` are intentional: keeping them preserves byte-identical - `__annotations__` with `ArrayMetadataV3` so the `==` check in + `__annotations__` with `ZarrV3ArrayMetadataJSON` so the `==` check in `tests/test_partial_equivalence.py` passes without special-casing those fields (PEP 655 explicitly permits `NotRequired` inside `total=False`). - Drift between this type and `ArrayMetadataV3` is prevented by + Drift between this type and `ZarrV3ArrayMetadataJSON` is prevented by `tests/test_partial_equivalence.py`. """ zarr_format: Literal[3] node_type: Literal["array"] - data_type: MetadataV3 + data_type: ZarrV3MetadataFieldJSON shape: tuple[int, ...] - chunk_grid: MetadataV3 - chunk_key_encoding: MetadataV3 + chunk_grid: ZarrV3MetadataFieldJSON + chunk_key_encoding: ZarrV3MetadataFieldJSON fill_value: JSONValue - codecs: tuple[MetadataV3, ...] + codecs: tuple[ZarrV3MetadataFieldJSON, ...] attributes: NotRequired[Mapping[str, JSONValue]] - storage_transformers: NotRequired[tuple[MetadataV3, ...]] + storage_transformers: NotRequired[tuple[ZarrV3MetadataFieldJSON, ...]] dimension_names: NotRequired[tuple[str | None, ...]] __all__ = [ - "ArrayMetadataV3", - "ArrayMetadataV3Partial", - "ExtensionFieldV3", + "ZarrV3ArrayMetadataJSON", + "ZarrV3ArrayMetadataJSONPartial", + "ZarrV3ExtensionField", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index fef5793626..e2783d296d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -4,6 +4,12 @@ Intended only to allow existing v2 arrays to be converted to v3 without having to rename chunks. Not recommended for new arrays. +Naming note: these are Zarr **v3** types. The leading `V2` in +`V2ChunkKeyEncodingMetadata` (and friends) is the encoding's registered +*entity name* (`"v2"`), not the format-version marker that `ZarrV2...` +names carry — this package's version-prefixed names always spell it +`ZarrV2` / `ZarrV3`. + See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-key-encoding """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py index b4f357117f..c8a9a150fc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py @@ -11,7 +11,7 @@ `CodecConfiguration`, etc., import directly from the leaf submodule. For the field-level "any codec entry" alias (used in array metadata's -`codecs` list and in sharding's inner pipelines), import `MetadataV3` +`codecs` list and in sharding's inner pipelines), import `ZarrV3MetadataFieldJSON` from `zarr_metadata.v3`. See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/index.html diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 7e9b071669..96c39e5916 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -9,7 +9,7 @@ from typing_extensions import TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v3._common import MetadataV3 +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON CAST_VALUE_CODEC_NAME: Final = "cast_value" """The `name` field value of the `cast_value` codec.""" @@ -71,7 +71,7 @@ class CastValueCodecConfiguration(TypedDict): bare-string primitive name or a `{name, configuration}` envelope. """ - data_type: MetadataV3 + data_type: ZarrV3MetadataFieldJSON rounding: NotRequired[CastRoundingMode] out_of_range: NotRequired[CastOutOfRangeMode] scalar_map: NotRequired[ScalarMap] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index ea35ae5f1d..6b9b46c43d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -18,7 +18,7 @@ """Literal type of the `name` field of the `crc32c` codec.""" -class Empty(TypedDict, closed=True): # type: ignore[call-arg] +class Empty(TypedDict, closed=True): """An empty mapping""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index a1488f7c30..a8c9247ec4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -8,7 +8,7 @@ from typing_extensions import TypedDict -from zarr_metadata.v3._common import MetadataV3 +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON SHARDING_INDEXED_CODEC_NAME: Final = "sharding_indexed" """The `name` field value of the `sharding_indexed` codec.""" @@ -40,8 +40,8 @@ class ShardingIndexedCodecConfiguration(TypedDict): """ chunk_shape: tuple[int, ...] - codecs: tuple[MetadataV3, ...] - index_codecs: tuple[MetadataV3, ...] + codecs: tuple[ZarrV3MetadataFieldJSON, ...] + index_codecs: tuple[ZarrV3MetadataFieldJSON, ...] index_location: NotRequired[ShardingIndexLocation] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py index 486a0897a5..bcbe675947 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py @@ -5,14 +5,10 @@ implementation (and zarrs), where consolidated metadata is embedded as an extension field on a group's `zarr.json`. -The shape modeled here (`{kind, must_understand, metadata}` with no `name` -field) reflects the original Zarr v3.0 reading of the extension-field -rules. Under the strict Zarr v3.1 reading, every extension field must -also include a `name: str` key, which would make this shape — and every -real-world consolidated metadata document in the wild — out of spec. -See `ExtensionFieldV3` and -https://github.com/zarr-developers/zarr-specs/issues/371 for the -ongoing discussion. +This is a known non-core interoperability extension. Its +`{kind, must_understand, metadata}` payload is an unknown top-level JSON value +to the core document model; implementations that recognize the convention may +interpret it through this dedicated type. """ from collections.abc import Mapping @@ -20,24 +16,24 @@ from typing_extensions import TypedDict -from zarr_metadata.v3.array import ArrayMetadataV3 -from zarr_metadata.v3.group import GroupMetadataV3 +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON -class ConsolidatedMetadataV3(TypedDict): +class ZarrV3ConsolidatedMetadataJSON(TypedDict): """ Inline consolidated metadata embedded in a v3 group. - The `metadata` map contains only v3 array and group entries - v2 - entries are excluded by design. Mixing v2 entries into a v3 - consolidated metadata document is invalid per spec. + The `metadata` map contains only v3 array and group entries. V2 entries + are excluded from this interoperability convention by design; the v3 core + specification does not define consolidated metadata. """ kind: Literal["inline"] must_understand: Literal[False] - metadata: Mapping[str, ArrayMetadataV3 | GroupMetadataV3] + metadata: Mapping[str, ZarrV3ArrayMetadataJSON | ZarrV3GroupMetadataJSON] __all__ = [ - "ConsolidatedMetadataV3", + "ZarrV3ConsolidatedMetadataJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 5291e5c309..b1b6b50308 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -10,7 +10,7 @@ from typing_extensions import ReadOnly, TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v3._common import MetadataV3 +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON STRUCT_DATA_TYPE_NAME: Final = "struct" """The `name` field value of the `struct` data type.""" @@ -33,7 +33,7 @@ class StructField(TypedDict): """ name: ReadOnly[str] - data_type: ReadOnly[MetadataV3] + data_type: ReadOnly[ZarrV3MetadataFieldJSON] class StructConfiguration(TypedDict): diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/group.py b/packages/zarr-metadata/src/zarr_metadata/v3/group.py index 27186b6059..033e91ff8c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/group.py @@ -9,14 +9,14 @@ from typing_extensions import TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v3.array import ExtensionFieldV3 +from zarr_metadata.v3.array import ZarrV3ExtensionField -class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class ZarrV3GroupMetadataJSON(TypedDict, extra_items=ZarrV3ExtensionField): """ Zarr v3 group metadata document (the `zarr.json` content for a group). - Extra keys are permitted if they conform to `ExtensionFieldV3`. + Extra keys may contain arbitrary JSON values. See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#group-metadata """ @@ -26,11 +26,11 @@ class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[ attributes: NotRequired[Mapping[str, JSONValue]] -class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class ZarrV3GroupMetadataJSONPartial(TypedDict, total=False, extra_items=ZarrV3ExtensionField): """ - Partial form of `GroupMetadataV3`: every field is `NotRequired`. + Partial form of `ZarrV3GroupMetadataJSON`: every field is `NotRequired`. - Field annotations and `extra_items=` mirror `GroupMetadataV3` exactly. + Field annotations and `extra_items=` mirror `ZarrV3GroupMetadataJSON` exactly. The only difference is `total=False`, which makes every key optional at the type level. @@ -40,12 +40,12 @@ class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV into a complete document elsewhere. The `NotRequired[...]` wrapper on `attributes` is intentional: keeping it - preserves byte-identical `__annotations__` with `GroupMetadataV3` so the + preserves byte-identical `__annotations__` with `ZarrV3GroupMetadataJSON` so the `==` check in `tests/test_partial_equivalence.py` passes without special-casing that field (PEP 655 explicitly permits `NotRequired` inside `total=False`). - Drift between this type and `GroupMetadataV3` is prevented by + Drift between this type and `ZarrV3GroupMetadataJSON` is prevented by `tests/test_partial_equivalence.py`. """ @@ -55,6 +55,6 @@ class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV __all__ = [ - "GroupMetadataV3", - "GroupMetadataV3Partial", + "ZarrV3GroupMetadataJSON", + "ZarrV3GroupMetadataJSONPartial", ] diff --git a/packages/zarr-metadata/tests/model/__init__.py b/packages/zarr-metadata/tests/model/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/zarr-metadata/tests/model/_cases.py b/packages/zarr-metadata/tests/model/_cases.py new file mode 100644 index 0000000000..15faa65539 --- /dev/null +++ b/packages/zarr-metadata/tests/model/_cases.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING, Generic, TypeVar + +import pytest + +if TYPE_CHECKING: + from contextlib import AbstractContextManager + +TIn = TypeVar("TIn") +TOut = TypeVar("TOut") + + +@dataclass(frozen=True) +class Expect(Generic[TIn, TOut]): + """A test case with explicit input, expected output, and a human-readable id.""" + + input: TIn + output: TOut + id: str + + +@dataclass(frozen=True) +class ExpectFail(Generic[TIn]): + """A test case that should raise an exception. + + `msg` is a regex matched against the exception text (pytest's native + `match=` semantics). Leave it `None` to assert only the exception type. Set + `escape=True` when `msg` is a literal that contains regex metacharacters + such as `(`, `[`, or `.`; `escape` has no effect when `msg` is `None`. + """ + + input: TIn + exception: type[Exception] + id: str + msg: str | None = None + escape: bool = False + + def raises(self) -> AbstractContextManager[pytest.ExceptionInfo[Exception]]: + if self.msg is None: + return pytest.raises(self.exception) + pattern = re.escape(self.msg) if self.escape else self.msg + return pytest.raises(self.exception, match=pattern) + + +def mutate_nested_containers(value: object) -> None: + """Recursively mutate every mutable container reachable inside `value`. + + Adds a marker key to every dict and appends a marker to every list, + descending through tuples. Used to prove a `to_json` document shares no + mutable state with the model that produced it. + """ + if isinstance(value, dict): + for item in value.values(): + mutate_nested_containers(item) + value["__mutated__"] = "__mutated__" + elif isinstance(value, list): + for item in value: + mutate_nested_containers(item) + value.append("__mutated__") + elif isinstance(value, tuple): + for item in value: + mutate_nested_containers(item) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py new file mode 100644 index 0000000000..467ef1e2ad --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -0,0 +1,1738 @@ +"""Tests for the metadata models in ``zarr_metadata.model``.""" + +import copy +import dataclasses +import json +from collections import UserDict +from collections.abc import Callable +from typing import TYPE_CHECKING, get_args + +import pytest +from typing_extensions import Unpack + +from tests.model._cases import Expect, ExpectFail, mutate_nested_containers +from zarr_metadata.model import ( + ARRAY_METADATA_OPTIONAL_KEYS_V3, + ARRAY_METADATA_REQUIRED_KEYS_V3, + ARRAY_METADATA_STANDARD_KEYS_V3, + UNSET, + MetadataValidationError, + ValidationProblem, + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadataPartial, + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadataPartial, + ZarrV3MetadataField, + ZarrV3NamedConfig, + is_array_metadata_v2, + is_array_metadata_v3, + is_json, + is_metadata_field_v3, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_json, + parse_metadata_field_v3, + validate_array_metadata_v2, + validate_array_metadata_v3, + validate_json, + validate_metadata_field_v3, +) +from zarr_metadata.model._validation import _prefix, arrays_to_tuples + +if TYPE_CHECKING: + from zarr_metadata._common import JSONValue + from zarr_metadata.v2 import ZarrV2CodecMetadata + +# --- public exports -------------------------------------------------------- + + +def test_guards_exported_from_package() -> None: + """The wire-type guard/parser functions are exported from the package.""" + import zarr_metadata.model + + for name in ( + "is_json", + "parse_json", + "is_metadata_field_v3", + "parse_metadata_field_v3", + "is_array_metadata_v3", + "parse_array_metadata_v3", + "is_array_metadata_v2", + "parse_array_metadata_v2", + ): + assert name in zarr_metadata.model.__all__ + assert hasattr(zarr_metadata.model, name) + + +def test_store_key_pairs_exported_from_package() -> None: + """Each store-key constant is exported together with its Literal type + alias, and the pair cannot drift apart.""" + import zarr_metadata.model as m + + pairs = [ + ("ARRAY_METADATA_STORE_KEY_V2", "ZarrV2ArrayMetadataStoreKey"), + ("ARRAY_METADATA_STORE_KEY_V3", "ZarrV3ArrayMetadataStoreKey"), + ("ATTRIBUTES_STORE_KEY_V2", "ZarrV2AttributesStoreKey"), + ("GROUP_METADATA_STORE_KEY_V2", "ZarrV2GroupMetadataStoreKey"), + ("GROUP_METADATA_STORE_KEY_V3", "ZarrV3GroupMetadataStoreKey"), + ("CONSOLIDATED_METADATA_STORE_KEY_V2", "ZarrV2ConsolidatedMetadataStoreKey"), + ] + for const_name, alias_name in pairs: + assert const_name in m.__all__ + assert alias_name in m.__all__ + assert (getattr(m, const_name),) == get_args(getattr(m, alias_name)) + + +def test_validation_diagnostics_exported_from_package() -> None: + """The validation-diagnostic types and validators are exported from the package.""" + import zarr_metadata.model + + for name in ( + "ValidationProblem", + "MetadataValidationError", + "validate_json", + "validate_metadata_field_v3", + "validate_array_metadata_v3", + "validate_array_metadata_v2", + ): + assert name in zarr_metadata.model.__all__ + assert hasattr(zarr_metadata.model, name) + + +def test_expect_expectfail_smoke() -> None: + """The Expect/ExpectFail test-case dataclasses behave as expected.""" + e = Expect(input=1, output=2, id="x") + assert (e.input, e.output, e.id) == (1, 2, "x") + f = ExpectFail(input=1, exception=ValueError, id="y", msg="boom") + with f.raises(): + raise ValueError("boom") + + +def test_v3_from_json_error_lists_all_problems() -> None: + """A malformed v3 document surfaces every problem via MetadataValidationError.problems.""" + doc: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) + del doc["shape"] + doc["data_type"] = 5 + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV3ArrayMetadata.from_json(doc) + locs = {p.loc for p in exc_info.value.problems} + assert ("shape",) in locs + assert ("data_type",) in locs + + +# --- JSON type / fill_value contract --------------------------------------- + + +def test_json_value_type_accepts_json_shapes() -> None: + # JSONValue is the package's public JSON type alias; assigning JSON-shaped + # values to it is valid. + """The JSONValue type alias accepts JSON-shaped values.""" + value: JSONValue = {"a": [1, 2.0, "x", True, None]} + assert value == {"a": [1, 2.0, "x", True, None]} + + +def test_string_nan_fill_value_roundtrips() -> None: + # Non-finite floats are represented as the spec strings ("NaN", "Infinity", + # "-Infinity") by the caller — the metadata layer does not interpret dtypes. + # The string form round-trips cleanly under default dataclass equality, + # unlike a raw float('nan') (which is an invalid fill_value the caller must + # not pass). + """A string 'NaN' fill_value round-trips cleanly (non-finite floats are the caller's responsibility).""" + m = ZarrV3ArrayMetadata.create_default(fill_value="NaN") + assert ZarrV3ArrayMetadata.from_json(m.to_json()) == m + assert ZarrV3ArrayMetadata.from_json(m.to_json()).fill_value == "NaN" + + +# --- ZarrV3NamedConfig.to_json ------------------------------------------------ + +ZARR_TO_JSON_CASES = [ + Expect( + ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": [1]}), + {"name": "regular", "configuration": {"chunk_shape": [1]}}, + id="with-configuration", + ), + Expect( + ZarrV3NamedConfig(name="bytes", configuration={}), + "bytes", + id="empty-configuration-shorthand", + ), +] + + +@pytest.mark.parametrize("case", ZARR_TO_JSON_CASES, ids=lambda c: c.id) +def test_zarr_metadata_v3_to_json(case: Expect[ZarrV3NamedConfig, object]) -> None: + """ZarrV3NamedConfig.to_json emits the canonical extension form.""" + assert case.input.to_json() == case.output + + +def test_zarr_metadata_v3_to_json_preserves_false_obligation() -> None: + """An empty optional extension stays an object so false is not lost.""" + model = ZarrV3NamedConfig(name="optional", configuration={}, must_understand=False) + assert model.to_json() == {"name": "optional", "must_understand": False} + + +# --- ZarrV3NamedConfig.from_json ----------------------------------------------- + +ZARR_FROM_JSON_CASES = [ + Expect("bytes", ZarrV3NamedConfig(name="bytes", configuration={}), id="bare-string"), + Expect( + {"name": "regular", "configuration": {"chunk_shape": [1]}}, + ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": (1,)}), + id="object-with-config", + ), + Expect( + {"name": "bytes"}, + ZarrV3NamedConfig(name="bytes", configuration={}), + id="object-without-config", + ), +] + + +@pytest.mark.parametrize("case", ZARR_FROM_JSON_CASES, ids=lambda c: c.id) +def test_zarr_metadata_v3_from_json(case: Expect[object, ZarrV3NamedConfig]) -> None: + """ZarrV3NamedConfig.from_json parses both the bare-string and object forms.""" + assert ZarrV3NamedConfig.from_json(case.input) == case.output + + +def test_zarr_metadata_v3_from_json_preserves_false_obligation() -> None: + """Explicit false is represented on the normalized model.""" + model = ZarrV3NamedConfig.from_json({"name": "optional", "must_understand": False}) + assert model.must_understand is False + + +# --- V3 baseline ----------------------------------------------------------- + + +def test_v3_to_json_emits_canonical_document() -> None: + """V3 to_json emits exactly the expected document (which covers every + spec-required key by construction).""" + out = ZarrV3ArrayMetadata.create_default( + shape=(10,), data_type=ZarrV3NamedConfig(name="int32", configuration={}) + ).to_json() + assert out == { + "zarr_format": 3, + "node_type": "array", + "shape": (10,), + "fill_value": 0, + "data_type": "int32", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (10,)}}, + "codecs": ("bytes",), + "chunk_key_encoding": "default", + } + + +def test_v3_dimension_names_included_when_present() -> None: + """V3 to_json includes dimension_names when they are set.""" + out: dict[str, object] = dict( + ZarrV3ArrayMetadata.create_default(dimension_names=("x",)).to_json() + ) + assert out["dimension_names"] == ("x",) + + +def test_v3_dimension_names_omitted_when_none() -> None: + """V3 to_json omits dimension_names when they are UNSET.""" + out = ZarrV3ArrayMetadata.create_default(dimension_names=UNSET).to_json() + assert "dimension_names" not in out + + +# --- BUG 1: attributes gated on dimension_names ---------------------------- + + +def test_v3_attributes_included_when_dimension_names_is_none() -> None: + """Attributes must be emitted regardless of dimension_names. + + Regression: attributes were gated on ``dimension_names is not None``, + so non-empty attributes were silently dropped when there were no + dimension names. + """ + out: dict[str, object] = dict( + ZarrV3ArrayMetadata.create_default( + dimension_names=UNSET, attributes={"foo": "bar"} + ).to_json() + ) + assert out["attributes"] == {"foo": "bar"} + + +# --- BUG 2: single storage transformer dropped ----------------------------- + + +def test_v3_single_storage_transformer_included() -> None: + """A single storage transformer must be emitted. + + Regression: the guard used ``> 1`` instead of ``> 0``, dropping a + lone storage transformer. + """ + st = ZarrV3NamedConfig(name="some_transformer", configuration={}) + out: dict[str, object] = dict( + ZarrV3ArrayMetadata.create_default(storage_transformers=(st,)).to_json() + ) + assert out["storage_transformers"] == ("some_transformer",) + + +def test_v3_no_storage_transformers_omitted() -> None: + """V3 to_json omits storage_transformers when empty.""" + out = ZarrV3ArrayMetadata.create_default(storage_transformers=()).to_json() + assert "storage_transformers" not in out + + +# --- V3 extra fields ------------------------------------------------------- + + +def test_v3_extra_fields_merged() -> None: + """V3 to_json merges extra_fields into the top-level document.""" + out = ZarrV3ArrayMetadata.create_default( + extra_fields={"my_ext": {"must_understand": False}} + ).to_json() + assert out["my_ext"] == {"must_understand": False} + + +def test_v3_extra_fields_overlapping_standard_field_rejected() -> None: + """Constructing a V3 model with an extra field that collides with a standard key is rejected.""" + with pytest.raises(ValueError): + ZarrV3ArrayMetadata.create_default(extra_fields={"shape": {"must_understand": False}}) + + +# --- V3 key/value ---------------------------------------------------------- + + +def test_v3_to_key_value_is_valid_json_under_zarr_json() -> None: + """V3 to_key_value produces valid JSON bytes under the zarr.json key.""" + kv = ZarrV3ArrayMetadata.create_default(attributes={"a": 1}).to_key_value() + assert set(kv) == {"zarr.json"} + parsed = json.loads(kv["zarr.json"].decode("utf-8")) + assert parsed["zarr_format"] == 3 + assert parsed["attributes"] == {"a": 1} + + +# --- V3 standard-key sets -------------------------------------------------- + + +def test_standard_keys_is_union_of_required_and_optional() -> None: + """The standard-key set is the union of the required and optional key sets.""" + assert ( + ARRAY_METADATA_STANDARD_KEYS_V3 + == ARRAY_METADATA_REQUIRED_KEYS_V3 | ARRAY_METADATA_OPTIONAL_KEYS_V3 + ) + + +def test_standard_keys_contains_known_fields_and_excludes_extensions() -> None: + """The standard-key set contains known fields and excludes extension keys.""" + assert { + "zarr_format", + "node_type", + "shape", + "codecs", + } <= ARRAY_METADATA_STANDARD_KEYS_V3 + assert "my_ext" not in ARRAY_METADATA_STANDARD_KEYS_V3 + + +# --- create_default -------------------------------------------------------- + + +def test_v3_create_default_is_valid_empty_array() -> None: + """V3 create_default builds a structurally valid empty array that round-trips.""" + m = ZarrV3ArrayMetadata.create_default() + assert m.shape == () + assert m.data_type == ZarrV3NamedConfig(name="uint8", configuration={}) + assert m.fill_value == 0 + assert m.attributes == {} + assert m.extra_fields == {} + # the default document is structurally valid and round-trips + assert validate_array_metadata_v3(m.to_json()) == [] + assert ZarrV3ArrayMetadata.from_json(m.to_json()) == m + + +def test_v3_create_default_applies_overrides() -> None: + """V3 create_default applies keyword overrides over the defaults.""" + m = ZarrV3ArrayMetadata.create_default(shape=(4, 4), attributes={"a": 1}) + assert m.shape == (4, 4) + assert m.attributes == {"a": 1} + # un-overridden fields keep their defaults + assert m.data_type == ZarrV3NamedConfig(name="uint8", configuration={}) + + +def test_v2_create_default_is_valid_empty_array() -> None: + """V2 create_default builds a structurally valid empty array that round-trips.""" + m = ZarrV2ArrayMetadata.create_default() + assert m.shape == () + assert m.chunks == () + assert m.fill_value == 0 + assert m.compressor is None + assert m.filters is None + assert m.attributes is UNSET + assert validate_array_metadata_v2(m.to_json()) == [] + assert ZarrV2ArrayMetadata.from_json(m.to_json()) == m + + +def test_v2_create_default_applies_overrides() -> None: + """V2 create_default applies keyword overrides over the defaults.""" + m = ZarrV2ArrayMetadata.create_default(shape=(8,), attributes={"k": "v"}) + assert m.shape == (8,) + assert m.attributes == {"k": "v"} + assert m.dtype == "|u1" # default dtype unchanged + + +# --- V3 update ------------------------------------------------------------- + +# Cluster 3: update same-shape pairs across versions — parametrized + +UPDATE_NEW_INSTANCE_PARAMS = [ + pytest.param(ZarrV3ArrayMetadata, id="v3"), + pytest.param(ZarrV2ArrayMetadata, id="v2"), +] + + +@pytest.mark.parametrize("model_cls", UPDATE_NEW_INSTANCE_PARAMS) +def test_update_returns_new_instance( + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], +) -> None: + """update returns a new instance with the field replaced, leaving the original unchanged.""" + base = model_cls.create_default(shape=(10,)) + updated = base.update(shape=(20,)) + assert updated.shape == (20,) + assert base.shape == (10,) # original unchanged + assert isinstance(updated, model_cls) + + +UPDATE_NO_ARGS_PARAMS = [ + pytest.param(ZarrV3ArrayMetadata, id="v3"), + pytest.param(ZarrV2ArrayMetadata, id="v2"), +] + + +@pytest.mark.parametrize("model_cls", UPDATE_NO_ARGS_PARAMS) +def test_update_no_args_returns_equal_model( + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], +) -> None: + """update with no arguments returns a model equal to the original.""" + base = model_cls.create_default() + assert base.update() == base + + +# V3-only update tests — kept direct (extra_fields is v3-specific) + + +def test_update_can_replace_extra_fields() -> None: + """update can replace the extra_fields mapping.""" + base = ZarrV3ArrayMetadata.create_default(extra_fields={}) + updated = base.update(extra_fields={"my_ext": {"must_understand": False}}) + assert updated.extra_fields == {"my_ext": {"must_understand": False}} + + +def test_update_replaces_extra_fields_rather_than_merging() -> None: + """update replaces extra_fields wholesale rather than merging.""" + base = ZarrV3ArrayMetadata.create_default(extra_fields={"a": {"must_understand": False}}) + updated = base.update(extra_fields={"b": {"must_understand": True}}) + assert updated.extra_fields == {"b": {"must_understand": True}} + + +def test_partial_keys_match_settable_model_fields() -> None: + """The partial TypedDict must list exactly the constructor-settable fields. + + Guards against drift: adding/removing a settable field on the model + without updating ``ZarrV3ArrayMetadataPartial`` fails here. + """ + settable = {f.name for f in dataclasses.fields(ZarrV3ArrayMetadata) if f.init} + assert set(ZarrV3ArrayMetadataPartial.__annotations__) == settable + + +# --- V2 model -------------------------------------------------------------- + + +def test_v2_partial_keys_match_settable_model_fields() -> None: + """The v2 partial TypedDict must list exactly the settable fields.""" + settable = {f.name for f in dataclasses.fields(ZarrV2ArrayMetadata) if f.init} + assert set(ZarrV2ArrayMetadataPartial.__annotations__) == settable + + +def test_v2_to_key_value_splits_zarray_and_zattrs() -> None: + """V2 to_key_value splits the document into .zarray and .zattrs.""" + kv = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_key_value() + assert set(kv) == {".zarray", ".zattrs"} + zarray = json.loads(kv[".zarray"].decode("utf-8")) + zattrs = json.loads(kv[".zattrs"].decode("utf-8")) + assert zarray["zarr_format"] == 2 + assert zattrs == {"a": 1} + + +def test_v2_zarray_excludes_attributes() -> None: + """The on-disk ``.zarray`` document must not contain user attributes. + + In v2, attributes live only in the sibling ``.zattrs`` file. The bundled + ``ZarrV2ArrayMetadataJSON`` / ``to_json()`` carry attributes for convenience, but + ``to_key_value()`` must split them out. + """ + kv = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_key_value() + zarray = json.loads(kv[".zarray"].decode("utf-8")) + assert "attributes" not in zarray + + +def test_v2_to_json_still_includes_attributes() -> None: + """``to_json()`` is the bundled in-memory form and keeps attributes.""" + out: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_json()) + assert out["attributes"] == {"a": 1} + + +# --- arrays_to_tuples helper ---------------------------------------------- + +ARRAYS_TO_TUPLES_CASES = [ + Expect([1, 2, 3], (1, 2, 3), id="top-level-list"), + Expect({"a": [1, [2, 3]], "b": "x"}, {"a": (1, (2, 3)), "b": "x"}, id="nested-in-dict"), + Expect(5, 5, id="scalar-int"), + Expect("s", "s", id="scalar-str"), + Expect(None, None, id="scalar-none"), + Expect( + {"name": "bytes", "configuration": {"nums": [1, 2]}}, + {"name": "bytes", "configuration": {"nums": (1, 2)}}, + id="dict-keys-preserved", + ), +] + + +@pytest.mark.parametrize("case", ARRAYS_TO_TUPLES_CASES, ids=lambda c: c.id) +def test_arrays_to_tuples(case: Expect[object, object]) -> None: + """arrays_to_tuples recursively converts JSON arrays to tuples.""" + assert arrays_to_tuples(case.input) == case.output + + +# --- ZarrV3ArrayMetadata.from_json ---------------------------------------- + + +def test_v3_from_json_reconstructs_required_fields() -> None: + """V3 from_json reconstructs the required fields from a document.""" + doc = ZarrV3ArrayMetadata.create_default( + shape=(7,), + attributes={"a": 1}, + data_type=ZarrV3NamedConfig(name="int32", configuration={}), + ).to_json() + model = ZarrV3ArrayMetadata.from_json(doc) + assert model.shape == (7,) + assert model.data_type == ZarrV3NamedConfig(name="int32", configuration={}) + assert model.attributes == {"a": 1} + + +def test_v3_from_json_defaults_for_omitted_optionals() -> None: + """V3 from_json supplies defaults for omitted optional fields.""" + doc = ZarrV3ArrayMetadata.create_default( + attributes={}, storage_transformers=(), dimension_names=UNSET + ).to_json() + # to_json omits these entirely; from_json must restore defaults + model = ZarrV3ArrayMetadata.from_json(doc) + assert model.attributes == {} + assert model.storage_transformers == () + assert model.dimension_names is UNSET + + +def test_v3_from_json_routes_unknown_keys_to_extra_fields() -> None: + """V3 from_json routes unknown top-level keys into extra_fields.""" + doc = ZarrV3ArrayMetadata.create_default( + extra_fields={"my_ext": {"must_understand": False}} + ).to_json() + model = ZarrV3ArrayMetadata.from_json(doc) + assert model.extra_fields == {"my_ext": {"must_understand": False}} + + +def test_v3_from_json_standard_keys_not_in_extra_fields() -> None: + """V3 from_json keeps standard keys out of extra_fields.""" + doc = ZarrV3ArrayMetadata.create_default( + shape=(10,), attributes={"a": 1}, dimension_names=("x",) + ).to_json() + model = ZarrV3ArrayMetadata.from_json(doc) + assert model.extra_fields == {} + + +def test_v3_from_json_nested_arrays_in_attributes_become_tuples() -> None: + """V3 from_json converts nested arrays in attributes into tuples.""" + doc = ZarrV3ArrayMetadata.create_default(attributes={"scale": [[1, 2], [3, 4]]}).to_json() + model = ZarrV3ArrayMetadata.from_json(doc) + assert model.attributes == {"scale": ((1, 2), (3, 4))} + + +# --- ZarrV3ArrayMetadata.from_key_value ---------------------------------- + + +def test_v3_from_key_value_parses_zarr_json() -> None: + """V3 from_key_value parses the zarr.json entry into a model.""" + kv = ZarrV3ArrayMetadata.create_default(shape=(3,)).to_key_value() + model = ZarrV3ArrayMetadata.from_key_value(kv) + assert model.shape == (3,) + + +# --- Cluster 2: from_key_value missing-key raises (parametrized) ----------- + +FROM_KEY_VALUE_MISSING_PARAMS = [ + pytest.param( + ZarrV3ArrayMetadata, + ExpectFail({}, MetadataValidationError, id="v3-missing-zarr-json", msg="missing store key"), + id="v3-missing-zarr-json", + ), + pytest.param( + ZarrV2ArrayMetadata, + ExpectFail({}, MetadataValidationError, id="v2-missing-zarray", msg="missing store key"), + id="v2-missing-zarray", + ), +] + + +@pytest.mark.parametrize(("model_cls", "case"), FROM_KEY_VALUE_MISSING_PARAMS) +def test_from_key_value_missing_key_raises( + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], + case: ExpectFail[dict[str, bytes]], +) -> None: + """from_key_value raises MetadataValidationError when the required store key is absent.""" + with case.raises(): + model_cls.from_key_value(case.input) + + +# --- Cluster 1: round-trips (model → json → model, parametrized) ----------- + +ROUNDTRIP_MODEL_JSON_PARAMS = [ + pytest.param( + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadata.create_default( + shape=(10,), + attributes={"a": 1}, + dimension_names=("x",), + storage_transformers=(ZarrV3NamedConfig(name="t", configuration={}),), + extra_fields={"ext": {"must_understand": False}}, + ), + id="v3-full", + ), + pytest.param( + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadata.create_default( + attributes={}, + dimension_names=UNSET, + storage_transformers=(), + extra_fields={}, + ), + id="v3-empty-optionals", + ), + pytest.param( + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadata.create_default(attributes={"a": 1}, filters=None, compressor=None), + id="v2-basic", + ), +] + + +@pytest.mark.parametrize(("model_cls", "model"), ROUNDTRIP_MODEL_JSON_PARAMS) +def test_roundtrip_model_json_model( + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], + model: ZarrV3ArrayMetadata | ZarrV2ArrayMetadata, +) -> None: + """A model round-trips through to_json/from_json back to an equal model.""" + assert model_cls.from_json(model.to_json()) == model + + +# --- Round-trips (model → key_value → model, parametrized) ----------------- + +ROUNDTRIP_KEY_VALUE_PARAMS = [ + pytest.param( + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadata.create_default(attributes={"a": 1}), + id="v3", + ), + pytest.param( + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadata.create_default(attributes={"a": 1}), + id="v2", + ), +] + + +@pytest.mark.parametrize(("model_cls", "model"), ROUNDTRIP_KEY_VALUE_PARAMS) +def test_roundtrip_via_key_value( + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], + model: ZarrV3ArrayMetadata | ZarrV2ArrayMetadata, +) -> None: + """A model round-trips through to_key_value/from_key_value back to an equal model.""" + assert model_cls.from_key_value(model.to_key_value()) == model + + +# --- Round-trips (json → model → json, direction distinct — kept direct) --- + + +def test_v3_roundtrip_json_model_json() -> None: + """A v3 document round-trips through from_json/to_json back to an equal document.""" + doc = ZarrV3ArrayMetadata.create_default( + shape=(10,), attributes={"a": 1}, dimension_names=("x",) + ).to_json() + assert ZarrV3ArrayMetadata.from_json(doc).to_json() == doc + + +def test_v2_roundtrip_json_model_json() -> None: + """A v2 document round-trips through from_json/to_json back to an equal document.""" + doc = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_json() + assert ZarrV2ArrayMetadata.from_json(doc).to_json() == doc + + +# --- to_json shares no mutable state with the model ------------------------ + +TO_JSON_NO_ALIASING_PARAMS = [ + pytest.param( + ZarrV3ArrayMetadata.create_default( + shape=(2,), + attributes={"a": {"b": [1]}}, + codecs=(ZarrV3NamedConfig(name="blosc", configuration={"opts": {"level": 1}}),), + extra_fields={"ext": {"must_understand": False, "cfg": {"x": [1]}}}, + ), + id="v3", + ), + pytest.param( + ZarrV2ArrayMetadata.create_default( + attributes={"a": {"b": [1]}}, + compressor={"id": "zstd", "opts": {"level": 1}}, + filters=({"id": "delta", "cfg": [1]},), + fill_value=[0, 0], + ), + id="v2", + ), +] + + +@pytest.mark.parametrize("model", TO_JSON_NO_ALIASING_PARAMS) +def test_to_json_shares_no_mutable_state_with_model( + model: ZarrV3ArrayMetadata | ZarrV2ArrayMetadata, +) -> None: + """Mutating a document returned by to_json leaves the model unchanged.""" + baseline = copy.deepcopy(model.to_json()) + mutate_nested_containers(model.to_json()) + assert model.to_json() == baseline + + +def test_v3_parser_accepts_bare_string_data_type() -> None: + """V3 from_json accepts a bare-string data_type and re-serializes it canonically.""" + doc = ZarrV3ArrayMetadata.create_default().to_json() + doc["data_type"] = "int32" + model = ZarrV3ArrayMetadata.from_json(doc) + assert model.data_type == ZarrV3NamedConfig(name="int32", configuration={}) + assert model.to_json()["data_type"] == "int32" + + +@pytest.mark.parametrize("name", ["bytes", "ANY string", "urn:example:codec"]) +def test_metadata_field_accepts_any_string_name(name: str) -> None: + """The structural layer checks the name type, not syntax or registration.""" + assert validate_metadata_field_v3({"name": name}) == [] + + +@pytest.mark.parametrize("value", [0, 1, "false", None]) +def test_metadata_field_must_understand_must_be_boolean(value: object) -> None: + """must_understand is a JSON boolean, not a truthy scalar.""" + problems = validate_metadata_field_v3({"name": "x", "must_understand": value}) + assert [(problem.loc, problem.kind) for problem in problems] == [ + (("must_understand",), "invalid_type") + ] + + +def test_metadata_field_rejects_unknown_envelope_member() -> None: + """Unknown envelope keys cannot be silently discarded during normalization.""" + problems = validate_metadata_field_v3({"name": "x", "typo": 1}) + assert [(problem.loc, problem.kind) for problem in problems] == [(("typo",), "invalid_value")] + + +@pytest.mark.parametrize("field", ["codecs", "storage_transformers"]) +def test_optional_extension_points_allow_must_understand_false(field: str) -> None: + """Codecs and storage transformers may be explicitly ignorable.""" + doc: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) + doc[field] = ({"name": "optional", "must_understand": False},) + assert validate_array_metadata_v3(doc) == [] + + +@pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) +def test_required_extension_points_reject_must_understand_false(field: str) -> None: + """Core extension points needed to locate or decode chunks cannot be ignored.""" + doc: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) + doc[field] = {"name": "optional", "must_understand": False} + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + ((field, "must_understand"), "invalid_value") + ] + + +def test_v3_codecs_cannot_be_empty() -> None: + """The core document requires at least one array-to-bytes codec.""" + doc: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) + doc["codecs"] = () + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + (("codecs",), "invalid_value") + ] + + +def test_v2_roundtrip_with_compressor_and_filters() -> None: + # Non-None compressor/filters must round-trip; extra assertion on .compressor. + """A v2 model with non-None compressor and filters round-trips.""" + compressor: ZarrV2CodecMetadata = {"id": "blosc", "clevel": 5} + filters: tuple[ZarrV2CodecMetadata, ...] = ({"id": "delta"},) + m = ZarrV2ArrayMetadata.create_default(compressor=compressor, filters=filters) + restored = ZarrV2ArrayMetadata.from_json(m.to_json()) + assert restored == m + assert restored.compressor == {"id": "blosc", "clevel": 5} + + +# --- ZarrV2ArrayMetadata.from_json ---------------------------------------- + + +def test_v2_from_json_reconstructs_fields() -> None: + """V2 from_json reconstructs the fields from a document.""" + doc = ZarrV2ArrayMetadata.create_default(shape=(4,), attributes={"a": 1}, dtype=" None: + """V2 from_json reads an absent attributes key as UNSET, distinct from an + explicit empty mapping.""" + absent = ZarrV2ArrayMetadata.from_json(ZarrV2ArrayMetadata.create_default().to_json()) + explicit = ZarrV2ArrayMetadata.from_json( + ZarrV2ArrayMetadata.create_default(attributes={}).to_json() + ) + assert absent.attributes is UNSET + assert explicit.attributes == {} + assert absent != explicit + + +# --- ZarrV2ArrayMetadata.from_key_value -------------------------------- + + +def test_v2_from_key_value_remerges_zattrs() -> None: + """V2 from_key_value re-merges .zattrs back into attributes.""" + kv = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}, shape=(10,)).to_key_value() + model = ZarrV2ArrayMetadata.from_key_value(kv) + assert model.attributes == {"a": 1} + assert model.shape == (10,) + + +@pytest.mark.parametrize("extra_key", ["attributes", "vendor_extension"]) +def test_v2_from_key_value_rejects_zarray_extra_members(extra_key: str) -> None: + """Raw `.zarray` documents reject every non-spec member.""" + doc: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default().to_json()) + doc.pop("attributes", None) + doc[extra_key] = {} + + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) + + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + ((extra_key,), "invalid_value") + ] + + +def test_v2_zattrs_presence_round_trips() -> None: + """The .zattrs file's presence is part of the store: an absent file reads + as UNSET and emits no .zattrs; an explicit empty file reads as {} and + emits .zattrs — the two stores stay distinct through a round-trip.""" + explicit_kv = dict(ZarrV2ArrayMetadata.create_default(attributes={}).to_key_value()) + assert ".zattrs" in explicit_kv + absent_kv = dict(explicit_kv) + del absent_kv[".zattrs"] + + absent = ZarrV2ArrayMetadata.from_key_value(absent_kv) + explicit = ZarrV2ArrayMetadata.from_key_value(explicit_kv) + assert absent.attributes is UNSET + assert explicit.attributes == {} + assert ".zattrs" not in absent.to_key_value() + assert ".zattrs" in explicit.to_key_value() + + +def test_v2_from_json_nested_arrays_in_attributes_become_tuples() -> None: + """V2 from_json converts nested arrays in attributes into tuples.""" + doc = ZarrV2ArrayMetadata.create_default(attributes={"axes": [[0, 1], [2, 3]]}).to_json() + model = ZarrV2ArrayMetadata.from_json(doc) + assert model.attributes == {"axes": ((0, 1), (2, 3))} + + +# --- scalar wire-type guards (is_/validate_/parse_) ------------------------ +# +# Each value is modelled once as Expect[object, frozenset[tuple[str | int, ...]]] +# where `output` is the set of expected problem locs validate_* must report — +# frozenset() means VALID. Valid iff output == frozenset(). + +JSON_VALIDATE_CASES: list[Expect[object, frozenset[tuple[str | int, ...]]]] = [ + Expect("s", frozenset(), id="str"), + Expect(1, frozenset(), id="int"), + Expect(1.5, frozenset(), id="float"), + Expect(True, frozenset(), id="bool"), + Expect(None, frozenset(), id="none"), + Expect({"a": [1, {"b": None}], "c": "x"}, frozenset(), id="nested-containers"), + Expect((1, 2, 3), frozenset(), id="tuple-array"), + Expect(float("nan"), frozenset({()}), id="nan"), + Expect(float("inf"), frozenset({()}), id="inf"), + Expect(float("-inf"), frozenset({()}), id="negative-inf"), + Expect(object(), frozenset({()}), id="object"), + Expect(b"abc", frozenset({()}), id="bytes"), + Expect(bytearray(b"abc"), frozenset({()}), id="bytearray"), + Expect({1: "x"}, frozenset({()}), id="non-str-key"), + Expect([1, object()], frozenset({(1,)}), id="non-json-list-item"), + Expect({"ok": object()}, frozenset({("ok",)}), id="non-json-value"), +] + + +@pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) +def test_is_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """is_json reports whether a value is JSON-serializable.""" + assert is_json(case.input) is (case.output == frozenset()) + + +@pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) +def test_validate_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """validate_json reports the problems (and their locs) for a value.""" + problems = validate_json(case.input) + assert (problems == []) is (case.output == frozenset()) + assert {p.loc for p in problems} >= case.output + + +@pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) +def test_parse_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """parse_json returns valid JSON values and raises on invalid ones.""" + if case.output == frozenset(): + parsed = parse_json(case.input) + assert arrays_to_tuples(parsed) == arrays_to_tuples(case.input) + else: + with pytest.raises(MetadataValidationError): + parse_json(case.input) + + +def test_parse_json_materializes_abstract_containers() -> None: + """Accepted Mapping and Sequence values normalize to JSON encoder containers.""" + value = UserDict({"values": range(3)}) + + parsed = parse_json(value) + + assert parsed == {"values": (0, 1, 2)} + assert type(parsed) is dict + assert type(parsed["values"]) is tuple + json.dumps(parsed, allow_nan=False) + + +def test_json_type_guard_rejects_abstract_sequence() -> None: + """A guard cannot narrow an abstract sequence that only the parser materializes.""" + assert not is_json(range(3)) + assert parse_json(range(3)) == (0, 1, 2) + + +def test_parse_metadata_field_materializes_abstract_containers() -> None: + """Named-config parsing produces canonical containers at every nesting level.""" + value = UserDict({"name": "example", "configuration": UserDict({"values": range(2)})}) + + parsed = parse_metadata_field_v3(value) + + assert isinstance(parsed, dict) + assert parsed == {"name": "example", "configuration": {"values": (0, 1)}} + assert type(parsed["configuration"]) is dict + + +def test_metadata_field_type_guard_rejects_abstract_mapping() -> None: + """A metadata-field guard only narrows concrete TypedDict-shaped objects.""" + value = UserDict({"name": "bytes"}) + + assert not is_metadata_field_v3(value) + assert parse_metadata_field_v3(value) == {"name": "bytes"} + + +def test_validate_json_reports_json_in_message() -> None: + """validate_json's message for a non-JSON value mentions JSON.""" + problems = validate_json(object()) + assert problems[0].loc == () + assert "JSON" in problems[0].message + + +METADATA_FIELD_VALIDATE_CASES: list[Expect[object, frozenset[tuple[str | int, ...]]]] = [ + Expect("bytes", frozenset(), id="bare-string"), + Expect({"name": "x", "configuration": {"a": 1}}, frozenset(), id="named-config"), + Expect({"name": "bytes"}, frozenset(), id="name-only"), + Expect(5, frozenset({()}), id="not-str-or-mapping"), + Expect({"configuration": {}}, frozenset({("name",)}), id="missing-name"), + Expect({"name": 3}, frozenset({("name",)}), id="non-str-name"), + Expect( + {"name": "x", "configuration": [1]}, + frozenset({("configuration",)}), + id="config-not-mapping", + ), + Expect( + {"name": "x", "configuration": {1: "y"}}, + frozenset({("configuration",)}), + id="config-non-str-key", + ), +] + + +@pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) +def test_is_metadata_field_v3(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """is_metadata_field_v3 reports whether a value is a v3 metadata field.""" + assert is_metadata_field_v3(case.input) is (case.output == frozenset()) + + +@pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) +def test_validate_metadata_field_v3( + case: Expect[object, frozenset[tuple[str | int, ...]]], +) -> None: + """validate_metadata_field_v3 reports the problems for a metadata-field value.""" + problems = validate_metadata_field_v3(case.input) + assert (problems == []) is (case.output == frozenset()) + assert {p.loc for p in problems} >= case.output + + +@pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) +def test_parse_metadata_field_v3( + case: Expect[object, frozenset[tuple[str | int, ...]]], +) -> None: + """parse_metadata_field_v3 returns valid fields and raises on invalid ones.""" + if case.output == frozenset(): + assert parse_metadata_field_v3(case.input) is case.input + else: + with pytest.raises(MetadataValidationError): + parse_metadata_field_v3(case.input) + + +# --- array-document wire-type guards (is_/validate_/parse_) ---------------- +# +# Each case starts from a valid document (built by `make`) and applies a +# mutation. `expected_locs` are loc paths `validate_*` must report for the +# invalid cases (a subset check, so accumulation of OTHER problems is allowed). + + +def _build_v3(**overrides: Unpack[ZarrV3ArrayMetadataPartial]) -> dict[str, object]: + return dict(ZarrV3ArrayMetadata.create_default(**overrides).to_json()) + + +def _build_v2(**overrides: Unpack[ZarrV2ArrayMetadataPartial]) -> dict[str, object]: + return dict(ZarrV2ArrayMetadata.create_default(**overrides).to_json()) + + +def _mutate(build: Callable[[], dict], mutate: Callable[[dict], object]) -> Callable[[], dict]: + def _factory() -> dict: + doc = build() + mutate(doc) + return doc + + return _factory + + +def _del(key: str) -> Callable[[dict], object]: + return lambda doc: doc.pop(key) + + +def _set(key: str, value: object) -> Callable[[dict], object]: + return lambda doc: doc.__setitem__(key, value) + + +V3_DOC_CASES: list[Expect[Callable[[], object], frozenset[tuple[str | int, ...]]]] = [ + Expect(_build_v3, frozenset(), id="valid"), + Expect( + lambda: _build_v3(shape=(10,), attributes={"a": 1}, dimension_names=("x",)), + frozenset(), + id="valid-with-attributes-and-dim-names", + ), + Expect( + lambda: _build_v3(extra_fields={"my_ext": {"must_understand": False}}), + frozenset(), + id="valid-with-extra-fields", + ), + Expect(_mutate(_build_v3, _del("shape")), frozenset({("shape",)}), id="missing-shape"), + Expect( + _mutate(_build_v3, _set("data_type", 5)), + frozenset({("data_type",)}), + id="bad-data-type", + ), + Expect( + _mutate(_build_v3, _set("shape", "not-a-shape")), + frozenset({("shape",)}), + id="shape-not-sequence", + ), + Expect( + _mutate(_build_v3, _set("shape", [1, "x"])), + frozenset({("shape",)}), + id="shape-non-int-item", + ), + Expect( + _mutate(_build_v3, _set("codecs", (5,))), + frozenset({("codecs", 0)}), + id="bad-codec-entry", + ), + Expect(lambda: [1, 2, 3], frozenset({()}), id="non-mapping-list"), + Expect(lambda: "nope", frozenset({()}), id="non-mapping-str"), + Expect( + _mutate(_mutate(_build_v3, _del("shape")), _set("data_type", 5)), + frozenset({("shape",), ("data_type",)}), + id="missing-shape-and-bad-data-type", + ), +] + +V2_DOC_CASES: list[Expect[Callable[[], object], frozenset[tuple[str | int, ...]]]] = [ + Expect(_build_v2, frozenset(), id="valid"), + Expect(lambda: _build_v2(attributes={"a": 1}), frozenset(), id="valid-with-attributes"), + Expect( + lambda: _build_v2(compressor=None, filters=None), + frozenset(), + id="valid-none-compressor-filters", + ), + Expect( + _mutate(_build_v2, _del("chunks")), + frozenset({("chunks",)}), + id="missing-chunks", + ), + Expect( + _mutate(_build_v2, _set("shape", [1, "x"])), + frozenset({("shape",)}), + id="bad-shape", + ), + Expect( + _mutate(_mutate(_build_v2, _del("chunks")), _set("shape", [1, "x"])), + frozenset({("chunks",), ("shape",)}), + id="missing-chunks-and-bad-shape", + ), +] + +ALL_DOC_CASES = [ + *( + pytest.param( + is_array_metadata_v3, + validate_array_metadata_v3, + parse_array_metadata_v3, + c, + id=f"v3-{c.id}", + ) + for c in V3_DOC_CASES + ), + *( + pytest.param( + is_array_metadata_v2, + validate_array_metadata_v2, + parse_array_metadata_v2, + c, + id=f"v2-{c.id}", + ) + for c in V2_DOC_CASES + ), +] + + +@pytest.mark.parametrize(("is_fn", "validate_fn", "parse_fn", "case"), ALL_DOC_CASES) +def test_array_metadata_guards( + is_fn: Callable[[object], bool], + validate_fn: Callable[[object], list[ValidationProblem]], + parse_fn: Callable[[object], object], + case: Expect[Callable[[], object], frozenset[tuple[str | int, ...]]], +) -> None: + """is_/validate_/parse_ array-metadata guards agree on validity and locs for each case.""" + doc = case.input() + valid = case.output == frozenset() + assert is_fn(doc) is valid + problems = validate_fn(doc) + assert (problems == []) is valid + assert {p.loc for p in problems} >= case.output + if valid: + assert parse_fn(doc) is doc + else: + with pytest.raises(MetadataValidationError): + parse_fn(doc) + + +# --- strict from_json validation ------------------------------------------- + + +FROM_JSON_REJECT_PARAMS = [ + pytest.param( + ZarrV3ArrayMetadata, + ExpectFail(lambda: {"zarr_format": 3}, MetadataValidationError, id="x"), + id="v3-missing-required", + ), + pytest.param( + ZarrV3ArrayMetadata, + ExpectFail(_mutate(_build_v3, _set("data_type", 5)), MetadataValidationError, id="x"), + id="v3-bad-field-type", + ), + pytest.param( + ZarrV2ArrayMetadata, + ExpectFail(lambda: {"zarr_format": 2}, MetadataValidationError, id="x"), + id="v2-missing-required", + ), + pytest.param( + ZarrV3NamedConfig, + ExpectFail(lambda: 5, MetadataValidationError, id="x"), + id="zarr-metadata-bad-input", + ), +] + + +@pytest.mark.parametrize(("model", "case"), FROM_JSON_REJECT_PARAMS) +def test_from_json_rejects_malformed( + model: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata | ZarrV3NamedConfig], + case: ExpectFail[Callable[[], object]], +) -> None: + """from_json raises MetadataValidationError on a malformed document.""" + with case.raises(): + model.from_json(case.input()) + + +# --- ValidationProblem / MetadataValidationError / _prefix ----------------- +# Small structural tests — not "parametrize over inputs" shaped, kept direct. + + +def test_validation_problem_str_with_loc() -> None: + """ValidationProblem.__str__ renders a non-empty loc as a dotted path.""" + p = ValidationProblem(loc=("codecs", 0, "name"), message="expected str", kind="invalid_type") + assert str(p) == "codecs.0.name: expected str" + + +def test_validation_problem_str_empty_loc() -> None: + """ValidationProblem.__str__ renders an empty loc as .""" + p = ValidationProblem(loc=(), message="not a mapping", kind="invalid_type") + assert str(p) == ": not a mapping" + + +def test_validation_problem_is_frozen() -> None: + """ValidationProblem is immutable (frozen dataclass).""" + p = ValidationProblem(loc=("shape",), message="x", kind="invalid_type") + with pytest.raises(dataclasses.FrozenInstanceError): + # setattr: assigning to a frozen field is an intentional runtime error, + # spelled dynamically so it is not also a static type error. + setattr(p, "message", "y") # noqa: B010 + + +def test_metadata_validation_error_holds_problems() -> None: + """MetadataValidationError carries its problem list and renders them in its message.""" + problems = [ + ValidationProblem(loc=("shape",), message="missing required key", kind="missing_key"), + ValidationProblem( + loc=("data_type",), message="expected a metadata field", kind="invalid_type" + ), + ] + err = MetadataValidationError(problems) + assert err.problems == problems + assert "shape: missing required key" in str(err) + assert "data_type: expected a metadata field" in str(err) + + +def test_prefix_prepends_loc_head() -> None: + """_prefix prepends a loc head to each problem's loc.""" + problems = [ValidationProblem(loc=("name",), message="expected str", kind="invalid_type")] + prefixed = _prefix(0, problems) + assert prefixed == [ + ValidationProblem(loc=(0, "name"), message="expected str", kind="invalid_type") + ] + + +# --- Stricter v2/v3 field validation and error kinds ------------------------- + + +def test_v2_dtype_must_be_string_or_records() -> None: + """A non-string, non-records v2 dtype is rejected with an invalid_type problem.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dtype": 42} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dtype",), "invalid_type")] + + +def test_v2_structured_dtype_records_accepted() -> None: + """A structured v2 dtype (field records, optionally nested/shaped) validates.""" + dtype = (("a", " None: + """A field record with the wrong arity is rejected.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dtype": (("a",),)} + problems = validate_array_metadata_v2(doc) + assert [p.loc for p in problems] == [("dtype",)] + + +def test_v2_order_literal_enforced() -> None: + """An order other than 'C' or 'F' is rejected with an invalid_value problem.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"order": "Q"} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("order",), "invalid_value")] + + +def test_v2_compressor_must_be_codec_or_none() -> None: + """A compressor that is not null or a codec config mapping is rejected.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"compressor": "zlib"} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("compressor",), "invalid_type")] + + +def test_v2_compressor_requires_string_id() -> None: + """A compressor mapping without a string id is rejected.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"compressor": {"level": 3}} + problems = validate_array_metadata_v2(doc) + assert [p.loc for p in problems] == [("compressor",)] + + +def test_v2_filters_must_be_codec_sequence_or_none() -> None: + """Filters that are not null or a sequence of codec configs are rejected.""" + for bad in (7, (5,), "gzip"): + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"filters": bad} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("filters",), "invalid_type")], bad + + +def test_v2_shape_and_chunks_must_have_equal_rank() -> None: + """Raw v2 metadata requires one chunk length per array dimension.""" + doc = dict(ZarrV2ArrayMetadata.create_default(shape=(2, 3)).to_json()) + doc["chunks"] = (1,) + + assert [(p.loc, p.kind) for p in validate_array_metadata_v2(doc)] == [ + (("chunks",), "invalid_value") + ] + with pytest.raises(MetadataValidationError, match="same number of dimensions"): + ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) + + +def test_v2_filters_must_be_nonempty_when_present() -> None: + """A non-null v2 filter sequence contains one or more codec configurations.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) + doc["filters"] = () + + assert [(p.loc, p.kind) for p in validate_array_metadata_v2(doc)] == [ + (("filters",), "invalid_value") + ] + with pytest.raises(MetadataValidationError, match="at least one filter"): + ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) + + +def test_v2_dimension_separator_literal_enforced() -> None: + """A dimension_separator other than '.' or '/' is rejected.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dimension_separator": "-"} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] + + +def test_v2_zarr_format_literal_enforced() -> None: + """A v2 document claiming zarr_format 3 is rejected with an invalid_value problem.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"zarr_format": 3} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] + + +def test_v3_zarr_format_literal_enforced() -> None: + """A v3 document claiming zarr_format 2 is rejected with an invalid_value problem.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"zarr_format": 2} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] + + +@pytest.mark.parametrize( + ("document", "validate"), + [ + pytest.param( + dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"zarr_format": 2.0}, + validate_array_metadata_v2, + id="v2", + ), + pytest.param( + dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"zarr_format": 3.0}, + validate_array_metadata_v3, + id="v3", + ), + ], +) +def test_array_zarr_format_rejects_float( + document: object, validate: Callable[[object], list[ValidationProblem]] +) -> None: + """Integer-valued floats do not satisfy integer format literals.""" + assert [(p.loc, p.kind) for p in validate(document)] == [(("zarr_format",), "invalid_value")] + + +def test_array_v2_rejects_unknown_document_member() -> None: + """The closed v2 merged-document shape rejects undeclared members.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"unexpected": 1} + + assert [(p.loc, p.kind) for p in validate_array_metadata_v2(doc)] == [ + (("unexpected",), "invalid_value") + ] + + +def test_array_v3_from_json_materializes_abstract_containers() -> None: + """A flexible input mapping becomes the canonical dict/tuple model shape.""" + doc = UserDict(dict(ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json())) + doc["shape"] = range(2) + + model = ZarrV3ArrayMetadata.from_json(doc) + + assert model.shape == (0, 1) + assert type(model.shape) is tuple + + +def test_from_key_value_rejects_non_standard_json_constant() -> None: + """Store JSON decoding rejects JavaScript NaN/Infinity constants.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) + doc["fill_value"] = float("nan") + raw = json.dumps(doc) + + with pytest.raises(MetadataValidationError, match="invalid JSON"): + ZarrV3ArrayMetadata.from_key_value({"zarr.json": raw.encode()}) + + +def test_to_key_value_rejects_non_finite_model_value() -> None: + """Strict encoding prevents directly-constructed models from writing invalid JSON.""" + model = ZarrV3ArrayMetadata.create_default(fill_value=float("nan")) + + with pytest.raises(ValueError, match="JSON compliant"): + model.to_key_value() + + +def test_v3_node_type_literal_enforced() -> None: + """A v3 array document claiming node_type 'group' is rejected.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"node_type": "group"} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("node_type",), "invalid_value")] + + +def test_missing_key_kind_is_machine_readable() -> None: + """A missing required key is distinguishable by kind, without message matching.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) + del doc["chunk_key_encoding"] + problems = validate_array_metadata_v3(doc) + assert problems == [ + ValidationProblem(("chunk_key_encoding",), "missing required key", "missing_key") + ] + + +# --- Unified error channels --------------------------------------------------- + + +def test_from_key_value_invalid_json_raises_metadata_error() -> None: + """Undecodable store bytes raise MetadataValidationError (kind invalid_json), not JSONDecodeError.""" + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV3ArrayMetadata.from_key_value({"zarr.json": b"{not json"}) + assert [p.kind for p in exc_info.value.problems] == ["invalid_json"] + + +def test_from_key_value_invalid_utf8_raises_metadata_error() -> None: + """Invalid UTF-8 store bytes use the same invalid_json error channel.""" + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV3ArrayMetadata.from_key_value({"zarr.json": b"\x80"}) + assert [p.kind for p in exc_info.value.problems] == ["invalid_json"] + + +def test_v2_from_key_value_scalar_root_raises_metadata_error() -> None: + """A scalar .zarray document fails through the unified metadata error channel.""" + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV2ArrayMetadata.from_key_value({".zarray": b"null"}) + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + ((), "invalid_type") + ] + + +def test_from_key_value_missing_key_kind() -> None: + """A missing store key surfaces as a missing_key problem at the store-key loc.""" + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV2ArrayMetadata.from_key_value({}) + assert exc_info.value.problems == [ + ValidationProblem((".zarray",), "missing store key", "missing_key") + ] + + +def test_extra_fields_overlap_raises_metadata_error() -> None: + """The extra-fields overlap invariant raises MetadataValidationError (a ValueError).""" + with pytest.raises(MetadataValidationError, match="Extra fields") as exc_info: + ZarrV3ArrayMetadata.create_default(extra_fields={"shape": {"must_understand": False}}) + assert [p.kind for p in exc_info.value.problems] == ["invalid_value"] + + +def test_extension_point_fields_annotated_with_role_alias() -> None: + """Extension-point fields are annotated with ZarrV3MetadataField (the + logical role), not ZarrV3NamedConfig (the current serialized form), so a + future widening of the field union does not move annotation sites.""" + assert ZarrV3MetadataField is ZarrV3NamedConfig + annotations = ZarrV3ArrayMetadata.__annotations__ + for field_name in ("data_type", "chunk_grid", "chunk_key_encoding"): + assert annotations[field_name] == "ZarrV3MetadataField" + for field_name in ("codecs", "storage_transformers"): + assert annotations[field_name] == "tuple[ZarrV3MetadataField, ...]" + + +# --- Adversarial-probe fixes: documents that used to pass validation --------- + + +def test_shape_rejects_json_booleans() -> None: + """JSON booleans are not integers: shape/chunks containing true/false are + rejected (bool is an int subclass in Python, so isinstance alone passes).""" + v3 = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"shape": (True, True)} + assert [p.loc for p in validate_array_metadata_v3(v3)] == [("shape",)] + v2 = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"chunks": (True,)} + assert [p.loc for p in validate_array_metadata_v2(v2)] == [("chunks",)] + + +def test_shape_rejects_negative_dimensions() -> None: + """Dimension lengths must be non-negative; a negative entry is invalid_value.""" + v3 = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"shape": (-1,)} + assert [(p.loc, p.kind) for p in validate_array_metadata_v3(v3)] == [ + (("shape",), "invalid_value") + ] + v2 = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"chunks": (-5,)} + assert [(p.loc, p.kind) for p in validate_array_metadata_v2(v2)] == [ + (("chunks",), "invalid_value") + ] + + +def test_dimension_names_length_must_match_shape() -> None: + """dimension_names must have one entry per dimension of shape.""" + doc = dict(ZarrV3ArrayMetadata.create_default(shape=(10,)).to_json()) | { + "dimension_names": ("x", "y", "z") + } + assert [(p.loc, p.kind) for p in validate_array_metadata_v3(doc)] == [ + (("dimension_names",), "invalid_value") + ] + + +def test_attributes_values_must_be_json() -> None: + """Attribute values are JSON-checked recursively (like fill_value), so a + non-serializable value is a validation problem, not a later TypeError.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"attributes": {"a": {1, 2}}} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("attributes", "a"), "invalid_type")] + + +def test_configuration_values_must_be_json() -> None: + """Configuration values are JSON-checked recursively, so an int-keyed dict + cannot pass validation and be silently rewritten by json.dumps.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | { + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": {1: 2}}} + } + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [ + (("chunk_grid", "configuration", "chunk_shape"), "invalid_type") + ] + + +def test_v3_extension_keys_must_be_strings() -> None: + """A non-string top-level key cannot be represented by a v3 document type.""" + doc: dict[object, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) + doc[1] = {"must_understand": False} + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + ((), "invalid_type") + ] + + +def test_v3_extension_values_must_be_json() -> None: + """Extension payloads are JSON-checked before a model is constructed.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) + doc["ext"] = {"must_understand": False, "payload": object()} + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + (("ext", "payload"), "invalid_type") + ] + + +def test_v3_json_extension_without_waiver_is_preserved_as_must_understand() -> None: + """A JSON extension without an explicit false waiver remains must-understand.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) + doc["ext"] = 1 + parsed = parse_array_metadata_v3(doc) + assert is_array_metadata_v3(parsed) + model = ZarrV3ArrayMetadata.from_json(doc) + assert model.extra_fields["ext"] == 1 + assert model.must_understand_fields == {"ext": 1} + + +def test_v2_codec_configuration_values_must_be_json() -> None: + """Non-JSON codec parameters are rejected for compressors and filters.""" + for field, value, expected_loc in ( + ("compressor", {"id": "x", "payload": object()}, ("compressor", "payload")), + ("filters", ({"id": "x", "payload": object()},), ("filters", 0, "payload")), + ): + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) + doc[field] = value + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v2(doc)] == [ + (expected_loc, "invalid_type") + ] + + +def test_dimension_sequences_reject_binary_values() -> None: + """Binary buffers are not JSON arrays even though they are integer sequences.""" + v3 = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"shape": b"\x02"} + v2 = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"chunks": b"\x02"} + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(v3)] == [ + (("shape",), "invalid_type") + ] + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v2(v2)] == [ + (("chunks",), "invalid_type") + ] + + +def test_array_parsers_normalize_json_lists_before_narrowing() -> None: + """Parsers return tuple-backed document types while guards reject raw list forms.""" + v3_raw = json.loads(json.dumps(ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json())) + v2_raw = json.loads(json.dumps(ZarrV2ArrayMetadata.create_default(shape=(2,)).to_json())) + + assert validate_array_metadata_v3(v3_raw) == [] + assert validate_array_metadata_v2(v2_raw) == [] + assert not is_array_metadata_v3(v3_raw) + assert not is_array_metadata_v2(v2_raw) + + v3_parsed = parse_array_metadata_v3(v3_raw) + v2_parsed = parse_array_metadata_v2(v2_raw) + assert isinstance(v3_parsed["shape"], tuple) + assert isinstance(v3_parsed["codecs"], tuple) + assert isinstance(v2_parsed["shape"], tuple) + assert isinstance(v2_parsed["chunks"], tuple) + + +def test_array_guards_reject_noncanonical_nested_json() -> None: + """Document guards cannot narrow values that only parsers can materialize.""" + v3 = dict(ZarrV3ArrayMetadata.create_default().to_json()) + v3["fill_value"] = range(2) + v2 = dict(ZarrV2ArrayMetadata.create_default().to_json()) + v2["fill_value"] = range(2) + + assert not is_array_metadata_v3(v3) + assert not is_array_metadata_v2(v2) + assert parse_array_metadata_v3(v3)["fill_value"] == (0, 1) + assert parse_array_metadata_v2(v2)["fill_value"] == (0, 1) + + +# --- must_understand partition (spec: MUST fail to open unrecognized fields) -- + + +def test_must_understand_fields_partition() -> None: + """must_understand_fields contains every extra field not explicitly waived + with must_understand: false, including implicitly-true and non-mapping + fields, so a reader can discharge the spec's fail-to-open duty by + subtracting the extensions it recognizes.""" + model = ZarrV3ArrayMetadata.create_default( + extra_fields={ + "ext_a": {"name": "a", "must_understand": False}, + "ext_b": {"name": "b"}, + "ext_c": {"name": "c", "must_understand": True}, + "ext_d": 123, + } + ) + assert set(model.must_understand_fields) == {"ext_b", "ext_c", "ext_d"} + recognized = {"ext_b"} + assert model.must_understand_fields.keys() - recognized == {"ext_c", "ext_d"} + + +def test_must_understand_fields_empty_when_all_waived() -> None: + """must_understand_fields is empty when every extra field is explicitly waived.""" + model = ZarrV3ArrayMetadata.create_default( + extra_fields={"ext_a": {"name": "a", "must_understand": False}} + ) + assert model.must_understand_fields == {} + + +def test_dimension_names_null_field_rejected() -> None: + """A dimension_names field whose VALUE is null is invalid: the spec permits + null as an element (an unnamed dimension), never as the field value — "not + specified" is spelled by omitting the key. Consumers bridging from an + in-memory None sentinel must drop the key, not write null.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"dimension_names": None} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dimension_names",), "invalid_type")] + # and the model's own None spelling correctly maps to key absence + assert ( + "dimension_names" not in ZarrV3ArrayMetadata.create_default(dimension_names=UNSET).to_json() + ) + + +# --- create_default derives the chunk grid from shape ------------------------ + + +def test_v3_create_default_chunk_grid_follows_shape() -> None: + """Overriding shape without chunk_grid derives a consistent default grid: + one chunk covering the array (chunk_shape == shape), instead of silently + keeping the scalar default's 0-d grid.""" + model = ZarrV3ArrayMetadata.create_default(shape=(100, 100)) + assert model.chunk_grid == ZarrV3NamedConfig( + name="regular", configuration={"chunk_shape": (100, 100)} + ) + + +def test_v3_create_default_explicit_chunk_grid_respected() -> None: + """An explicit chunk_grid override wins over the shape-derived default.""" + grid = ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": (10, 10)}) + model = ZarrV3ArrayMetadata.create_default(shape=(100, 100), chunk_grid=grid) + assert model.chunk_grid == grid + + +def test_v2_create_default_chunks_follow_shape() -> None: + """Overriding shape without chunks derives chunks == shape.""" + model = ZarrV2ArrayMetadata.create_default(shape=(100, 100)) + assert model.chunks == (100, 100) + + +def test_v2_create_default_explicit_chunks_respected() -> None: + """An explicit chunks override wins over the shape-derived default.""" + model = ZarrV2ArrayMetadata.create_default(shape=(100, 100), chunks=(10, 10)) + assert model.chunks == (10, 10) + + +def test_v3_create_default_zero_length_dimensions() -> None: + """chunk_shape == shape is spec-sound even with zero-length dimensions: + 'The chunk shape elements are non-zero when the corresponding dimensions + of the arrays have non-zero length' — the constraint is conditional, so a + zero chunk length is permitted exactly where the dimension is empty.""" + model = ZarrV3ArrayMetadata.create_default(shape=(0, 3)) + assert model.chunk_grid.configuration["chunk_shape"] == (0, 3) + + +def test_create_default_derivation_is_one_way() -> None: + """Overriding the chunk grid (v3) or chunks (v2) without shape leaves the + scalar default shape=() untouched: a user-supplied chunk_grid is an + extension point taken verbatim, and deriving shape from it would require + interpreting grid configurations, which the model layer never does.""" + grid = ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": (10, 10)}) + v3 = ZarrV3ArrayMetadata.create_default(chunk_grid=grid) + assert v3.shape == () + assert v3.chunk_grid == grid + v2 = ZarrV2ArrayMetadata.create_default(chunks=(10, 10)) + assert v2.shape == () + assert v2.chunks == (10, 10) + + +# --- v2 dimension_separator default (roborev job 426) ------------------------- + + +def test_v2_absent_dimension_separator_means_dot() -> None: + """A .zarray that omits dimension_separator uses the v2 convention default + '.', not '/': chunk keys of real-world default-separator v2 arrays look + like '0.0'. The model normalizes the absent key to an explicit '.' — a + semantics-preserving spelling normalization.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) + del doc["dimension_separator"] + model = ZarrV2ArrayMetadata.from_json(doc) + assert model.dimension_separator == "." + assert model.to_json()["dimension_separator"] == "." + + +def test_v2_from_key_value_without_separator_means_dot() -> None: + """The .zarray store-file path applies the same '.' default for an absent + dimension_separator key.""" + doc = { + k: v + for k, v in ZarrV2ArrayMetadata.create_default().to_json().items() + if k not in ("dimension_separator", "attributes") + } + import json as _json + + model = ZarrV2ArrayMetadata.from_key_value({".zarray": _json.dumps(doc).encode()}) + assert model.dimension_separator == "." + + +def test_v2_null_dimension_separator_rejected() -> None: + """dimension_separator may be absent, '.', or '/' — never null: the + document grammar has no null spelling for this field.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dimension_separator": None} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] + + +def test_dimension_names_absent_and_all_null_are_distinct() -> None: + """An absent dimension_names field and an explicit all-null one are + semantically different documents: the explicit form says every dimension + has a name, which is null; absence says there are no dimension names. + The model preserves the distinction (UNSET vs a tuple of Nones), and both + spellings round-trip faithfully.""" + absent_doc = dict(ZarrV3ArrayMetadata.create_default(shape=(2, 3)).to_json()) + explicit_doc = absent_doc | {"dimension_names": (None, None)} + + absent = ZarrV3ArrayMetadata.from_json(absent_doc) + explicit = ZarrV3ArrayMetadata.from_json(explicit_doc) + + assert absent.dimension_names is UNSET + assert explicit.dimension_names == (None, None) + assert absent != explicit + assert "dimension_names" not in absent.to_json() + assert absent.to_json() == absent_doc + assert explicit.to_json() == explicit_doc diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py new file mode 100644 index 0000000000..4b8c22b84d --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -0,0 +1,572 @@ +"""Tests for the group and consolidated metadata models in `zarr_metadata.model`.""" + +import copy +import dataclasses +import json +from collections import UserDict +from collections.abc import Callable + +import pytest + +from tests.model._cases import mutate_nested_containers +from zarr_metadata.model import UNSET +from zarr_metadata.model._array import ZarrV3ArrayMetadata +from zarr_metadata.model._group import ( + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV2GroupMetadataPartial, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, + ZarrV3GroupMetadataPartial, +) +from zarr_metadata.model._validation import ( + MetadataValidationError, + ValidationProblem, + is_group_metadata_v2, + is_group_metadata_v3, + parse_group_metadata_v2, + parse_group_metadata_v3, + validate_group_metadata_v2, + validate_group_metadata_v3, +) + +# --- ZarrV3GroupMetadata --------------------------------------------------- + + +def test_group_v3_roundtrip() -> None: + """A v3 group document round-trips through the model unchanged.""" + doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": (1, 2)}} + model = ZarrV3GroupMetadata.from_json(doc) + assert model.to_json() == doc + + +def test_group_v3_omits_empty_attributes() -> None: + """to_json omits the attributes key when attributes is empty.""" + model = ZarrV3GroupMetadata.create_default() + assert "attributes" not in model.to_json() + + +def test_group_v3_lists_become_tuples() -> None: + """from_json converts JSON arrays in attributes to tuples.""" + doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": [1, 2]}} + model = ZarrV3GroupMetadata.from_json(doc) + assert model.attributes == {"a": (1, 2)} + + +def test_group_v3_extra_fields_roundtrip() -> None: + """Unknown top-level keys land in extra_fields and reappear in to_json.""" + doc = { + "zarr_format": 3, + "node_type": "group", + "my_extension": {"name": "thing", "must_understand": False}, + } + model = ZarrV3GroupMetadata.from_json(doc) + assert model.extra_fields == {"my_extension": {"name": "thing", "must_understand": False}} + assert model.to_json() == doc + + +def test_group_v3_json_extra_field_roundtrips_as_must_understand() -> None: + """A non-object extra field is preserved and implicitly requires understanding.""" + doc = {"zarr_format": 3, "node_type": "group", "ext": [1, 2]} + model = ZarrV3GroupMetadata.from_json(doc) + assert model.to_json()["ext"] == (1, 2) + assert model.must_understand_fields == {"ext": (1, 2)} + + +def test_group_v3_extra_fields_overlap_rejected() -> None: + """Constructing a v3 group model with extra_fields shadowing a standard key raises.""" + with pytest.raises(ValueError, match="Extra fields"): + ZarrV3GroupMetadata( + attributes={}, + consolidated_metadata=UNSET, + extra_fields={"node_type": {"name": "x", "must_understand": False}}, + ) + + +def test_group_v3_consolidated_extra_field_rejected() -> None: + """extra_fields may not shadow the consolidated_metadata convention key.""" + with pytest.raises(ValueError, match="Extra fields"): + ZarrV3GroupMetadata( + attributes={}, + consolidated_metadata=UNSET, + extra_fields={"consolidated_metadata": {"name": "x", "must_understand": False}}, + ) + + +def test_group_v3_missing_required_key() -> None: + """parse_group_metadata_v3 reports each missing required key.""" + with pytest.raises(MetadataValidationError, match="node_type"): + parse_group_metadata_v3({"zarr_format": 3}) + + +def test_group_v3_bad_attributes() -> None: + """parse_group_metadata_v3 rejects a non-mapping attributes value.""" + with pytest.raises(MetadataValidationError, match="attributes"): + parse_group_metadata_v3({"zarr_format": 3, "node_type": "group", "attributes": 5}) + + +@pytest.mark.parametrize( + ("document", "validate"), + [ + pytest.param( + {"zarr_format": 2.0}, + validate_group_metadata_v2, + id="v2", + ), + pytest.param( + {"zarr_format": 3.0, "node_type": "group"}, + validate_group_metadata_v3, + id="v3", + ), + ], +) +def test_group_zarr_format_rejects_float( + document: object, validate: Callable[[object], list[ValidationProblem]] +) -> None: + """Integer-valued floats do not satisfy integer format literals.""" + assert [(p.loc, p.kind) for p in validate(document)] == [(("zarr_format",), "invalid_value")] + + +def test_group_v2_rejects_unknown_document_member() -> None: + """The closed v2 merged-document shape rejects undeclared members.""" + assert [(p.loc, p.kind) for p in validate_group_metadata_v2({"zarr_format": 2, "x": 1})] == [ + (("x",), "invalid_value") + ] + + +@pytest.mark.parametrize( + ("parse", "document"), + [ + pytest.param(parse_group_metadata_v2, {"zarr_format": 2}, id="v2"), + pytest.param( + parse_group_metadata_v3, + {"zarr_format": 3, "node_type": "group"}, + id="v3", + ), + ], +) +def test_group_parser_materializes_abstract_mapping( + parse: Callable[[object], object], document: dict[str, object] +) -> None: + """A successful group parser always returns the declared concrete TypedDict shape.""" + parsed = parse(UserDict(document)) + + assert type(parsed) is dict + assert parsed == document + + +def test_group_guards_reject_noncanonical_nested_json() -> None: + """Document guards cannot narrow values that only parsers can materialize.""" + v3 = {"zarr_format": 3, "node_type": "group", "extension": range(2)} + v2 = {"zarr_format": 2, "attributes": {"values": range(2)}} + + assert not is_group_metadata_v3(v3) + assert not is_group_metadata_v2(v2) + assert parse_group_metadata_v3(v3)["extension"] == (0, 1) + assert parse_group_metadata_v2(v2)["attributes"] == {"values": (0, 1)} + + +def test_group_v3_extension_fields_are_validated() -> None: + """Group extension payloads must be JSON values with a must-understand flag.""" + doc = { + "zarr_format": 3, + "node_type": "group", + "ext": {"must_understand": False, "payload": object()}, + } + assert [(problem.loc, problem.kind) for problem in validate_group_metadata_v3(doc)] == [ + (("ext", "payload"), "invalid_type") + ] + + +def test_group_v3_key_value_roundtrip() -> None: + """from_key_value(to_key_value()) is the identity for v3 groups.""" + model = ZarrV3GroupMetadata.create_default(attributes={"a": 1}) + assert ZarrV3GroupMetadata.from_key_value(model.to_key_value()) == model + + +def test_group_v3_update() -> None: + """update replaces the given fields and returns a new instance.""" + base = ZarrV3GroupMetadata.create_default() + updated = base.update(attributes={"a": 1}) + assert updated.attributes == {"a": 1} + assert base.attributes == {} + + +# --- ZarrV2GroupMetadata --------------------------------------------------- + + +def test_group_v2_key_value_split() -> None: + """v2 to_key_value writes .zgroup and .zattrs; from_key_value merges them.""" + model = ZarrV2GroupMetadata.create_default(attributes={"a": 1}) + kv = model.to_key_value() + assert set(kv) == {".zgroup", ".zattrs"} + assert json.loads(kv[".zgroup"]) == {"zarr_format": 2} + assert ZarrV2GroupMetadata.from_key_value(kv) == model + + +@pytest.mark.parametrize("extra_key", ["attributes", "vendor_extension"]) +def test_v2_group_from_key_value_rejects_zgroup_extra_members(extra_key: str) -> None: + """Raw `.zgroup` documents reject every non-spec member.""" + doc: dict[str, object] = {"zarr_format": 2, extra_key: {}} + + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV2GroupMetadata.from_key_value({".zgroup": json.dumps(doc).encode()}) + + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + ((extra_key,), "invalid_value") + ] + + +def test_group_v2_zattrs_presence_round_trips() -> None: + """A v2 group with no .zattrs file parses with UNSET attributes and emits + no .zattrs; an explicit empty .zattrs stays a file — the stores remain + distinct through a round-trip.""" + absent = ZarrV2GroupMetadata.from_key_value({".zgroup": b'{"zarr_format": 2}'}) + assert absent.attributes is UNSET + assert ".zattrs" not in absent.to_key_value() + explicit = ZarrV2GroupMetadata.from_key_value( + {".zgroup": b'{"zarr_format": 2}', ".zattrs": b"{}"} + ) + assert explicit.attributes == {} + assert ".zattrs" in explicit.to_key_value() + assert absent != explicit + + +def test_group_v2_json_roundtrip() -> None: + """A merged-form v2 group document round-trips through the model unchanged.""" + doc = {"zarr_format": 2, "attributes": {"a": 1}} + model = ZarrV2GroupMetadata.from_json(doc) + assert model.to_json() == doc + + +def test_group_v2_omits_empty_attributes() -> None: + """to_json omits the attributes key when attributes is empty.""" + assert "attributes" not in ZarrV2GroupMetadata.create_default().to_json() + + +def test_group_v2_not_a_mapping() -> None: + """parse_group_metadata_v2 rejects a non-mapping document.""" + with pytest.raises(MetadataValidationError, match="expected a mapping"): + parse_group_metadata_v2([1, 2, 3]) + + +def test_group_v2_missing_required_key() -> None: + """parse_group_metadata_v2 reports a missing zarr_format key.""" + with pytest.raises(MetadataValidationError, match="zarr_format"): + parse_group_metadata_v2({}) + + +# --- Partial TypedDict drift guards ----------------------------------------- + + +def test_group_partial_keys_match_settable_model_fields() -> None: + """Each group partial TypedDict must list exactly the settable model fields. + + Guards against drift: adding/removing a settable field on a group model + without updating its `*Partial` TypedDict fails here. + """ + for model_cls, partial_cls in ( + (ZarrV3GroupMetadata, ZarrV3GroupMetadataPartial), + (ZarrV2GroupMetadata, ZarrV2GroupMetadataPartial), + ): + settable = {f.name for f in dataclasses.fields(model_cls) if f.init} + assert set(partial_cls.__annotations__) == settable + + +# --- ZarrV3ConsolidatedMetadata -------------------------------------------- + + +def test_consolidated_v3_roundtrip() -> None: + """A v3 group with inline consolidated metadata round-trips, with child + entries parsed into array/group models.""" + child = ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json() + doc = { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": child, "g": {"zarr_format": 3, "node_type": "group"}}, + }, + } + model = ZarrV3GroupMetadata.from_json(doc) + assert isinstance(model.consolidated_metadata, ZarrV3ConsolidatedMetadata) + assert isinstance(model.consolidated_metadata.metadata["a"], ZarrV3ArrayMetadata) + assert isinstance(model.consolidated_metadata.metadata["g"], ZarrV3GroupMetadata) + assert model.to_json() == doc + + +def test_consolidated_v3_must_understand_true_rejected() -> None: + """ZarrV3ConsolidatedMetadata enforces must_understand=False at runtime.""" + with pytest.raises(ValueError, match="must_understand"): + ZarrV3ConsolidatedMetadata(must_understand=True, metadata={}) + + +def test_consolidated_v3_from_json_must_understand_true_rejected() -> None: + """from_json rejects a consolidated document carrying must_understand=true.""" + with pytest.raises(MetadataValidationError, match="must_understand"): + ZarrV3ConsolidatedMetadata.from_json( + {"kind": "inline", "must_understand": True, "metadata": {}} + ) + + +def test_consolidated_v3_entry_without_node_type_rejected() -> None: + """from_json rejects a consolidated entry lacking a recognizable node_type.""" + with pytest.raises(MetadataValidationError, match="node_type"): + ZarrV3ConsolidatedMetadata.from_json( + {"kind": "inline", "must_understand": False, "metadata": {"a": {"zarr_format": 3}}} + ) + + +def test_consolidated_v3_not_a_mapping() -> None: + """from_json rejects a non-mapping consolidated document.""" + with pytest.raises(MetadataValidationError, match="expected a mapping"): + ZarrV3ConsolidatedMetadata.from_json(5) + + +# --- ZarrV2ConsolidatedMetadata -------------------------------------------- + + +def test_consolidated_v2_verbatim_roundtrip() -> None: + """The v2 .zmetadata model holds the flat file-keyed map verbatim, + including nodes that have no .zattrs entry.""" + doc = { + "zarr_consolidated_format": 1, + "metadata": { + ".zgroup": {"zarr_format": 2}, + "a/.zarray": { + "zarr_format": 2, + "shape": (2,), + "chunks": (2,), + "dtype": "|u1", + "fill_value": 0, + "order": "C", + "compressor": None, + "filters": None, + }, + }, + } + model = ZarrV2ConsolidatedMetadata.from_json(doc) + assert model.to_json() == doc + + +def test_consolidated_v2_key_value_roundtrip() -> None: + """from_key_value(to_key_value()) is the identity for .zmetadata documents.""" + model = ZarrV2ConsolidatedMetadata.from_json( + {"zarr_consolidated_format": 1, "metadata": {".zgroup": {"zarr_format": 2}}} + ) + assert ZarrV2ConsolidatedMetadata.from_key_value(model.to_key_value()) == model + + +def test_consolidated_v2_lists_become_tuples() -> None: + """from_json converts JSON arrays inside entries to tuples.""" + doc = { + "zarr_consolidated_format": 1, + "metadata": {"a/.zarray": {"shape": [2, 3]}}, + } + model = ZarrV2ConsolidatedMetadata.from_json(doc) + assert model.metadata == {"a/.zarray": {"shape": (2, 3)}} + + +def test_consolidated_v2_envelope_validation() -> None: + """from_json rejects a .zmetadata document missing the metadata key.""" + with pytest.raises(MetadataValidationError, match="metadata"): + ZarrV2ConsolidatedMetadata.from_json({"zarr_consolidated_format": 1}) + + +def test_consolidated_v2_not_a_mapping() -> None: + """from_json rejects a non-mapping .zmetadata document.""" + with pytest.raises(MetadataValidationError, match="expected a mapping"): + ZarrV2ConsolidatedMetadata.from_json([1]) + + +def test_consolidated_v2_format_literal_enforced() -> None: + """A .zmetadata document must declare consolidated format 1.""" + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV2ConsolidatedMetadata.from_json({"zarr_consolidated_format": 2, "metadata": {}}) + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + (("zarr_consolidated_format",), "invalid_value") + ] + + +def test_consolidated_v2_metadata_values_must_be_json() -> None: + """Non-JSON values in the flat metadata map are rejected during ingestion.""" + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV2ConsolidatedMetadata.from_json( + {"zarr_consolidated_format": 1, "metadata": {".zgroup": object()}} + ) + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + (("metadata", ".zgroup"), "invalid_type") + ] + + +def test_group_v2_from_key_value_scalar_root_raises_metadata_error() -> None: + """A scalar .zgroup document fails through the unified metadata error channel.""" + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV2GroupMetadata.from_key_value({".zgroup": b"null"}) + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + ((), "invalid_type") + ] + + +# --- Literal-value enforcement ----------------------------------------------- + + +def test_group_v3_literals_enforced() -> None: + """A v3 group doc with wrong zarr_format or node_type is rejected with invalid_value.""" + base = ZarrV3GroupMetadata.create_default().to_json() + for key, bad in (("zarr_format", 2), ("node_type", "array")): + problems = validate_group_metadata_v3(dict(base) | {key: bad}) + assert [(p.loc, p.kind) for p in problems] == [((key,), "invalid_value")], key + + +def test_group_v2_zarr_format_literal_enforced() -> None: + """A v2 group doc claiming zarr_format 3 is rejected with invalid_value.""" + problems = validate_group_metadata_v2({"zarr_format": 3}) + assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] + + +# --- Consolidated envelope validated by the group validator ------------------ + + +def test_group_v3_validator_agrees_with_from_json_on_consolidated() -> None: + """The group validator validates the consolidated envelope and entries, so + is_group_metadata_v3 never vouches for a document from_json would reject.""" + bad_docs = ( + # empty envelope: missing kind/must_understand/metadata + {"zarr_format": 3, "node_type": "group", "consolidated_metadata": {}}, + # entry without a recognizable node_type + { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": {"zarr_format": 3}}, + }, + }, + # must_understand: true + { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": True, + "metadata": {}, + }, + }, + ) + for doc in bad_docs: + assert validate_group_metadata_v3(doc) != [], doc + with pytest.raises(MetadataValidationError): + ZarrV3GroupMetadata.from_json(doc) + + +def test_group_v3_valid_consolidated_passes_validator() -> None: + """A well-formed consolidated group validates cleanly (control case).""" + child = ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json() + doc = { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": child, "g": {"zarr_format": 3, "node_type": "group"}}, + }, + } + assert validate_group_metadata_v3(doc) == [] + + +def test_v3_consolidated_rejects_unknown_envelope_member() -> None: + """The inline consolidated envelope is closed and never drops accepted members.""" + doc = { + "kind": "inline", + "must_understand": False, + "metadata": {}, + "unexpected": 1, + } + + with pytest.raises(MetadataValidationError, match="unexpected"): + ZarrV3ConsolidatedMetadata.from_json(doc) + + +def test_v2_consolidated_rejects_unknown_document_member() -> None: + """The v2 consolidated document is closed and never drops accepted members.""" + doc = {"zarr_consolidated_format": 1, "metadata": {}, "unexpected": 1} + + with pytest.raises(MetadataValidationError, match="unexpected"): + ZarrV2ConsolidatedMetadata.from_json(doc) + + +# --- must_understand partition ------------------------------------------------ + + +def test_group_must_understand_fields_partition() -> None: + """The group model partitions extra fields by the spec's implicit-true rule, + like the array model.""" + model = ZarrV3GroupMetadata.create_default( + extra_fields={ + "waived": {"name": "w", "must_understand": False}, + "implicit": {"name": "i"}, + } + ) + assert set(model.must_understand_fields) == {"implicit"} + + +def test_group_v3_null_consolidated_metadata_repaired_to_absence() -> None: + """consolidated_metadata: null was written by a historical zarr-python bug. + Those stores must remain readable, but the bug spelling is not honored: + it is read as absence (UNSET) and never written back — the round-trip + deliberately repairs the document rather than preserving the bug.""" + null_doc = {"zarr_format": 3, "node_type": "group", "consolidated_metadata": None} + assert validate_group_metadata_v3(null_doc) == [] + model = ZarrV3GroupMetadata.from_json(null_doc) + assert model.consolidated_metadata is UNSET + assert "consolidated_metadata" not in model.to_json() + assert model == ZarrV3GroupMetadata.from_json({"zarr_format": 3, "node_type": "group"}) + + +# --- to_json shares no mutable state with the model ------------------------ + +TO_JSON_NO_ALIASING_PARAMS = [ + pytest.param( + ZarrV3GroupMetadata.create_default( + attributes={"a": {"b": [1]}}, + consolidated_metadata=ZarrV3ConsolidatedMetadata( + metadata={ + "child": ZarrV3ArrayMetadata.create_default(attributes={"x": {"y": 1}}), + "grp": ZarrV3GroupMetadata.create_default(attributes={"x": {"y": 1}}), + } + ), + extra_fields={"ext": {"must_understand": False, "cfg": {"x": [1]}}}, + ), + id="v3-group", + ), + pytest.param( + ZarrV2GroupMetadata.create_default(attributes={"a": {"b": [1]}}), + id="v2-group", + ), + pytest.param( + ZarrV3ConsolidatedMetadata( + metadata={"child": ZarrV3ArrayMetadata.create_default(attributes={"x": {"y": 1}})} + ), + id="v3-consolidated", + ), + pytest.param( + ZarrV2ConsolidatedMetadata(metadata={"a/.zarray": {"nested": {"x": [1]}}}), + id="v2-consolidated", + ), +] + + +@pytest.mark.parametrize("model", TO_JSON_NO_ALIASING_PARAMS) +def test_to_json_shares_no_mutable_state_with_model( + model: ZarrV3GroupMetadata + | ZarrV2GroupMetadata + | ZarrV3ConsolidatedMetadata + | ZarrV2ConsolidatedMetadata, +) -> None: + """Mutating a document returned by to_json leaves the model unchanged.""" + baseline = copy.deepcopy(model.to_json()) + mutate_nested_containers(model.to_json()) + assert model.to_json() == baseline diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py new file mode 100644 index 0000000000..e5714bb8fb --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -0,0 +1,302 @@ +"""Executable example: integrating the metadata models with pydantic (v2). + +Pydantic's native dataclass introspection CAN be made to work (see +`test_native_dataclass_introspection_is_possible_but_diverges`): the models +keep their annotation-only imports behind `TYPE_CHECKING`, so a bare +`TypeAdapter(ZarrV3ArrayMetadata)` raises `class-not-fully-defined`, but +`rebuild(_types_namespace=...)` with the names supplied resolves the schema. +It is still the wrong tool: it validates the MODEL SHAPE, not the DOCUMENT — +no `from_json` normalization (a bare-string `data_type` is rejected), and +pydantic's lax coercion silently re-opens holes the library's validators +close (`shape=[True, -5]` coerces to `(1, -5)`; a wrong `dimension_names` +count passes). The recommended integration delegates wholesale — treat the +model as an opaque value: + +- `InstanceOf` makes pydantic's core schema an is-instance check (no field + introspection), +- validation goes through `from_json` (the single source of truth for what + a well-formed document is, including normalization: bare-string metadata + fields, arrays-to-tuples), +- serialization goes through `to_json` (the canonical document form). + +`MetadataValidationError` subclasses `ValueError`, so pydantic converts a +failed parse into its own `ValidationError` with the loc-annotated problem +messages intact. +""" + +from collections.abc import Mapping +from typing import Annotated, Generic, TypeVar + +import pytest +from pydantic import ( + BaseModel, + BeforeValidator, + ConfigDict, + InstanceOf, + PlainSerializer, + PydanticSchemaGenerationError, + PydanticUserError, + TypeAdapter, + ValidationError, + model_validator, +) + +from zarr_metadata import JSONValue +from zarr_metadata.model import ZarrV3ArrayMetadata, ZarrV3NamedConfig + +# --- the integration (this is the example) ----------------------------------- + + +def _as_array_metadata_v3(value: object) -> ZarrV3ArrayMetadata: + """Accept an existing model instance or a raw metadata document.""" + if isinstance(value, ZarrV3ArrayMetadata): + return value + return ZarrV3ArrayMetadata.from_json(value) + + +# return_type is explicit because to_json's own annotation (`ZarrV3ArrayMetadataJSON`) +# is a TYPE_CHECKING-only name pydantic cannot resolve at runtime. +ArrayMetadataV3Field = Annotated[ + InstanceOf[ZarrV3ArrayMetadata], + BeforeValidator(_as_array_metadata_v3), + PlainSerializer(ZarrV3ArrayMetadata.to_json, return_type=dict), +] +"""A pydantic-ready field type for v3 array metadata. + +Validates raw documents via `from_json`, passes model instances through, +and serializes to the canonical document form via `to_json`. +""" + + +class ArrayManifest(BaseModel): + """Example consumer model: a named array with its metadata document.""" + + path: str + metadata: ArrayMetadataV3Field + + +# --- tests pinning the example ------------------------------------------------ + +VALID_DOC = { + "zarr_format": 3, + "node_type": "array", + "shape": [10], + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}}, + "chunk_key_encoding": {"name": "default"}, + "codecs": [{"name": "bytes"}], +} + + +def test_raw_document_is_validated_into_a_model() -> None: + """A raw metadata document on a pydantic field is parsed by from_json, + with the library's normalization applied (tuples, canonical field form).""" + manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) + assert isinstance(manifest.metadata, ZarrV3ArrayMetadata) + assert manifest.metadata.shape == (10,) + assert manifest.metadata.data_type.name == "uint8" + + +def test_model_instance_passes_through() -> None: + """An already-constructed model instance is accepted unchanged.""" + model = ZarrV3ArrayMetadata.from_json(VALID_DOC) + manifest = ArrayManifest(path="a/b", metadata=model) + assert manifest.metadata is model + + +def test_invalid_document_surfaces_problems_in_validation_error() -> None: + """A structurally-invalid document fails pydantic validation, carrying the + loc-annotated problem messages from MetadataValidationError.""" + doc = dict(VALID_DOC) + del doc["chunk_key_encoding"] + with pytest.raises(ValidationError) as exc_info: + ArrayManifest.model_validate({"path": "a/b", "metadata": doc}) + assert "chunk_key_encoding: missing required key" in str(exc_info.value) + + +def test_dump_emits_canonical_document() -> None: + """model_dump serializes the field via to_json — the canonical document, + not pydantic's field-by-field view of the dataclass.""" + manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) + dumped = manifest.model_dump() + assert dumped["metadata"] == manifest.metadata.to_json() + # Empty configurations use the extension-definition shorthand form. + assert dumped["metadata"]["data_type"] == "uint8" + + +def test_json_roundtrip_through_pydantic() -> None: + """model_dump_json output re-validates to an equal manifest (JSON emits + tuples as arrays; from_json converts them back).""" + manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) + revived = ArrayManifest.model_validate_json(manifest.model_dump_json()) + assert revived == manifest + + +def test_type_adapter_standalone() -> None: + """The annotated alias also works without a BaseModel, via TypeAdapter.""" + adapter = TypeAdapter(ArrayMetadataV3Field) + model = adapter.validate_python(VALID_DOC) + assert isinstance(model, ZarrV3ArrayMetadata) + assert adapter.dump_python(model) == model.to_json() + + +# --- the road not taken: native dataclass introspection ---------------------- + + +def test_native_dataclass_introspection_is_not_supported() -> None: + """Pydantic cannot field-introspect the model dataclasses: the UNSET + sentinel (PEP 661, typing_extensions.Sentinel) in the optional-field + annotations has no pydantic schema (as of pydantic 2.13), so even the + rebuild-with-namespace recipe fails. Introspection was already the wrong + tool before the sentinel existed — it validated the model shape rather + than the document, and its lax coercion re-opened validator holes (e.g. + shape=[True, -5] coerced to (1, -5)) — so the delegation patterns above + are the only supported integrations. If this test ever fails because + pydantic learned to handle sentinels, revisit whether the introspection + path needs its divergences documented again.""" + from zarr_metadata._common import JSONValue + from zarr_metadata.model import UNSET + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON, ZarrV3ExtensionField + + def build_and_use() -> None: + adapter = TypeAdapter(ZarrV3ArrayMetadata) + adapter.rebuild( + force=True, + _types_namespace={ + "JSONValue": JSONValue, + "ZarrV3ExtensionField": ZarrV3ExtensionField, + "ZarrV3MetadataFieldJSON": ZarrV3MetadataFieldJSON, + "ZarrV3ArrayMetadataJSON": ZarrV3ArrayMetadataJSON, + "ZarrV3NamedConfig": ZarrV3NamedConfig, + "ZarrV3MetadataField": ZarrV3NamedConfig, + "UNSET": UNSET, + }, + ) + adapter.validate_python({}) + + with pytest.raises((AttributeError, PydanticSchemaGenerationError, PydanticUserError)): + build_and_use() + + +# --- a first-class pydantic model, engine-backed (the pydantic-zarr pattern) -- +# +# When a consumer wants a real BaseModel — JSON schema generation, and +# generics for typed attributes, as in pydantic-zarr's ArraySpec — the model +# fields are pydantic-native, but validation and serialization still route +# through the library: a mode="before" validator canonicalizes every input +# document with from_json(...).to_json(), so the structural validators and +# normalization run BEFORE pydantic parses fields (no coercion divergence), +# and the document form is the bridge in both directions. + +AttrsT = TypeVar("AttrsT") + + +class NamedConfig(BaseModel): + """Pydantic mirror of a normalized metadata extension envelope.""" + + name: str + configuration: dict[str, JSONValue] = {} + must_understand: bool = True + + +class ArrayMetadataV3Spec(BaseModel, Generic[AttrsT]): + """A pydantic-native, attribute-typed view of a v3 array metadata document. + + The library is the engine: every input is canonicalized and structurally + validated by `ZarrV3ArrayMetadata.from_json` before pydantic sees the + fields, and `to_document` / `to_metadata_model` emit through the library. + """ + + model_config = ConfigDict(frozen=True) + + zarr_format: int = 3 + node_type: str = "array" + shape: tuple[int, ...] + data_type: NamedConfig + chunk_grid: NamedConfig + chunk_key_encoding: NamedConfig + fill_value: JSONValue + codecs: tuple[NamedConfig, ...] + attributes: AttrsT + dimension_names: tuple[str | None, ...] | None = None + storage_transformers: tuple[NamedConfig, ...] = () + + @model_validator(mode="before") + @classmethod + def _canonicalize(cls, data: object) -> object: + """Route every input document through the library's validation and + normalization; pydantic then parses only canonical documents.""" + if isinstance(data, Mapping): + doc = dict(ZarrV3ArrayMetadata.from_json(data).to_json()) + for key in ("data_type", "chunk_grid", "chunk_key_encoding"): + if isinstance(doc[key], str): + doc[key] = {"name": doc[key]} + for key in ("codecs", "storage_transformers"): + doc[key] = tuple( + {"name": item} if isinstance(item, str) else item for item in doc.get(key, ()) + ) + doc.setdefault("attributes", {}) + return doc + return data + + def to_metadata_model(self) -> ZarrV3ArrayMetadata: + """Bridge back to the canonical model, via the document form. + + In the document, "no dimension names" is key-absence, not null; the + pydantic-side None translates to dropping the key. + """ + doc = self.model_dump() + if doc["dimension_names"] is None: + del doc["dimension_names"] + return ZarrV3ArrayMetadata.from_json(doc) + + def to_document(self) -> dict[str, object]: + """The canonical document (omit-empty conventions applied).""" + return dict(self.to_metadata_model().to_json()) + + +class MicroscopyAttrs(BaseModel): + """Example of consumer-typed attributes, pydantic-zarr style.""" + + resolution_um: float + + +def test_spec_typed_attributes() -> None: + """The generic parameter types the attributes, so consumers get validated, + attribute-level access — the pydantic-zarr ArraySpec pattern.""" + doc = dict(VALID_DOC) | {"attributes": {"resolution_um": 0.5}} + spec = ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(doc) + assert spec.attributes.resolution_um == 0.5 + assert spec.data_type == NamedConfig(name="uint8") + + +def test_spec_engine_validates_before_pydantic() -> None: + """The library's structural validation runs before pydantic's parsing, so + coercion cannot re-open validator holes (contrast with the native + introspection test above, where [True, -5] coerced to (1, -5)).""" + with pytest.raises(ValidationError, match="shape"): + ArrayMetadataV3Spec[MicroscopyAttrs].model_validate( + dict(VALID_DOC) | {"shape": [True, -5], "attributes": {"resolution_um": 0.5}} + ) + + +def test_spec_bridges_to_canonical_model_and_document() -> None: + """to_metadata_model / to_document round-trip through the document form, + and the emitted document matches what the library itself would emit.""" + doc = dict(VALID_DOC) | {"attributes": {"resolution_um": 0.5}} + spec = ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(doc) + model = spec.to_metadata_model() + assert isinstance(model, ZarrV3ArrayMetadata) + assert spec.to_document() == dict(model.to_json()) + # and back: the document revalidates to an equal spec + assert ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(spec.to_document()) == spec + + +def test_spec_json_schema_generation() -> None: + """A real BaseModel means model_json_schema works — the capability the + opaque InstanceOf pattern cannot provide.""" + schema = ArrayMetadataV3Spec[MicroscopyAttrs].model_json_schema() + assert schema["properties"]["shape"]["type"] == "array" + assert "MicroscopyAttrs" in schema["$defs"] diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py new file mode 100644 index 0000000000..d15b3f118c --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -0,0 +1,264 @@ +"""Tests for `zarr_metadata.pydantic`, the optional pydantic field-type module. + +The hand-rolled recipes in `test_pydantic.py` document how the integration +works; this module ships it. Instances are the CORE model classes (no +parallel hierarchy), so values interoperate freely with non-pydantic code. +""" + +import json +import warnings + +import pytest +from jsonschema import Draft202012Validator +from pydantic import BaseModel, TypeAdapter, ValidationError + +import zarr_metadata.pydantic as zmp +from zarr_metadata.model import ( + ZarrV2ArrayMetadata, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV3ArrayMetadata, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, + ZarrV3NamedConfig, +) + +V3_ARRAY_DOC = dict(ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()) +V2_ARRAY_DOC = dict(ZarrV2ArrayMetadata.create_default(shape=(4,), chunks=(2,)).to_json()) +V3_GROUP_DOC = {"zarr_format": 3, "node_type": "group", "attributes": {"a": 1}} +V2_GROUP_DOC = {"zarr_format": 2, "attributes": {"a": 1}} +V3_CONSOLIDATED_DOC = { + "kind": "inline", + "must_understand": False, + "metadata": {"a": dict(V3_ARRAY_DOC)}, +} +V2_CONSOLIDATED_DOC = { + "zarr_consolidated_format": 1, + "metadata": {".zgroup": {"zarr_format": 2}}, +} + +FIELD_CASES = [ + pytest.param(zmp.ZarrV3ArrayMetadata, ZarrV3ArrayMetadata, V3_ARRAY_DOC, id="array-v3"), + pytest.param(zmp.ZarrV2ArrayMetadata, ZarrV2ArrayMetadata, V2_ARRAY_DOC, id="array-v2"), + pytest.param(zmp.ZarrV3GroupMetadata, ZarrV3GroupMetadata, V3_GROUP_DOC, id="group-v3"), + pytest.param(zmp.ZarrV2GroupMetadata, ZarrV2GroupMetadata, V2_GROUP_DOC, id="group-v2"), + pytest.param( + zmp.ZarrV3ConsolidatedMetadata, + ZarrV3ConsolidatedMetadata, + V3_CONSOLIDATED_DOC, + id="consolidated-v3", + ), + pytest.param( + zmp.ZarrV2ConsolidatedMetadata, + ZarrV2ConsolidatedMetadata, + V2_CONSOLIDATED_DOC, + id="consolidated-v2", + ), + pytest.param(zmp.ZarrV3MetadataField, ZarrV3NamedConfig, {"name": "bytes"}, id="field-v3"), +] + + +@pytest.mark.parametrize(("field_type", "model_cls", "doc"), FIELD_CASES) +def test_field_type_validates_and_dumps_canonically( + field_type: object, model_cls: type, doc: dict[str, object] +) -> None: + """Each field type parses its raw document into the CORE model class, + passes existing instances through unchanged, and dumps the canonical + document via to_json.""" + adapter = TypeAdapter(field_type) + model = adapter.validate_python(doc) + assert type(model) is model_cls + assert adapter.validate_python(model) is model + assert adapter.dump_python(model) == model.to_json() + + +def test_core_instances_interoperate() -> None: + """A core model instance (e.g. handed out by zarr-python) drops straight + into a pydantic field — the reason the module ships Annotated aliases over + the core classes rather than pydantic-aware subclasses.""" + + class Manifest(BaseModel): + metadata: zmp.ZarrV3ArrayMetadata + + core = ZarrV3ArrayMetadata.from_json(V3_ARRAY_DOC) + manifest = Manifest(metadata=core) + assert manifest.metadata is core + + +def test_validation_error_carries_problems() -> None: + """A defective document fails with the library's loc-annotated messages.""" + + class Manifest(BaseModel): + metadata: zmp.ZarrV3ArrayMetadata + + doc = dict(V3_ARRAY_DOC) + del doc["chunk_key_encoding"] + with pytest.raises(ValidationError, match="chunk_key_encoding: missing required key"): + Manifest.model_validate({"metadata": doc}) + + +def test_json_schema_generation() -> None: + """model_json_schema works, describing the document form each field accepts.""" + + class Manifest(BaseModel): + metadata: zmp.ZarrV3ArrayMetadata + codec: zmp.ZarrV3MetadataField + + schema = Manifest.model_json_schema() + metadata_schema = schema["$defs"]["ZarrV3ArrayMetadataJSON"] + assert schema["properties"]["metadata"]["$ref"] == "#/$defs/ZarrV3ArrayMetadataJSON" + assert metadata_schema["required"] == [ + "zarr_format", + "node_type", + "data_type", + "shape", + "chunk_grid", + "chunk_key_encoding", + "fill_value", + "codecs", + ] + assert metadata_schema["properties"]["zarr_format"] == { + "const": 3, + "title": "Zarr Format", + "type": "integer", + } + assert schema["properties"]["codec"]["anyOf"] == [ + {"type": "string"}, + {"$ref": "#/$defs/ZarrV3NamedConfigJSON"}, + ] + + +def test_json_schema_generation_emits_no_warnings() -> None: + """Consumers can generate every public integration schema without warning filters.""" + field_types = ( + zmp.ZarrV3ArrayMetadata, + zmp.ZarrV2ArrayMetadata, + zmp.ZarrV3GroupMetadata, + zmp.ZarrV2GroupMetadata, + zmp.ZarrV3ConsolidatedMetadata, + zmp.ZarrV2ConsolidatedMetadata, + zmp.ZarrV3MetadataField, + ) + + with warnings.catch_warnings(): + warnings.simplefilter("error") + for field_type in field_types: + TypeAdapter(field_type).json_schema() + + +def test_v2_recursive_structured_dtype_is_in_pydantic_schema() -> None: + """The schema accepts nested structured dtypes supported by the v2 specification.""" + doc = json.loads(json.dumps(V2_ARRAY_DOC)) + doc["dtype"] = [["outer", [["inner", " None: + adapter = TypeAdapter(field_type) + with pytest.raises(ValidationError): + adapter.validate_python(document) + assert list(Draft202012Validator(adapter.json_schema()).iter_errors(document)) + + +def test_v3_array_schema_rejects_empty_codecs() -> None: + """The generated schema mirrors the runtime non-empty codec pipeline rule.""" + doc = json.loads(json.dumps(V3_ARRAY_DOC)) + doc["codecs"] = [] + + _assert_runtime_and_schema_reject(zmp.ZarrV3ArrayMetadata, doc) + + +def test_array_schemas_reject_negative_dimensions() -> None: + """Both array schemas mirror the runtime non-negative dimension rule.""" + for field_type, source in ( + (zmp.ZarrV3ArrayMetadata, V3_ARRAY_DOC), + (zmp.ZarrV2ArrayMetadata, V2_ARRAY_DOC), + ): + doc = json.loads(json.dumps(source)) + doc["shape"] = [-1] + _assert_runtime_and_schema_reject(field_type, doc) + + +def test_v2_array_schema_rejects_empty_filters() -> None: + """The v2 schema mirrors the runtime one-or-more filter rule.""" + doc = json.loads(json.dumps(V2_ARRAY_DOC)) + doc["filters"] = [] + + _assert_runtime_and_schema_reject(zmp.ZarrV2ArrayMetadata, doc) + + +@pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) +def test_v3_array_schema_rejects_false_at_mandatory_extension_points(field: str) -> None: + """Mandatory v3 extension points cannot opt out of understanding.""" + doc = json.loads(json.dumps(V3_ARRAY_DOC)) + doc[field] = {"name": "example", "must_understand": False} + + _assert_runtime_and_schema_reject(zmp.ZarrV3ArrayMetadata, doc) + + +def test_metadata_field_schema_rejects_unknown_members() -> None: + """Named-configuration envelopes are closed in both runtime and schema validation.""" + _assert_runtime_and_schema_reject( + zmp.ZarrV3MetadataField, + {"name": "example", "unexpected": 1}, + ) + + +@pytest.mark.parametrize( + ("field_type", "source"), + [ + (zmp.ZarrV2ArrayMetadata, V2_ARRAY_DOC), + (zmp.ZarrV2GroupMetadata, V2_GROUP_DOC), + (zmp.ZarrV2ConsolidatedMetadata, V2_CONSOLIDATED_DOC), + ], +) +def test_v2_schema_rejects_unknown_document_members( + field_type: object, source: dict[str, object] +) -> None: + """Closed v2 merged documents expose their runtime boundary in JSON Schema.""" + doc = json.loads(json.dumps(source)) + doc["unexpected"] = 1 + + _assert_runtime_and_schema_reject(field_type, doc) + + +def test_v3_array_schema_allows_unknown_extension_fields() -> None: + """Schema constraints do not close the v3 top-level extension namespace.""" + doc = json.loads(json.dumps(V3_ARRAY_DOC)) + doc["vendor_extension"] = {"anything": [1, 2]} + adapter = TypeAdapter(zmp.ZarrV3ArrayMetadata) + + assert adapter.validate_python(doc).extra_fields == {"vendor_extension": {"anything": (1, 2)}} + assert Draft202012Validator(adapter.json_schema()).is_valid(doc) + + +def test_json_roundtrip() -> None: + """model_dump_json output re-validates to an equal pydantic model.""" + + class Manifest(BaseModel): + metadata: zmp.ZarrV3ArrayMetadata + + manifest = Manifest.model_validate({"metadata": V3_ARRAY_DOC}) + assert Manifest.model_validate_json(manifest.model_dump_json()) == manifest + + +def test_metadata_field_serializes_shorthand_and_false_object() -> None: + """The optional integration exposes the core model's canonical extension form.""" + adapter = TypeAdapter(zmp.ZarrV3MetadataField) + assert adapter.dump_python(adapter.validate_python({"name": "bytes"})) == "bytes" + assert adapter.dump_python( + adapter.validate_python({"name": "optional", "must_understand": False}) + ) == {"name": "optional", "must_understand": False} + + +def test_core_package_does_not_import_pydantic() -> None: + """Importing zarr_metadata (in a fresh interpreter) must not import + pydantic: the integration is opt-in via zarr_metadata.pydantic.""" + import subprocess + import sys + + code = "import sys, zarr_metadata; assert 'pydantic' not in sys.modules, 'leaked'" + subprocess.run([sys.executable, "-c", code], check=True) diff --git a/packages/zarr-metadata/tests/model/test_sentinel.py b/packages/zarr-metadata/tests/model/test_sentinel.py new file mode 100644 index 0000000000..252a4cdd30 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_sentinel.py @@ -0,0 +1,89 @@ +"""Tests for the pickling and copying behavior of the `UNSET` sentinel. + +The sentinel's contract is identity, so it must never be reconstructed from +state. typing_extensions >= 4.16 pickles sentinels by reference (a lookup of +the sentinel's name on its defining module), which preserves the singleton +across process boundaries; these tests pin that behavior, since models hold +`UNSET` as field values and must survive pickling and deep-copying. + +The model round-trip tests compare whole structures: dataclass equality +compares every field, and `UNSET` compares by identity, so an impostor +sentinel produced by state-based pickling would fail the equality check. +""" + +from __future__ import annotations + +import copy +import pickle + +import pytest +from typing_extensions import Sentinel + +from zarr_metadata.model import ( + UNSET, + ZarrV2ArrayMetadata, + ZarrV2GroupMetadata, + ZarrV3ArrayMetadata, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, +) + +# Whole-model cases covering the states we know are problematic for +# serialization: every optional-key field in the UNSET (absent) state, the +# same fields in the present state (including present-but-empty, which must +# stay distinct from absent), and UNSET nested inside consolidated metadata. +MODEL_CASES = { + "array-v3-dimension-names-unset": ZarrV3ArrayMetadata.create_default(shape=(4,)), + "array-v3-dimension-names-set": ZarrV3ArrayMetadata.create_default(shape=(2, 2)).update( + dimension_names=("x", None) + ), + "array-v2-attributes-unset": ZarrV2ArrayMetadata.create_default(shape=(4,)), + "array-v2-attributes-empty": ZarrV2ArrayMetadata.create_default(shape=(4,), attributes={}), + "group-v2-attributes-unset": ZarrV2GroupMetadata.create_default(), + "group-v2-attributes-set": ZarrV2GroupMetadata.create_default(attributes={"a": 1}), + "group-v3-consolidated-unset": ZarrV3GroupMetadata.create_default(), + "group-v3-consolidated-with-unset-inside": ZarrV3GroupMetadata.create_default( + consolidated_metadata=ZarrV3ConsolidatedMetadata( + metadata={ + "child": ZarrV3ArrayMetadata.create_default(shape=(4,)), + "subgroup": ZarrV3GroupMetadata.create_default(), + } + ) + ), +} + + +def test_unset_pickle_round_trip_preserves_identity() -> None: + restored = pickle.loads(pickle.dumps(UNSET)) + assert restored is UNSET + + +def test_unset_copy_preserves_identity() -> None: + assert copy.copy(UNSET) is UNSET + assert copy.deepcopy(UNSET) is UNSET + + +@pytest.mark.parametrize("model", MODEL_CASES.values(), ids=MODEL_CASES.keys()) +def test_model_pickle_round_trip( + model: ZarrV2ArrayMetadata | ZarrV3ArrayMetadata | ZarrV2GroupMetadata | ZarrV3GroupMetadata, +) -> None: + restored = pickle.loads(pickle.dumps(model)) + assert restored == model + + +@pytest.mark.parametrize("model", MODEL_CASES.values(), ids=MODEL_CASES.keys()) +def test_model_deepcopy( + model: ZarrV2ArrayMetadata | ZarrV3ArrayMetadata | ZarrV2GroupMetadata | ZarrV3GroupMetadata, +) -> None: + assert copy.deepcopy(model) == model + + +def test_non_importable_sentinel_fails_to_pickle() -> None: + """Sentinels pickle by reference, never by state. A sentinel that is not + an importable attribute of its module has no reference to pickle, so + dumping it must fail loudly — a successful dump here would mean the + implementation regressed to state-based pickling, which would produce + identity-breaking impostor objects on the receiving side.""" + local_sentinel = Sentinel("local_sentinel") + with pytest.raises((pickle.PicklingError, TypeError)): + pickle.dumps(local_sentinel) diff --git a/packages/zarr-metadata/tests/test_partial_equivalence.py b/packages/zarr-metadata/tests/test_partial_equivalence.py index 995a6a21e1..33492b2356 100644 --- a/packages/zarr-metadata/tests/test_partial_equivalence.py +++ b/packages/zarr-metadata/tests/test_partial_equivalence.py @@ -14,17 +14,17 @@ import pytest -from zarr_metadata.v2.array import ArrayMetadataV2, ArrayMetadataV2Partial -from zarr_metadata.v2.group import GroupMetadataV2, GroupMetadataV2Partial -from zarr_metadata.v3.array import ArrayMetadataV3, ArrayMetadataV3Partial -from zarr_metadata.v3.group import GroupMetadataV3, GroupMetadataV3Partial +from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON, ZarrV2ArrayMetadataJSONPartial +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON, ZarrV2GroupMetadataJSONPartial +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON, ZarrV3ArrayMetadataJSONPartial +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON, ZarrV3GroupMetadataJSONPartial # (full, partial) pairs to check. Add new pairs here as more are introduced. PAIRS: list[tuple[type, type]] = [ - (ArrayMetadataV3, ArrayMetadataV3Partial), - (GroupMetadataV3, GroupMetadataV3Partial), - (ArrayMetadataV2, ArrayMetadataV2Partial), - (GroupMetadataV2, GroupMetadataV2Partial), + (ZarrV3ArrayMetadataJSON, ZarrV3ArrayMetadataJSONPartial), + (ZarrV3GroupMetadataJSON, ZarrV3GroupMetadataJSONPartial), + (ZarrV2ArrayMetadataJSON, ZarrV2ArrayMetadataJSONPartial), + (ZarrV2GroupMetadataJSON, ZarrV2GroupMetadataJSONPartial), ] diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index d3270579c3..e65c680fd1 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -1,5 +1,7 @@ """Test that the curated front-door names are accessible from the top-level zarr_metadata package.""" +import importlib +import pkgutil import re from typing import get_args @@ -21,26 +23,43 @@ def _group_rank(s: str) -> int: EXPECTED = [ # Category A — metadata-document types - "ArrayMetadataV2", - "ArrayMetadataV2Partial", - "ZArrayMetadata", - "GroupMetadataV2", - "GroupMetadataV2Partial", - "ZGroupMetadata", - "ConsolidatedMetadataV2", - "ZAttrsMetadata", - "CodecMetadataV2", - "ArrayMetadataV3", - "ArrayMetadataV3Partial", - "ExtensionFieldV3", - "GroupMetadataV3", - "GroupMetadataV3Partial", - "ConsolidatedMetadataV3", - "NamedConfigV3", - "MetadataV3", + "ZarrV2ArrayMetadataJSON", + "ZarrV2ArrayMetadataJSONPartial", + "ZarrV2ZArrayJSON", + "ZarrV2GroupMetadataJSON", + "ZarrV2GroupMetadataJSONPartial", + "ZarrV2ZGroupJSON", + "ZarrV2ConsolidatedMetadataJSON", + "ZarrV2ZAttrsJSON", + "ZarrV2CodecMetadata", + "ZarrV3ArrayMetadataJSON", + "ZarrV3ArrayMetadataJSONPartial", + "ZarrV3ExtensionField", + "ZarrV3GroupMetadataJSON", + "ZarrV3GroupMetadataJSONPartial", + "ZarrV3ConsolidatedMetadataJSON", + "ZarrV3NamedConfigJSON", + "ZarrV3MetadataFieldJSON", "JSONValue", + # Category A' — metadata models (in-memory dataclasses over the documents) + "ZarrV2ArrayMetadata", + "ZarrV2ArrayMetadataPartial", + "ZarrV3ArrayMetadata", + "ZarrV3ArrayMetadataPartial", + "ZarrV2GroupMetadata", + "ZarrV2GroupMetadataPartial", + "ZarrV3GroupMetadata", + "ZarrV3GroupMetadataPartial", + "ZarrV2ConsolidatedMetadata", + "ZarrV3ConsolidatedMetadata", + "ZarrV3NamedConfig", + "ZarrV3MetadataField", + "ValidationProblem", + "MetadataValidationError", + "ProblemKind", + "UNSET", # v2 data-type encoding union - "DataTypeMetadataV2", + "ZarrV2DataTypeMetadata", # Category B — codec canonical unions "BloscCodecMetadata", "BytesCodecMetadata", @@ -129,9 +148,9 @@ def _group_rank(s: str) -> int: "RawBytesFillValue", # Category E — constant+Literal pairs "ARRAY_ORDER_V2", - "ArrayOrderV2", + "ZarrV2ArrayOrder", "ARRAY_DIMENSION_SEPARATOR_V2", - "ArrayDimensionSeparatorV2", + "ZarrV2ArrayDimensionSeparator", "ENDIANNESS", "Endianness", "BYTES_CODEC_NAME", @@ -200,6 +219,109 @@ def test_all_is_grouped_and_unique() -> None: assert len(zm.__all__) == len(set(zm.__all__)) +# --- naming grammar --------------------------------------------------------- + +# Core document/model names: the format version comes first (`ZarrV2` / +# `ZarrV3`), then the CamelCase entity, then an optional role suffix +# (`JSON`, `JSONPartial`, `Partial`, `StoreKey`) — validated loosely here +# because `JSON` decomposes into single-letter words under any strict +# word-splitting regex. +_CORE_NAME = re.compile(r"^ZarrV[23](?:[A-Z][a-z0-9]*)+$") + +# Zarr v3 extension-entity names: the registered entity comes first (`Blosc`, +# `Uint8`, ... — `V2` here is the *entity name* of the v2-compatibility chunk +# key encoding, not a format-version marker, which is always spelled +# `ZarrV2`/`ZarrV3`), followed by exactly one role suffix. +_EXTENSION_ROLES = ( + "CodecConfiguration", + "CodecMetadata", + "CodecName", + "CodecObject", + "ChunkGridConfiguration", + "ChunkGridMetadata", + "ChunkGridName", + "ChunkGridObject", + "ChunkKeyEncodingConfiguration", + "ChunkKeyEncodingMetadata", + "ChunkKeyEncodingName", + "ChunkKeyEncodingObject", + "ChunkKeyEncodingSeparator", + "DataTypeName", + "FillValue", + "Configuration", + "Component", +) +_EXTENSION_NAME = re.compile(r"^(?:[A-Z][a-z0-9]*)+?(?:" + "|".join(_EXTENSION_ROLES) + r")$") + +# Standalone vocabulary: scalar Literal aliases, structural helper shapes, and +# the validation diagnostics. Closed by hand — a new name belongs here only if +# it is genuinely role-less; anything document- or entity-shaped must fit the +# grammars above instead. +_STANDALONE_VOCAB = frozenset( + { + "Base64Bytes", + "BloscCName", + "BloscShuffle", + "CastOutOfRangeMode", + "CastRoundingMode", + "Endianness", + "HexFloat16", + "HexFloat32", + "HexFloat64", + "JSONValue", + "MetadataValidationError", + "NumpyDatetime64", + "NumpyTimeUnit", + "NumpyTimedelta64", + "ProblemKind", + "RectilinearDimSpec", + "ScalarMap", + "ScalarMapEntry", + "ShardingIndexLocation", + "Struct", + "StructField", + "ValidationProblem", + } +) + + +def _public_type_names() -> set[tuple[str, str]]: + """Every (module, CamelCase name) pair exported via a public `__all__`.""" + module_names = {"zarr_metadata"} + for info in pkgutil.walk_packages(zm.__path__, prefix="zarr_metadata."): + if not any(part.startswith("_") for part in info.name.split(".")[1:]): + module_names.add(info.name) + out: set[tuple[str, str]] = set() + for module_name in module_names: + module = importlib.import_module(module_name) + for name in getattr(module, "__all__", ()): + if name.startswith("_") or name.isupper() or name.islower(): + continue + out.add((module_name, name)) + return out + + +def test_public_type_names_comply_with_naming_grammar() -> None: + """Every public type name parses against the package naming grammar: + version-first core names, entity-plus-role extension names, or the closed + standalone vocabulary.""" + exported = _public_type_names() + violations = [ + f"{module}.{name}" + for module, name in sorted(exported) + if name not in _STANDALONE_VOCAB + and not _CORE_NAME.match(name) + and not _EXTENSION_NAME.match(name) + ] + assert not violations, f"names outside the naming grammar: {violations}" + + +def test_standalone_vocab_is_not_stale() -> None: + """Every allowlisted vocabulary name is still actually exported.""" + exported_names = {name for _, name in _public_type_names()} + assert exported_names >= _STANDALONE_VOCAB + + def test_promoted_pairs_drift() -> None: pairs = [ (zm.ENDIANNESS, zm.Endianness), @@ -209,7 +331,7 @@ def test_promoted_pairs_drift() -> None: (zm.NUMPY_TIME_UNIT, zm.NumpyTimeUnit), (zm.CAST_ROUNDING_MODE, zm.CastRoundingMode), (zm.CAST_OUT_OF_RANGE_MODE, zm.CastOutOfRangeMode), - (zm.ARRAY_ORDER_V2, zm.ArrayOrderV2), + (zm.ARRAY_ORDER_V2, zm.ZarrV2ArrayOrder), ] for const, lit in pairs: assert set(const) == set(get_args(lit)) diff --git a/packages/zarr-metadata/tests/v2/array/test_fixtures.py b/packages/zarr-metadata/tests/v2/array/test_fixtures.py index 1576aae8db..578d3647ab 100644 --- a/packages/zarr-metadata/tests/v2/array/test_fixtures.py +++ b/packages/zarr-metadata/tests/v2/array/test_fixtures.py @@ -1,7 +1,7 @@ """Decode v2 array metadata fixtures via pydantic. Each `*.json` file in this directory is a representative on-disk -`.zarray` that should validate cleanly as `ZArrayMetadata` (the strict +`.zarray` that should validate cleanly as `ZarrV2ZArrayJSON` (the strict on-disk shape). User attributes live in sibling `.zattrs` files and are not part of these fixtures. @@ -17,11 +17,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v2.array import ZArrayMetadata +from zarr_metadata.v2.array import ZarrV2ZArrayJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(ZArrayMetadata) +ADAPTER = TypeAdapter(ZarrV2ZArrayJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) diff --git a/packages/zarr-metadata/tests/v2/consolidated/test_fixtures.py b/packages/zarr-metadata/tests/v2/consolidated/test_fixtures.py index 9dad66d074..e802c5bef8 100644 --- a/packages/zarr-metadata/tests/v2/consolidated/test_fixtures.py +++ b/packages/zarr-metadata/tests/v2/consolidated/test_fixtures.py @@ -8,11 +8,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v2.consolidated import ConsolidatedMetadataV2 +from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(ConsolidatedMetadataV2) +ADAPTER = TypeAdapter(ZarrV2ConsolidatedMetadataJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) diff --git a/packages/zarr-metadata/tests/v2/group/test_fixtures.py b/packages/zarr-metadata/tests/v2/group/test_fixtures.py index 1ad88a577b..29652185b2 100644 --- a/packages/zarr-metadata/tests/v2/group/test_fixtures.py +++ b/packages/zarr-metadata/tests/v2/group/test_fixtures.py @@ -1,7 +1,7 @@ """Decode v2 group metadata fixtures via pydantic. Each `*.json` file in this directory is a representative on-disk -`.zgroup` that should validate cleanly as `ZGroupMetadata` (the strict +`.zgroup` that should validate cleanly as `ZarrV2ZGroupJSON` (the strict on-disk shape). User attributes live in sibling `.zattrs` files and are not part of these fixtures. """ @@ -14,11 +14,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v2.group import ZGroupMetadata +from zarr_metadata.v2.group import ZarrV2ZGroupJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(ZGroupMetadata) +ADAPTER = TypeAdapter(ZarrV2ZGroupJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) diff --git a/packages/zarr-metadata/tests/v3/array/test_fixtures.py b/packages/zarr-metadata/tests/v3/array/test_fixtures.py index fccd00d481..c84cc4042b 100644 --- a/packages/zarr-metadata/tests/v3/array/test_fixtures.py +++ b/packages/zarr-metadata/tests/v3/array/test_fixtures.py @@ -1,7 +1,7 @@ """Decode v3 array metadata fixtures via pydantic. Each `*.json` file in this directory is a representative on-disk -`zarr.json` that should validate cleanly as `ArrayMetadataV3`. +`zarr.json` that should validate cleanly as `ZarrV3ArrayMetadataJSON`. Fixtures are named for the variant they exercise (regular vs rectilinear grid, blosc/gzip/zstd/sharding_indexed codecs, named-config dtypes, optional fields, extra fields). @@ -15,11 +15,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v3.array import ArrayMetadataV3 +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(ArrayMetadataV3) +ADAPTER = TypeAdapter(ZarrV3ArrayMetadataJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) diff --git a/packages/zarr-metadata/tests/v3/array/with_extra_field.json b/packages/zarr-metadata/tests/v3/array/with_extra_field.json index 46a7f0f235..bd7a9f5b45 100644 --- a/packages/zarr-metadata/tests/v3/array/with_extra_field.json +++ b/packages/zarr-metadata/tests/v3/array/with_extra_field.json @@ -16,6 +16,6 @@ ], "my_custom_extension": { "must_understand": false, - "purpose": "exercise the extra_items=ExtensionFieldV3 path" + "purpose": "exercise the extra_items=ZarrV3ExtensionField path" } } diff --git a/packages/zarr-metadata/tests/v3/consolidated/test_fixtures.py b/packages/zarr-metadata/tests/v3/consolidated/test_fixtures.py index 4d9e300bae..d052b16986 100644 --- a/packages/zarr-metadata/tests/v3/consolidated/test_fixtures.py +++ b/packages/zarr-metadata/tests/v3/consolidated/test_fixtures.py @@ -8,11 +8,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v3.consolidated import ConsolidatedMetadataV3 +from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(ConsolidatedMetadataV3) +ADAPTER = TypeAdapter(ZarrV3ConsolidatedMetadataJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) diff --git a/packages/zarr-metadata/tests/v3/group/test_fixtures.py b/packages/zarr-metadata/tests/v3/group/test_fixtures.py index 2015d5ce96..ffcdedef2b 100644 --- a/packages/zarr-metadata/tests/v3/group/test_fixtures.py +++ b/packages/zarr-metadata/tests/v3/group/test_fixtures.py @@ -8,11 +8,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v3.group import GroupMetadataV3 +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(GroupMetadataV3) +ADAPTER = TypeAdapter(ZarrV3GroupMetadataJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) From 0c07bf5ed6619b7e5c0b01aece7a93fc88de07b6 Mon Sep 17 00:00:00 2001 From: Goutam Adwant <8672451+goutamadwant@users.noreply.github.com> Date: Tue, 28 Jul 2026 22:24:08 +0200 Subject: [PATCH 23/24] fix: let open_like create arrays by default (#4146) * fix: let open_like create arrays by default * test: cover open_like read-only mode --------- Co-authored-by: Davis Bennett --- changes/3352.bugfix.md | 3 +++ src/zarr/api/asynchronous.py | 6 ++++- src/zarr/api/synchronous.py | 6 +++-- tests/test_api.py | 47 ++++++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 changes/3352.bugfix.md diff --git a/changes/3352.bugfix.md b/changes/3352.bugfix.md new file mode 100644 index 0000000000..7461486776 --- /dev/null +++ b/changes/3352.bugfix.md @@ -0,0 +1,3 @@ +Fix `zarr.api.asynchronous.open_like` so it can create a new array by default when the +target path does not already exist. It now defaults to `mode="a"`; when using a read-only +store to open an existing array, pass `mode="r"` explicitly. diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index c751d6a31c..f5e614a051 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -1292,7 +1292,9 @@ async def open_like(a: ArrayLike, path: str, **kwargs: Any) -> AnyAsyncArray: path : str The path to the new array. **kwargs - Any keyword arguments to pass to the array constructor. + Additional keyword arguments passed to `open_array`. + If `mode` is omitted or `None`, it defaults to `"a"`. Pass `mode="r"` when + opening an existing array from a read-only store. Returns ------- @@ -1300,6 +1302,8 @@ async def open_like(a: ArrayLike, path: str, **kwargs: Any) -> AnyAsyncArray: The opened array. """ like_kwargs = _like_args(a) | kwargs + if like_kwargs.get("mode") is None: + like_kwargs["mode"] = "a" return await open_array(path=path, **like_kwargs) # type: ignore[arg-type] diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 3231837a04..dc12d5f7af 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -1399,11 +1399,13 @@ def open_like(a: ArrayLike, path: str, **kwargs: Any) -> AnyArray: path : str The path to the new array. **kwargs - Any keyword arguments to pass to the array constructor. + Additional keyword arguments passed to `open_array`. + If `mode` is omitted or `None`, it defaults to `"a"`. Pass `mode="r"` when + opening an existing array from a read-only store. Returns ------- - AsyncArray + Array The opened array. """ return Array(sync(async_api.open_like(a, path=path, **kwargs))) diff --git a/tests/test_api.py b/tests/test_api.py index 1b4414ae63..cbe8ea3b44 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -168,6 +168,53 @@ async def test_array_like_creation( assert np.all(Array(new_arr)[:] == expect_fill) +@pytest.mark.parametrize("mode_kwargs", [{}, {"mode": None}]) +async def test_open_like_creates_array_by_default( + zarr_format: ZarrFormat, mode_kwargs: dict[str, None] +) -> None: + ref_arr = zarr.create_array( + store={}, + shape=(11, 12), + dtype="uint8", + chunks=(11, 12), + zarr_format=zarr_format, + fill_value=100, + ) + + new_arr = await zarr.api.asynchronous.open_like( + ref_arr, + path="foo", + store={}, + zarr_format=zarr_format, + **mode_kwargs, + ) + + assert new_arr.shape == ref_arr.shape + assert new_arr.chunks == ref_arr.chunks + assert new_arr.dtype == ref_arr.dtype + assert np.all(Array(new_arr)[:] == ref_arr.fill_value) + + +async def test_open_like_default_mode_rejects_read_only_store( + zarr_format: ZarrFormat, +) -> None: + ref_arr = zarr.create_array( + store={}, + shape=(11, 12), + dtype="uint8", + chunks=(11, 12), + zarr_format=zarr_format, + ) + + with pytest.raises(ValueError, match="Store is read-only but mode is 'a'"): + await zarr.api.asynchronous.open_like( + ref_arr, + path="foo", + store=MemoryStore(read_only=True), + zarr_format=zarr_format, + ) + + # TODO: parametrize over everything this function takes @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_create_array(store: Store, zarr_format: ZarrFormat) -> None: From 417403857b81e69ba8f0868ac607851291d76d72 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 28 Jul 2026 22:41:03 +0200 Subject: [PATCH 24/24] docs: bring the 4191 changelog entry up to date Merge upstream/main (#4146 open_like, #4119 zarr-metadata model layer) and refresh this PR's news fragment so it describes what the branch actually contains now: - The pipeline example no longer sweeps "several array layouts"; it compares the two pipelines on one sharded array across two stores and two codec regimes, which is what shows the thread-pool tradeoff. - Mention the removal of the hardware-specific speedup figures from the FusedCodecPipeline release note, which is part of this PR's diff but went undescribed. Both incoming PRs carried their own fragments (changes/3352.bugfix.md and packages/zarr-metadata/changes/4119.*), so no other entry was needed. Assisted-by: ClaudeCode:claude-opus-5 --- changes/4191.doc.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/changes/4191.doc.md b/changes/4191.doc.md index cb1398dd7c..fb01f9141f 100644 --- a/changes/4191.doc.md +++ b/changes/4191.doc.md @@ -4,6 +4,11 @@ coalescing for partial reads of sharded arrays. Added two runnable examples that accompany the post: `examples/codec_pipeline_performance` compares the `BatchedCodecPipeline` and -`FusedCodecPipeline` across several array layouts, and -`examples/sharding_coalescing` demonstrates how read coalescing reduces the -number of store requests when reading subregions of a sharded array. +`FusedCodecPipeline` on a sharded array across two stores and two codec +regimes, showing when the fused pipeline's thread pool helps and when it does +not, and `examples/sharding_coalescing` demonstrates how read coalescing +reduces the number of store requests when reading subregions of a sharded +array. + +Also removed the hardware-specific speedup figures from the `FusedCodecPipeline` +release note, since they depend on the array layout, codec, and machine.