From 8da13336dfa60c2b4412097bc5e58488aebbb66c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 31 May 2026 19:28:07 +0200 Subject: [PATCH 01/70] chore(deps): bump the actions group across 1 directory with 8 updates (#176) Bumps the actions group with 8 updates in the / directory: | Package | From | To | | --- | --- | --- | | [prefix-dev/setup-pixi](https://github.com/prefix-dev/setup-pixi) | `0.9.5` | `0.9.6` | | [codecov/codecov-action](https://github.com/codecov/codecov-action) | `6.0.0` | `6.0.1` | | [github/issue-metrics](https://github.com/github/issue-metrics) | `4.2.2` | `4.2.7` | | [j178/prek-action](https://github.com/j178/prek-action) | `2.0.3` | `2.0.4` | | [actions/upload-artifact](https://github.com/actions/upload-artifact) | `7.0.0` | `7.0.1` | | [actions/download-artifact](https://github.com/actions/download-artifact) | `7.0.0` | `8.0.1` | | [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) | `1.13.0` | `1.14.0` | | [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) | `0.5.3` | `0.5.6` | Updates `prefix-dev/setup-pixi` from 0.9.5 to 0.9.6 - [Release notes](https://github.com/prefix-dev/setup-pixi/releases) - [Commits](https://github.com/prefix-dev/setup-pixi/compare/1b2de7f3351f171c8b4dfeb558c639cb58ed4ec0...5185adfbffb4bd703da3010310260805d89ebb11) Updates `codecov/codecov-action` from 6.0.0 to 6.0.1 - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/57e3a136b779b570ffcdbf80b3bdc90e7fab3de2...e79a6962e0d4c0c17b229090214935d2e33f8354) Updates `github/issue-metrics` from 4.2.2 to 4.2.7 - [Release notes](https://github.com/github/issue-metrics/releases) - [Commits](https://github.com/github/issue-metrics/compare/c9e9838147fd355dace335ba787f01b6641a400a...1e38d5e62363e14db8019ed7d106b9855bdba6cc) Updates `j178/prek-action` from 2.0.3 to 2.0.4 - [Release notes](https://github.com/j178/prek-action/releases) - [Commits](https://github.com/j178/prek-action/compare/6ad80277337ad479fe43bd70701c3f7f8aa74db3...bdca6f102f98e2b4c7029491a53dfd366469e33d) Updates `actions/upload-artifact` from 7.0.0 to 7.0.1 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v7...043fb46d1a93c77aae656e7c1c64a875d1fc6a0a) Updates `actions/download-artifact` from 7.0.0 to 8.0.1 - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v7...3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c) Updates `pypa/gh-action-pypi-publish` from 1.13.0 to 1.14.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.13.0...cef221092ed1bacb1cc03d23a2d87d1d172e277b) Updates `zizmorcore/zizmor-action` from 0.5.3 to 0.5.6 - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/b1d7e1fb5de872772f31590499237e7cce841e8e...5f14fd08f7cf1cb1609c1e344975f152c7ee938d) --- updated-dependencies: - dependency-name: prefix-dev/setup-pixi dependency-version: 0.9.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: codecov/codecov-action dependency-version: 6.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: github/issue-metrics dependency-version: 4.2.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: j178/prek-action dependency-version: 2.0.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/upload-artifact dependency-version: 7.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/download-artifact dependency-version: 8.0.1 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: pypa/gh-action-pypi-publish dependency-version: 1.14.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: zizmorcore/zizmor-action dependency-version: 0.5.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/downstream.yml | 2 +- .github/workflows/gpu_test.yml | 2 +- .github/workflows/hypothesis.yaml | 2 +- .github/workflows/issue-metrics.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/test.yml | 4 ++-- .github/workflows/zarr-metadata-release.yml | 12 ++++++------ .github/workflows/zizmor.yml | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index 74026233c4..3eb6898895 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -34,7 +34,7 @@ jobs: persist-credentials: false - name: Set up pixi - uses: prefix-dev/setup-pixi@1b2de7f3351f171c8b4dfeb558c639cb58ed4ec0 # v0.9.5 + uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6 with: manifest-path: xarray/pixi.toml diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index 403441b306..333769cb9e 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -76,7 +76,7 @@ jobs: hatch env run --env "$HATCH_ENV" run-coverage - name: Upload coverage - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: gpu diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index 4f9467be7d..a456b2aa0a 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -93,7 +93,7 @@ jobs: key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} - name: Upload coverage - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: tests diff --git a/.github/workflows/issue-metrics.yml b/.github/workflows/issue-metrics.yml index 14fba5b9ec..510849ef3e 100644 --- a/.github/workflows/issue-metrics.yml +++ b/.github/workflows/issue-metrics.yml @@ -33,7 +33,7 @@ jobs: echo "last_month=$first_day..$last_day" >> "$GITHUB_ENV" - name: Run issue-metrics tool - uses: github/issue-metrics@c9e9838147fd355dace335ba787f01b6641a400a # v4.2.2 + uses: github/issue-metrics@1e38d5e62363e14db8019ed7d106b9855bdba6cc # v4.2.7 env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} SEARCH_QUERY: 'repo:zarr-developers/zarr-python is:issue created:${{ env.last_month }} -reason:"not planned"' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 768e660ec2..fec211b4dd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -30,4 +30,4 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: enable-cache: true - - uses: j178/prek-action@6ad80277337ad479fe43bd70701c3f7f8aa74db3 # v2.0.3 + - uses: j178/prek-action@bdca6f102f98e2b4c7029491a53dfd366469e33d # v2.0.4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 03143d3e5b..62e571856b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -78,7 +78,7 @@ jobs: hatch env run --env "$HATCH_ENV" run-coverage - name: Upload coverage if: ${{ matrix.dependency-set == 'optional' && matrix.os == 'ubuntu-latest' }} - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: tests @@ -125,7 +125,7 @@ jobs: run: | hatch env run --env "$HATCH_ENV" run-coverage - name: Upload coverage - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} flags: tests diff --git a/.github/workflows/zarr-metadata-release.yml b/.github/workflows/zarr-metadata-release.yml index 809d502f16..9639fcfdd3 100644 --- a/.github/workflows/zarr-metadata-release.yml +++ b/.github/workflows/zarr-metadata-release.yml @@ -35,7 +35,7 @@ jobs: - name: Build run: hatch build - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: zarr-metadata-dist path: packages/zarr-metadata/dist @@ -45,7 +45,7 @@ jobs: needs: [build] runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: zarr-metadata-dist path: dist @@ -76,7 +76,7 @@ jobs: id-token: write # required for OIDC trusted publishing attestations: write # required for artifact attestations steps: - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: zarr-metadata-dist path: dist @@ -87,7 +87,7 @@ jobs: subject-path: dist/* - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 upload_testpypi: name: Upload to TestPyPI @@ -101,7 +101,7 @@ jobs: id-token: write attestations: write steps: - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: zarr-metadata-dist path: dist @@ -112,6 +112,6 @@ jobs: subject-path: dist/* - name: Publish package to TestPyPI - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 with: repository-url: https://test.pypi.org/legacy/ diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index da19f22421..7ac4fe5d0e 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -32,4 +32,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3 + uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6 From ce2cfd7ba44bc6f9b44e63ec372e026a4860c409 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 4 Jul 2026 22:41:46 +0200 Subject: [PATCH 02/70] =?UTF-8?q?feat(zarr-metadata):=20add=20model.=5Fval?= =?UTF-8?q?idation=20=E2=80=94=20structural=20validators?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/__init__.py | 1 + .../src/zarr_metadata/model/_validation.py | 271 ++++++++++++++++++ 2 files changed, 272 insertions(+) create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/__init__.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/_validation.py diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py new file mode 100644 index 0000000000..e1382a6f0a --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -0,0 +1 @@ +"""In-memory models for Zarr metadata documents.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py new file mode 100644 index 0000000000..d0e30e1cda --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -0,0 +1,271 @@ +"""Structural validation for Zarr metadata documents. + +Validators check JSON structure (shapes, key presence, primitive kinds), +not domain validity. Each concept gets a `validate_*` function returning +every problem found, an `is_*` type guard, and a `parse_*` function that +narrows or raises `MetadataValidationError`. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Final, cast + +from typing_extensions import TypeIs + +from zarr_metadata import ArrayMetadataV2, ArrayMetadataV3, MetadataV3 +from zarr_metadata._common import JSONValue + + +@dataclass(frozen=True, slots=True) +class ValidationProblem: + """A single structural problem found while validating a metadata document. + + `loc` is the path from the document root to the offending value, e.g. + `("codecs", 0, "name")`. An empty `loc` refers to the document as a whole. + """ + + loc: tuple[str | int, ...] + message: str + + def __str__(self) -> str: + location = ".".join(str(part) for part in self.loc) if self.loc else "" + return f"{location}: {self.message}" + + +class MetadataValidationError(ValueError): + """Raised when a value fails structural metadata validation. + + Carries every problem found (not just the first) in `.problems`. + """ + + def __init__(self, problems: list[ValidationProblem]) -> None: + self.problems = problems + super().__init__("\n".join(str(problem) for problem in problems)) + + +def _prefix(loc_head: str | int, problems: list[ValidationProblem]) -> list[ValidationProblem]: + """Prepend `loc_head` to the `loc` of every problem (for nested validators).""" + return [ValidationProblem((loc_head, *p.loc), p.message) for p in problems] + + +def validate_json(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not JSON-serializable (recursively).""" + if isinstance(value, (str, int, float, bool)) or value is None: + return [] + problems: list[ValidationProblem] = [] + if isinstance(value, Mapping): + for key, item in cast("Mapping[object, object]", value).items(): + if not isinstance(key, str): + problems.append(ValidationProblem((), f"non-string key {key!r} in JSON object")) + continue + problems.extend(_prefix(key, validate_json(item))) + return problems + if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): + for index, item in enumerate(cast("Sequence[object]", value)): + problems.extend(_prefix(index, validate_json(item))) + return problems + return [ValidationProblem((), f"not a JSON-serializable value: {value!r}")] + + +def is_json(value: object) -> TypeIs[JSONValue]: + """Whether `value` is a JSON-serializable structure (recursively).""" + return not validate_json(value) + + +def parse_json(value: object) -> JSONValue: + """Return `value` narrowed to `JSONValue`, or raise `MetadataValidationError`.""" + problems = validate_json(value) + if problems: + raise MetadataValidationError(problems) + return cast(JSONValue, value) + + +# The standard top-level keys of a v3 array metadata document. Anything outside +# this set is an extension field. Built from the TypedDict's required/optional +# key sets (which resolve inherited keys, unlike `__annotations__`). +ARRAY_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( + ArrayMetadataV3.__required_keys__ +) +ARRAY_METADATA_OPTIONAL_KEYS_V3: Final[frozenset[str]] = frozenset( + ArrayMetadataV3.__optional_keys__ +) +ARRAY_METADATA_STANDARD_KEYS_V3: Final[frozenset[str]] = ( + ARRAY_METADATA_REQUIRED_KEYS_V3 | ARRAY_METADATA_OPTIONAL_KEYS_V3 +) + +ARRAY_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( + ArrayMetadataV2.__required_keys__ +) + + +def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a v3 metadata field. + + A metadata field is a bare name string or a `{name, configuration}` mapping. + """ + if isinstance(value, str): + return [] + if not isinstance(value, Mapping): + return [ + ValidationProblem((), "expected a metadata field (string or {name, configuration})") + ] + field = cast("Mapping[object, object]", value) + problems: list[ValidationProblem] = [] + if not isinstance(field.get("name"), str): + problems.append(ValidationProblem(("name",), "expected a string name")) + if "configuration" in field: + configuration = field["configuration"] + if not isinstance(configuration, Mapping): + problems.append(ValidationProblem(("configuration",), "expected a mapping")) + elif not all(isinstance(k, str) for k in cast("Mapping[object, object]", configuration)): + problems.append(ValidationProblem(("configuration",), "expected string keys")) + return problems + + +def is_metadata_field_v3(value: object) -> TypeIs[MetadataV3]: + """Whether `value` is a v3 metadata field: a bare name or a named config.""" + return not validate_metadata_field_v3(value) + + +def parse_metadata_field_v3(value: object) -> MetadataV3: + """Return `value` narrowed to `MetadataV3`, or raise `MetadataValidationError`.""" + problems = validate_metadata_field_v3(value) + if problems: + raise MetadataValidationError(problems) + return cast(MetadataV3, value) + + +def _is_int_sequence(value: object) -> bool: + """Whether `value` is a non-string sequence of integers.""" + return ( + not isinstance(value, str) + and isinstance(value, Sequence) + and all(isinstance(item, int) for item in cast("Sequence[object]", value)) + ) + + +def _validate_attributes(value: object) -> list[ValidationProblem]: + """Validate an `attributes` value: a mapping with string keys. + + Returns a problem at `("attributes",)` if it is not, else `[]`. Shared by the + v2 and v3 validators. Unlike the other `validate_*` functions (which + return value-relative locs for the caller to `_prefix`), this emits the + already-parent-relative `("attributes",)` loc, since it is only ever called + with a document's `attributes` value. + """ + if not isinstance(value, Mapping) or not all( + isinstance(k, str) for k in cast("Mapping[object, object]", value) + ): + return [ValidationProblem(("attributes",), "expected a mapping with string keys")] + return [] + + +def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v3 array doc. + + Checks structure, not domain validity. Unknown top-level keys are allowed + (they map to `extra_fields`). + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key") + for key in sorted(ARRAY_METADATA_REQUIRED_KEYS_V3 - doc.keys()) + ] + if "shape" in doc and not _is_int_sequence(doc["shape"]): + problems.append(ValidationProblem(("shape",), "expected a sequence of int")) + if "fill_value" in doc: + problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) + for key in ("data_type", "chunk_grid", "chunk_key_encoding"): + if key in doc: + problems.extend(_prefix(key, validate_metadata_field_v3(doc[key]))) + for key in ("codecs", "storage_transformers"): + if key in doc: + entries = doc[key] + if isinstance(entries, str) or not isinstance(entries, Sequence): + problems.append(ValidationProblem((key,), "expected a sequence")) + else: + for index, entry in enumerate(cast("Sequence[object]", entries)): + problems.extend(_prefix(key, _prefix(index, validate_metadata_field_v3(entry)))) + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + if "dimension_names" in doc: + # Simple typed sequences (dimension_names, shape, chunks) report a single + # field-level loc, not per-bad-item locs; per-index locs are reserved for + # the metadata-field lists (codecs, storage_transformers). + names = doc["dimension_names"] + if isinstance(names, str) or not isinstance(names, Sequence): + problems.append(ValidationProblem(("dimension_names",), "expected a sequence")) + elif not all( + item is None or isinstance(item, str) for item in cast("Sequence[object]", names) + ): + problems.append( + ValidationProblem(("dimension_names",), "expected items of str or None") + ) + return problems + + +def is_array_metadata_v3(value: object) -> TypeIs[ArrayMetadataV3]: + """Whether `value` is a structurally-valid v3 array metadata document.""" + return not validate_array_metadata_v3(value) + + +def parse_array_metadata_v3(value: object) -> ArrayMetadataV3: + """Return `value` narrowed to `ArrayMetadataV3`, or raise `MetadataValidationError`.""" + problems = validate_array_metadata_v3(value) + if problems: + raise MetadataValidationError(problems) + return cast(ArrayMetadataV3, value) + + +def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v2 array doc. + + Checks structure, not domain validity. `compressor`/`filters` are required + keys but may be `None`. + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key") + for key in sorted(ARRAY_METADATA_REQUIRED_KEYS_V2 - doc.keys()) + ] + problems.extend( + ValidationProblem((key,), "expected a sequence of int") + for key in ("shape", "chunks") + if key in doc and not _is_int_sequence(doc[key]) + ) + if "fill_value" in doc: + problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + return problems + + +def is_array_metadata_v2(value: object) -> TypeIs[ArrayMetadataV2]: + """Whether `value` is a structurally-valid v2 array metadata document.""" + return not validate_array_metadata_v2(value) + + +def parse_array_metadata_v2(value: object) -> ArrayMetadataV2: + """Return `value` narrowed to `ArrayMetadataV2`, or raise `MetadataValidationError`.""" + problems = validate_array_metadata_v2(value) + if problems: + raise MetadataValidationError(problems) + return cast(ArrayMetadataV2, value) + + +def _arrays_to_tuples(obj: object) -> object: + """Recursively convert every list in a JSON-decoded structure to a tuple.""" + if isinstance(obj, list): + return tuple(_arrays_to_tuples(item) for item in cast("list[object]", obj)) + if isinstance(obj, dict): + return { + key: _arrays_to_tuples(value) + for key, value in cast("dict[object, object]", obj).items() + } + return obj From eee768bdb419fe9c4f4f703da61956f54e176a77 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 10:19:35 +0200 Subject: [PATCH 03/70] =?UTF-8?q?feat(zarr-metadata):=20add=20model.=5Farr?= =?UTF-8?q?ay=20=E2=80=94=20array=20metadata=20models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/_array.py | 348 ++++++++++++++++++ .../src/zarr_metadata/model/_validation.py | 7 +- 2 files changed, 351 insertions(+), 4 deletions(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/_array.py diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py new file mode 100644 index 0000000000..ea2745284d --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -0,0 +1,348 @@ +"""In-memory models for Zarr array metadata documents.""" + +from __future__ import annotations + +import dataclasses +import json +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Final, Literal + +from typing_extensions import TypedDict, Unpack + +from zarr_metadata.model._validation import ( + ARRAY_METADATA_STANDARD_KEYS_V3, + arrays_to_tuples, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_metadata_field_v3, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata import ( + ArrayDimensionSeparatorV2, + ArrayMetadataV2, + ArrayMetadataV3, + ArrayOrderV2, + ExtensionFieldV3, + MetadataV3, + ) + from zarr_metadata._common import JSONValue + from zarr_metadata.v2 import CodecMetadataV2, DataTypeMetadataV2 + +ArrayMetadataStoreKeyV3 = Literal["zarr.json"] +ARRAY_METADATA_STORE_KEY_V3: Final[ArrayMetadataStoreKeyV3] = "zarr.json" + +ArrayMetadataStoreKeyV2 = Literal[".zarray"] +ARRAY_METADATA_STORE_KEY_V2: Final[ArrayMetadataStoreKeyV2] = ".zarray" + +AttributesStoreKeyV2 = Literal[".zattrs"] +ATTRIBUTES_STORE_KEY_V2: Final[AttributesStoreKeyV2] = ".zattrs" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ZarrMetadataV3: + """A v3 metadata field in normalized form: a name plus a configuration. + + This is the in-memory model of `MetadataV3` (a bare name string or a + `{name, configuration}` mapping): the bare-name and missing-configuration + forms normalize to an empty configuration. + """ + + name: str + configuration: dict[str, JSONValue] + + def to_json(self) -> MetadataV3: + return {"name": self.name, "configuration": self.configuration} + + @classmethod + def from_json(cls, data: object) -> ZarrMetadataV3: + field = parse_metadata_field_v3(data) + if isinstance(field, str): + return cls(name=field, configuration={}) + configuration = arrays_to_tuples(dict(field.get("configuration", {}))) + return cls(name=field["name"], configuration=configuration) # type: ignore[arg-type] + + +class ArrayMetadataModelV3Partial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `ArrayMetadataModelV3`. + + Every key is optional and typed with the model's own (not serialized) + value types, so it describes valid keyword arguments to + `ArrayMetadataModelV3.update`. The `init=False` fields `zarr_format` and + `node_type` are intentionally excluded, since they cannot be passed to + `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_array.py::test_partial_keys_match_settable_model_fields`. + """ + + shape: tuple[int, ...] + fill_value: JSONValue + data_type: ZarrMetadataV3 + chunk_grid: ZarrMetadataV3 + codecs: tuple[ZarrMetadataV3, ...] + chunk_key_encoding: ZarrMetadataV3 + dimension_names: tuple[str | None, ...] | None + attributes: dict[str, JSONValue] + storage_transformers: tuple[ZarrMetadataV3, ...] + extra_fields: dict[str, ExtensionFieldV3] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ArrayMetadataModelV3: + """In-memory model of a v3 array metadata document. + + A canonical, lossless representation of the `zarr.json` content for an + array. Extension points (`data_type`, `chunk_grid`, `chunk_key_encoding`, + `codecs`, `storage_transformers`) are held as `ZarrMetadataV3` name + + configuration pairs and are never interpreted; `fill_value` is held + verbatim in its JSON form. + """ + + zarr_format: Literal[3] = field(default=3, init=False) + node_type: Literal["array"] = field(default="array", init=False) + shape: tuple[int, ...] + fill_value: JSONValue + data_type: ZarrMetadataV3 + chunk_grid: ZarrMetadataV3 + codecs: tuple[ZarrMetadataV3, ...] + chunk_key_encoding: ZarrMetadataV3 + dimension_names: tuple[str | None, ...] | None + attributes: dict[str, JSONValue] + storage_transformers: tuple[ZarrMetadataV3, ...] + extra_fields: dict[str, ExtensionFieldV3] + + @classmethod + def create_default( + cls, **overrides: Unpack[ArrayMetadataModelV3Partial] + ) -> ArrayMetadataModelV3: + """ + Create a default (empty) v3 array metadata model, with optional overrides. + + The default is a structurally-valid scalar `uint8` array — the array + analog of `list()` returning `[]`. Any field can be overridden by keyword + (the same fields accepted by `update`). + """ + default = cls( + shape=(), + fill_value=0, + data_type=ZarrMetadataV3(name="uint8", configuration={}), + chunk_grid=ZarrMetadataV3(name="regular", configuration={"chunk_shape": ()}), + codecs=(ZarrMetadataV3(name="bytes", configuration={}),), + chunk_key_encoding=ZarrMetadataV3(name="default", configuration={}), + dimension_names=None, + attributes={}, + storage_transformers=(), + extra_fields={}, + ) + return default.update(**overrides) + + def update(self, **kwargs: Unpack[ArrayMetadataModelV3Partial]) -> ArrayMetadataModelV3: + """ + Return a new `ArrayMetadataModelV3` with the given fields updated. + + Only the constructor-settable fields listed in + `ArrayMetadataModelV3Partial` can be updated; any attempt to update + other fields (including the fixed `zarr_format` / `node_type`) is + rejected at the type level. Each given field fully replaces its + previous value, including `extra_fields`. + + This is useful for test fixtures that want to override a few fields of a + base template without having to re-specify the entire document. + """ + return dataclasses.replace(self, **kwargs) + + def __post_init__(self) -> None: + if set(self.extra_fields.keys()).intersection(ARRAY_METADATA_STANDARD_KEYS_V3): + raise ValueError("Extra fields cannot overlap with standard ArrayMetadataV3 fields") + + def to_json(self) -> ArrayMetadataV3: + out: ArrayMetadataV3 = { + "zarr_format": self.zarr_format, + "node_type": self.node_type, + "shape": self.shape, + "fill_value": self.fill_value, + "data_type": self.data_type.to_json(), + "chunk_grid": self.chunk_grid.to_json(), + "codecs": tuple(codec.to_json() for codec in self.codecs), + "chunk_key_encoding": self.chunk_key_encoding.to_json(), + } + if self.dimension_names is not None: + out["dimension_names"] = self.dimension_names + if len(self.attributes) > 0: + out["attributes"] = self.attributes + if len(self.storage_transformers) > 0: + out["storage_transformers"] = tuple( + transformer.to_json() for transformer in self.storage_transformers + ) + # Extra fields are the TypedDict's `extra_items` (PEP 728). Assign them + # by key rather than `out.update(**...)`: type checkers understand the + # indexed-write path against `extra_items`, but not the `update(**...)` + # overload. + for key, value in self.extra_fields.items(): + out[key] = value + return out + + @classmethod + def from_json(cls, data: object) -> ArrayMetadataModelV3: + parsed = parse_array_metadata_v3(arrays_to_tuples(data)) + extra_fields: dict[str, ExtensionFieldV3] = { + k: v # type: ignore[misc] + for k, v in parsed.items() + if k not in ARRAY_METADATA_STANDARD_KEYS_V3 + } + return cls( + shape=parsed["shape"], + fill_value=parsed["fill_value"], # type: ignore[arg-type] # fill_value: object in upstream TypedDict + data_type=ZarrMetadataV3.from_json(parsed["data_type"]), + chunk_grid=ZarrMetadataV3.from_json(parsed["chunk_grid"]), + codecs=tuple(ZarrMetadataV3.from_json(c) for c in parsed["codecs"]), + chunk_key_encoding=ZarrMetadataV3.from_json(parsed["chunk_key_encoding"]), + dimension_names=parsed.get("dimension_names"), + attributes=dict(parsed.get("attributes", {})), + storage_transformers=tuple( + ZarrMetadataV3.from_json(t) for t in parsed.get("storage_transformers", ()) + ), + extra_fields=extra_fields, + ) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV3: + return cls.from_json(json.loads(mapping[ARRAY_METADATA_STORE_KEY_V3])) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + return { + ARRAY_METADATA_STORE_KEY_V3: json.dumps(self.to_json(), indent=indent).encode("utf-8") + } + + +class ArrayMetadataModelV2Partial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `ArrayMetadataModelV2`. + + Every key is optional and typed with the model's own value types, so it + describes valid keyword arguments to `ArrayMetadataModelV2.update` and + `create_default`. The `init=False` field `zarr_format` is intentionally + excluded, since it cannot be passed to `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_array.py::test_v2_partial_keys_match_settable_model_fields`. + """ + + shape: tuple[int, ...] + dtype: DataTypeMetadataV2 + chunks: tuple[int, ...] + fill_value: JSONValue + order: ArrayOrderV2 + compressor: CodecMetadataV2 | None + filters: tuple[CodecMetadataV2, ...] | None + dimension_separator: ArrayDimensionSeparatorV2 + attributes: dict[str, JSONValue] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ArrayMetadataModelV2: + """In-memory model of a v2 array metadata document. + + A canonical, lossless representation of the `.zarray` content plus the + sibling `.zattrs` attributes. `dtype`, `compressor`, and `filters` are + held in their raw JSON forms and are never interpreted; `fill_value` is + held verbatim in its JSON form. + """ + + zarr_format: Literal[2] = field(default=2, init=False) + shape: tuple[int, ...] + dtype: DataTypeMetadataV2 + chunks: tuple[int, ...] + fill_value: JSONValue + order: ArrayOrderV2 + compressor: CodecMetadataV2 | None + filters: tuple[CodecMetadataV2, ...] | None + dimension_separator: ArrayDimensionSeparatorV2 = field(default="/") + attributes: dict[str, JSONValue] + + def update(self, **kwargs: Unpack[ArrayMetadataModelV2Partial]) -> ArrayMetadataModelV2: + """ + Return a new `ArrayMetadataModelV2` with the given fields updated. + + Only the constructor-settable fields listed in + `ArrayMetadataModelV2Partial` can be updated; the fixed `zarr_format` is + rejected at the type level. Each given field fully replaces its previous + value. + """ + return dataclasses.replace(self, **kwargs) + + @classmethod + def create_default( + cls, **overrides: Unpack[ArrayMetadataModelV2Partial] + ) -> ArrayMetadataModelV2: + """ + Create a default (empty) v2 array metadata model, with optional overrides. + + The default is a structurally-valid scalar `uint8` (`"|u1"`) array — the + array analog of `list()` returning `[]`. Any field can be overridden by + keyword (the same fields accepted by `update`). + """ + default = cls( + shape=(), + dtype="|u1", + chunks=(), + fill_value=0, + order="C", + compressor=None, + filters=None, + attributes={}, + ) + return default.update(**overrides) + + def to_json(self) -> ArrayMetadataV2: + out: ArrayMetadataV2 = { + "zarr_format": self.zarr_format, + "shape": self.shape, + "dtype": self.dtype, + "order": self.order, + "chunks": self.chunks, + "fill_value": self.fill_value, + "dimension_separator": self.dimension_separator, + "attributes": self.attributes, + "compressor": self.compressor, + "filters": self.filters, + } + return out + + @classmethod + def from_json(cls, data: object) -> ArrayMetadataModelV2: + parsed = parse_array_metadata_v2(arrays_to_tuples(data)) + return cls( + shape=parsed["shape"], + dtype=parsed["dtype"], + chunks=parsed["chunks"], + fill_value=parsed["fill_value"], # type: ignore[arg-type] # fill_value: object in upstream TypedDict + order=parsed["order"], + compressor=parsed["compressor"], + filters=parsed["filters"], + dimension_separator=parsed.get("dimension_separator", "/"), + attributes=dict(parsed.get("attributes", {})), + ) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV2: + zarray = json.loads(mapping[ARRAY_METADATA_STORE_KEY_V2]) + zattrs: dict[str, JSONValue] = ( + json.loads(mapping[ATTRIBUTES_STORE_KEY_V2]) + if ATTRIBUTES_STORE_KEY_V2 in mapping + else {} + ) + return cls.from_json({**zarray, "attributes": zattrs}) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + # Attributes live only in the sibling `.zattrs` file; the `.zarray` + # document must exclude them. + zarray = {k: v for k, v in self.to_json().items() if k != "attributes"} + return { + ARRAY_METADATA_STORE_KEY_V2: json.dumps(zarray, indent=indent).encode("utf-8"), + ATTRIBUTES_STORE_KEY_V2: json.dumps(self.attributes, indent=indent).encode("utf-8"), + } diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index d0e30e1cda..25f3422c50 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -259,13 +259,12 @@ def parse_array_metadata_v2(value: object) -> ArrayMetadataV2: return cast(ArrayMetadataV2, value) -def _arrays_to_tuples(obj: object) -> object: +def arrays_to_tuples(obj: object) -> object: """Recursively convert every list in a JSON-decoded structure to a tuple.""" if isinstance(obj, list): - return tuple(_arrays_to_tuples(item) for item in cast("list[object]", obj)) + return tuple(arrays_to_tuples(item) for item in cast("list[object]", obj)) if isinstance(obj, dict): return { - key: _arrays_to_tuples(value) - for key, value in cast("dict[object, object]", obj).items() + key: arrays_to_tuples(value) for key, value in cast("dict[object, object]", obj).items() } return obj From 6b391474ea5be5c9ef41ee791bcfa502b8c75640 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 10:23:53 +0200 Subject: [PATCH 04/70] test(zarr-metadata): port array model test suite Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/__init__.py | 60 + .../zarr-metadata/tests/model/__init__.py | 0 packages/zarr-metadata/tests/model/_cases.py | 45 + .../zarr-metadata/tests/model/test_array.py | 1019 +++++++++++++++++ 4 files changed, 1124 insertions(+) create mode 100644 packages/zarr-metadata/tests/model/__init__.py create mode 100644 packages/zarr-metadata/tests/model/_cases.py create mode 100644 packages/zarr-metadata/tests/model/test_array.py diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index e1382a6f0a..8776b4fab3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -1 +1,61 @@ """In-memory models for Zarr metadata documents.""" + +from zarr_metadata.model._array import ( + ARRAY_METADATA_STORE_KEY_V2, + ARRAY_METADATA_STORE_KEY_V3, + ATTRIBUTES_STORE_KEY_V2, + ArrayMetadataModelV2, + ArrayMetadataModelV2Partial, + ArrayMetadataModelV3, + ArrayMetadataModelV3Partial, + ZarrMetadataV3, +) +from zarr_metadata.model._validation import ( + ARRAY_METADATA_OPTIONAL_KEYS_V3, + ARRAY_METADATA_REQUIRED_KEYS_V2, + ARRAY_METADATA_REQUIRED_KEYS_V3, + ARRAY_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ValidationProblem, + is_array_metadata_v2, + is_array_metadata_v3, + is_json, + is_metadata_field_v3, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_json, + parse_metadata_field_v3, + validate_array_metadata_v2, + validate_array_metadata_v3, + validate_json, + validate_metadata_field_v3, +) + +__all__ = [ + "ARRAY_METADATA_OPTIONAL_KEYS_V3", + "ARRAY_METADATA_REQUIRED_KEYS_V2", + "ARRAY_METADATA_REQUIRED_KEYS_V3", + "ARRAY_METADATA_STANDARD_KEYS_V3", + "ARRAY_METADATA_STORE_KEY_V2", + "ARRAY_METADATA_STORE_KEY_V3", + "ATTRIBUTES_STORE_KEY_V2", + "ArrayMetadataModelV2", + "ArrayMetadataModelV2Partial", + "ArrayMetadataModelV3", + "ArrayMetadataModelV3Partial", + "MetadataValidationError", + "ValidationProblem", + "ZarrMetadataV3", + "is_array_metadata_v2", + "is_array_metadata_v3", + "is_json", + "is_metadata_field_v3", + "parse_array_metadata_v2", + "parse_array_metadata_v3", + "parse_json", + "parse_metadata_field_v3", + "validate_array_metadata_v2", + "validate_array_metadata_v3", + "validate_json", + "validate_metadata_field_v3", +] diff --git a/packages/zarr-metadata/tests/model/__init__.py b/packages/zarr-metadata/tests/model/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/zarr-metadata/tests/model/_cases.py b/packages/zarr-metadata/tests/model/_cases.py new file mode 100644 index 0000000000..f925c69dd6 --- /dev/null +++ b/packages/zarr-metadata/tests/model/_cases.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING, Generic, TypeVar + +import pytest + +if TYPE_CHECKING: + from contextlib import AbstractContextManager + +TIn = TypeVar("TIn") +TOut = TypeVar("TOut") + + +@dataclass(frozen=True) +class Expect(Generic[TIn, TOut]): + """A test case with explicit input, expected output, and a human-readable id.""" + + input: TIn + output: TOut + id: str + + +@dataclass(frozen=True) +class ExpectFail(Generic[TIn]): + """A test case that should raise an exception. + + `msg` is a regex matched against the exception text (pytest's native + `match=` semantics). Leave it `None` to assert only the exception type. Set + `escape=True` when `msg` is a literal that contains regex metacharacters + such as `(`, `[`, or `.`; `escape` has no effect when `msg` is `None`. + """ + + input: TIn + exception: type[Exception] + id: str + msg: str | None = None + escape: bool = False + + def raises(self) -> AbstractContextManager[pytest.ExceptionInfo[Exception]]: + if self.msg is None: + return pytest.raises(self.exception) + pattern = re.escape(self.msg) if self.escape else self.msg + return pytest.raises(self.exception, match=pattern) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py new file mode 100644 index 0000000000..80dac709de --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -0,0 +1,1019 @@ +"""Tests for the metadata models in ``zarr_metadata.model``.""" + +import dataclasses +import json +from collections.abc import Callable +from typing import TYPE_CHECKING + +import pytest + +from tests.model._cases import Expect, ExpectFail +from zarr_metadata.model import ( + ARRAY_METADATA_OPTIONAL_KEYS_V3, + ARRAY_METADATA_REQUIRED_KEYS_V3, + ARRAY_METADATA_STANDARD_KEYS_V3, + ArrayMetadataModelV2, + ArrayMetadataModelV2Partial, + ArrayMetadataModelV3, + ArrayMetadataModelV3Partial, + MetadataValidationError, + ValidationProblem, + ZarrMetadataV3, + is_array_metadata_v2, + is_array_metadata_v3, + is_json, + is_metadata_field_v3, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_json, + parse_metadata_field_v3, + validate_array_metadata_v2, + validate_array_metadata_v3, + validate_json, + validate_metadata_field_v3, +) +from zarr_metadata.model._validation import _prefix, arrays_to_tuples + +if TYPE_CHECKING: + from zarr_metadata._common import JSONValue + from zarr_metadata.v2 import CodecMetadataV2 + +# --- public exports -------------------------------------------------------- + + +def test_guards_exported_from_package() -> None: + """The wire-type guard/parser functions are exported from the package.""" + import zarr_metadata.model + + for name in ( + "is_json", + "parse_json", + "is_metadata_field_v3", + "parse_metadata_field_v3", + "is_array_metadata_v3", + "parse_array_metadata_v3", + "is_array_metadata_v2", + "parse_array_metadata_v2", + ): + assert name in zarr_metadata.model.__all__ + assert hasattr(zarr_metadata.model, name) + + +def test_validation_diagnostics_exported_from_package() -> None: + """The validation-diagnostic types and validators are exported from the package.""" + import zarr_metadata.model + + for name in ( + "ValidationProblem", + "MetadataValidationError", + "validate_json", + "validate_metadata_field_v3", + "validate_array_metadata_v3", + "validate_array_metadata_v2", + ): + assert name in zarr_metadata.model.__all__ + assert hasattr(zarr_metadata.model, name) + + +def test_expect_expectfail_smoke() -> None: + """The Expect/ExpectFail test-case dataclasses behave as expected.""" + e = Expect(input=1, output=2, id="x") + assert (e.input, e.output, e.id) == (1, 2, "x") + f = ExpectFail(input=1, exception=ValueError, id="y", msg="boom") + with f.raises(): + raise ValueError("boom") + + +def test_v3_from_json_error_lists_all_problems() -> None: + """A malformed v3 document surfaces every problem via MetadataValidationError.problems.""" + doc: dict[str, object] = dict(ArrayMetadataModelV3.create_default().to_json()) + del doc["shape"] + doc["data_type"] = 5 + with pytest.raises(MetadataValidationError) as exc_info: + ArrayMetadataModelV3.from_json(doc) + locs = {p.loc for p in exc_info.value.problems} + assert ("shape",) in locs + assert ("data_type",) in locs + + +# --- JSON type / fill_value contract --------------------------------------- + + +def test_json_value_type_accepts_json_shapes() -> None: + # JSONValue is the package's public JSON type alias; assigning JSON-shaped + # values to it is valid. + """The JSONValue type alias accepts JSON-shaped values.""" + value: JSONValue = {"a": [1, 2.0, "x", True, None]} + assert value == {"a": [1, 2.0, "x", True, None]} + + +def test_string_nan_fill_value_roundtrips() -> None: + # Non-finite floats are represented as the spec strings ("NaN", "Infinity", + # "-Infinity") by the caller — the metadata layer does not interpret dtypes. + # The string form round-trips cleanly under default dataclass equality, + # unlike a raw float('nan') (which is an invalid fill_value the caller must + # not pass). + """A string 'NaN' fill_value round-trips cleanly (non-finite floats are the caller's responsibility).""" + m = ArrayMetadataModelV3.create_default(fill_value="NaN") + assert ArrayMetadataModelV3.from_json(m.to_json()) == m + assert ArrayMetadataModelV3.from_json(m.to_json()).fill_value == "NaN" + + +# --- ZarrMetadataV3.to_json ------------------------------------------------ + +ZARR_TO_JSON_CASES = [ + Expect( + ZarrMetadataV3(name="regular", configuration={"chunk_shape": [1]}), + {"name": "regular", "configuration": {"chunk_shape": [1]}}, + id="with-configuration", + ), + Expect( + ZarrMetadataV3(name="bytes", configuration={}), + {"name": "bytes", "configuration": {}}, + id="without-configuration", + ), +] + + +@pytest.mark.parametrize("case", ZARR_TO_JSON_CASES, ids=lambda c: c.id) +def test_zarr_metadata_v3_to_json(case: Expect[ZarrMetadataV3, dict[str, object]]) -> None: + """ZarrMetadataV3.to_json emits the canonical object form.""" + assert case.input.to_json() == case.output + + +# --- ZarrMetadataV3.from_json ----------------------------------------------- + +ZARR_FROM_JSON_CASES = [ + Expect("bytes", ZarrMetadataV3(name="bytes", configuration={}), id="bare-string"), + Expect( + {"name": "regular", "configuration": {"chunk_shape": [1]}}, + ZarrMetadataV3(name="regular", configuration={"chunk_shape": (1,)}), + id="object-with-config", + ), + Expect( + {"name": "bytes"}, + ZarrMetadataV3(name="bytes", configuration={}), + id="object-without-config", + ), +] + + +@pytest.mark.parametrize("case", ZARR_FROM_JSON_CASES, ids=lambda c: c.id) +def test_zarr_metadata_v3_from_json(case: Expect[object, ZarrMetadataV3]) -> None: + """ZarrMetadataV3.from_json parses both the bare-string and object forms.""" + assert ZarrMetadataV3.from_json(case.input) == case.output + + +# --- V3 baseline ----------------------------------------------------------- + + +def test_v3_to_json_includes_required_fields() -> None: + """V3 to_json emits all required fields with the expected values.""" + out = ArrayMetadataModelV3.create_default( + shape=(10,), data_type=ZarrMetadataV3(name="int32", configuration={}) + ).to_json() + assert out["zarr_format"] == 3 + assert out["node_type"] == "array" + assert out["shape"] == (10,) + assert out["fill_value"] == 0 + assert out["data_type"] == {"name": "int32", "configuration": {}} + assert out["codecs"] == ({"name": "bytes", "configuration": {}},) + + +def test_v3_dimension_names_included_when_present() -> None: + """V3 to_json includes dimension_names when they are set.""" + out: dict[str, object] = dict( + ArrayMetadataModelV3.create_default(dimension_names=("x",)).to_json() + ) + assert out["dimension_names"] == ("x",) + + +def test_v3_dimension_names_omitted_when_none() -> None: + """V3 to_json omits dimension_names when they are None.""" + out = ArrayMetadataModelV3.create_default(dimension_names=None).to_json() + assert "dimension_names" not in out + + +# --- BUG 1: attributes gated on dimension_names ---------------------------- + + +def test_v3_attributes_included_when_dimension_names_is_none() -> None: + """Attributes must be emitted regardless of dimension_names. + + Regression: attributes were gated on ``dimension_names is not None``, + so non-empty attributes were silently dropped when there were no + dimension names. + """ + out: dict[str, object] = dict( + ArrayMetadataModelV3.create_default( + dimension_names=None, attributes={"foo": "bar"} + ).to_json() + ) + assert out["attributes"] == {"foo": "bar"} + + +# --- BUG 2: single storage transformer dropped ----------------------------- + + +def test_v3_single_storage_transformer_included() -> None: + """A single storage transformer must be emitted. + + Regression: the guard used ``> 1`` instead of ``> 0``, dropping a + lone storage transformer. + """ + st = ZarrMetadataV3(name="some_transformer", configuration={}) + out: dict[str, object] = dict( + ArrayMetadataModelV3.create_default(storage_transformers=(st,)).to_json() + ) + assert out["storage_transformers"] == ({"name": "some_transformer", "configuration": {}},) + + +def test_v3_no_storage_transformers_omitted() -> None: + """V3 to_json omits storage_transformers when empty.""" + out = ArrayMetadataModelV3.create_default(storage_transformers=()).to_json() + assert "storage_transformers" not in out + + +# --- V3 extra fields ------------------------------------------------------- + + +def test_v3_extra_fields_merged() -> None: + """V3 to_json merges extra_fields into the top-level document.""" + out = ArrayMetadataModelV3.create_default( + extra_fields={"my_ext": {"must_understand": False}} + ).to_json() + assert out["my_ext"] == {"must_understand": False} + + +def test_v3_extra_fields_overlapping_standard_field_rejected() -> None: + """Constructing a V3 model with an extra field that collides with a standard key is rejected.""" + with pytest.raises(ValueError): + ArrayMetadataModelV3.create_default(extra_fields={"shape": {"must_understand": False}}) + + +# --- V3 key/value ---------------------------------------------------------- + + +def test_v3_to_key_value_is_valid_json_under_zarr_json() -> None: + """V3 to_key_value produces valid JSON bytes under the zarr.json key.""" + kv = ArrayMetadataModelV3.create_default(attributes={"a": 1}).to_key_value() + assert set(kv) == {"zarr.json"} + parsed = json.loads(kv["zarr.json"].decode("utf-8")) + assert parsed["zarr_format"] == 3 + assert parsed["attributes"] == {"a": 1} + + +# --- V3 standard-key sets -------------------------------------------------- + + +def test_standard_keys_is_union_of_required_and_optional() -> None: + """The standard-key set is the union of the required and optional key sets.""" + assert ( + ARRAY_METADATA_STANDARD_KEYS_V3 + == ARRAY_METADATA_REQUIRED_KEYS_V3 | ARRAY_METADATA_OPTIONAL_KEYS_V3 + ) + + +def test_standard_keys_contains_known_fields_and_excludes_extensions() -> None: + """The standard-key set contains known fields and excludes extension keys.""" + assert { + "zarr_format", + "node_type", + "shape", + "codecs", + } <= ARRAY_METADATA_STANDARD_KEYS_V3 + assert "my_ext" not in ARRAY_METADATA_STANDARD_KEYS_V3 + + +# --- create_default -------------------------------------------------------- + + +def test_v3_create_default_is_valid_empty_array() -> None: + """V3 create_default builds a structurally valid empty array that round-trips.""" + m = ArrayMetadataModelV3.create_default() + assert m.shape == () + assert m.data_type == ZarrMetadataV3(name="uint8", configuration={}) + assert m.fill_value == 0 + assert m.attributes == {} + assert m.extra_fields == {} + # the default document is structurally valid and round-trips + assert validate_array_metadata_v3(m.to_json()) == [] + assert ArrayMetadataModelV3.from_json(m.to_json()) == m + + +def test_v3_create_default_applies_overrides() -> None: + """V3 create_default applies keyword overrides over the defaults.""" + m = ArrayMetadataModelV3.create_default(shape=(4, 4), attributes={"a": 1}) + assert m.shape == (4, 4) + assert m.attributes == {"a": 1} + # un-overridden fields keep their defaults + assert m.data_type == ZarrMetadataV3(name="uint8", configuration={}) + + +def test_v2_create_default_is_valid_empty_array() -> None: + """V2 create_default builds a structurally valid empty array that round-trips.""" + m = ArrayMetadataModelV2.create_default() + assert m.shape == () + assert m.chunks == () + assert m.fill_value == 0 + assert m.compressor is None + assert m.filters is None + assert m.attributes == {} + assert validate_array_metadata_v2(m.to_json()) == [] + assert ArrayMetadataModelV2.from_json(m.to_json()) == m + + +def test_v2_create_default_applies_overrides() -> None: + """V2 create_default applies keyword overrides over the defaults.""" + m = ArrayMetadataModelV2.create_default(shape=(8,), attributes={"k": "v"}) + assert m.shape == (8,) + assert m.attributes == {"k": "v"} + assert m.dtype == "|u1" # default dtype unchanged + + +# --- V3 update ------------------------------------------------------------- + +# Cluster 3: update same-shape pairs across versions — parametrized + +UPDATE_NEW_INSTANCE_PARAMS = [ + pytest.param(ArrayMetadataModelV3, id="v3"), + pytest.param(ArrayMetadataModelV2, id="v2"), +] + + +@pytest.mark.parametrize("model_cls", UPDATE_NEW_INSTANCE_PARAMS) +def test_update_returns_new_instance( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], +) -> None: + """update returns a new instance with the field replaced, leaving the original unchanged.""" + base = model_cls.create_default(shape=(10,)) + updated = base.update(shape=(20,)) + assert updated.shape == (20,) + assert base.shape == (10,) # original unchanged + assert isinstance(updated, model_cls) + + +UPDATE_NO_ARGS_PARAMS = [ + pytest.param(ArrayMetadataModelV3, id="v3"), + pytest.param(ArrayMetadataModelV2, id="v2"), +] + + +@pytest.mark.parametrize("model_cls", UPDATE_NO_ARGS_PARAMS) +def test_update_no_args_returns_equal_model( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], +) -> None: + """update with no arguments returns a model equal to the original.""" + base = model_cls.create_default() + assert base.update() == base + + +# V3-only update tests — kept direct (extra_fields is v3-specific) + + +def test_update_can_replace_extra_fields() -> None: + """update can replace the extra_fields mapping.""" + base = ArrayMetadataModelV3.create_default(extra_fields={}) + updated = base.update(extra_fields={"my_ext": {"must_understand": False}}) + assert updated.extra_fields == {"my_ext": {"must_understand": False}} + + +def test_update_replaces_extra_fields_rather_than_merging() -> None: + """update replaces extra_fields wholesale rather than merging.""" + base = ArrayMetadataModelV3.create_default(extra_fields={"a": {"must_understand": False}}) + updated = base.update(extra_fields={"b": {"must_understand": True}}) + assert updated.extra_fields == {"b": {"must_understand": True}} + + +def test_partial_keys_match_settable_model_fields() -> None: + """The partial TypedDict must list exactly the constructor-settable fields. + + Guards against drift: adding/removing a settable field on the model + without updating ``ArrayMetadataModelV3Partial`` fails here. + """ + settable = {f.name for f in dataclasses.fields(ArrayMetadataModelV3) if f.init} + assert set(ArrayMetadataModelV3Partial.__annotations__) == settable + + +# --- V2 model -------------------------------------------------------------- + + +def test_v2_partial_keys_match_settable_model_fields() -> None: + """The v2 partial TypedDict must list exactly the settable fields.""" + settable = {f.name for f in dataclasses.fields(ArrayMetadataModelV2) if f.init} + assert set(ArrayMetadataModelV2Partial.__annotations__) == settable + + +def test_v2_to_key_value_splits_zarray_and_zattrs() -> None: + """V2 to_key_value splits the document into .zarray and .zattrs.""" + kv = ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_key_value() + assert set(kv) == {".zarray", ".zattrs"} + zarray = json.loads(kv[".zarray"].decode("utf-8")) + zattrs = json.loads(kv[".zattrs"].decode("utf-8")) + assert zarray["zarr_format"] == 2 + assert zattrs == {"a": 1} + + +def test_v2_zarray_excludes_attributes() -> None: + """The on-disk ``.zarray`` document must not contain user attributes. + + In v2, attributes live only in the sibling ``.zattrs`` file. The bundled + ``ArrayMetadataV2`` / ``to_json()`` carry attributes for convenience, but + ``to_key_value()`` must split them out. + """ + kv = ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_key_value() + zarray = json.loads(kv[".zarray"].decode("utf-8")) + assert "attributes" not in zarray + + +def test_v2_to_json_still_includes_attributes() -> None: + """``to_json()`` is the bundled in-memory form and keeps attributes.""" + out: dict[str, object] = dict( + ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_json() + ) + assert out["attributes"] == {"a": 1} + + +# --- arrays_to_tuples helper ---------------------------------------------- + +ARRAYS_TO_TUPLES_CASES = [ + Expect([1, 2, 3], (1, 2, 3), id="top-level-list"), + Expect({"a": [1, [2, 3]], "b": "x"}, {"a": (1, (2, 3)), "b": "x"}, id="nested-in-dict"), + Expect(5, 5, id="scalar-int"), + Expect("s", "s", id="scalar-str"), + Expect(None, None, id="scalar-none"), + Expect( + {"name": "bytes", "configuration": {"nums": [1, 2]}}, + {"name": "bytes", "configuration": {"nums": (1, 2)}}, + id="dict-keys-preserved", + ), +] + + +@pytest.mark.parametrize("case", ARRAYS_TO_TUPLES_CASES, ids=lambda c: c.id) +def test_arrays_to_tuples(case: Expect[object, object]) -> None: + """arrays_to_tuples recursively converts JSON arrays to tuples.""" + assert arrays_to_tuples(case.input) == case.output + + +# --- ArrayMetadataModelV3.from_json ---------------------------------------- + + +def test_v3_from_json_reconstructs_required_fields() -> None: + """V3 from_json reconstructs the required fields from a document.""" + doc = ArrayMetadataModelV3.create_default( + shape=(7,), + attributes={"a": 1}, + data_type=ZarrMetadataV3(name="int32", configuration={}), + ).to_json() + model = ArrayMetadataModelV3.from_json(doc) + assert model.shape == (7,) + assert model.data_type == ZarrMetadataV3(name="int32", configuration={}) + assert model.attributes == {"a": 1} + + +def test_v3_from_json_defaults_for_omitted_optionals() -> None: + """V3 from_json supplies defaults for omitted optional fields.""" + doc = ArrayMetadataModelV3.create_default( + attributes={}, storage_transformers=(), dimension_names=None + ).to_json() + # to_json omits these entirely; from_json must restore defaults + model = ArrayMetadataModelV3.from_json(doc) + assert model.attributes == {} + assert model.storage_transformers == () + assert model.dimension_names is None + + +def test_v3_from_json_routes_unknown_keys_to_extra_fields() -> None: + """V3 from_json routes unknown top-level keys into extra_fields.""" + doc = ArrayMetadataModelV3.create_default( + extra_fields={"my_ext": {"must_understand": False}} + ).to_json() + model = ArrayMetadataModelV3.from_json(doc) + assert model.extra_fields == {"my_ext": {"must_understand": False}} + + +def test_v3_from_json_standard_keys_not_in_extra_fields() -> None: + """V3 from_json keeps standard keys out of extra_fields.""" + doc = ArrayMetadataModelV3.create_default(attributes={"a": 1}, dimension_names=("x",)).to_json() + model = ArrayMetadataModelV3.from_json(doc) + assert model.extra_fields == {} + + +def test_v3_from_json_nested_arrays_in_attributes_become_tuples() -> None: + """V3 from_json converts nested arrays in attributes into tuples.""" + doc = ArrayMetadataModelV3.create_default(attributes={"scale": [[1, 2], [3, 4]]}).to_json() + model = ArrayMetadataModelV3.from_json(doc) + assert model.attributes == {"scale": ((1, 2), (3, 4))} + + +# --- ArrayMetadataModelV3.from_key_value ---------------------------------- + + +def test_v3_from_key_value_parses_zarr_json() -> None: + """V3 from_key_value parses the zarr.json entry into a model.""" + kv = ArrayMetadataModelV3.create_default(shape=(3,)).to_key_value() + model = ArrayMetadataModelV3.from_key_value(kv) + assert model.shape == (3,) + + +# --- Cluster 2: from_key_value missing-key raises (parametrized) ----------- + +FROM_KEY_VALUE_MISSING_PARAMS = [ + pytest.param( + ArrayMetadataModelV3, + ExpectFail({}, KeyError, id="v3-missing-zarr-json"), + id="v3-missing-zarr-json", + ), + pytest.param( + ArrayMetadataModelV2, + ExpectFail({}, KeyError, id="v2-missing-zarray"), + id="v2-missing-zarray", + ), +] + + +@pytest.mark.parametrize(("model_cls", "case"), FROM_KEY_VALUE_MISSING_PARAMS) +def test_from_key_value_missing_key_raises( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], + case: ExpectFail[dict[str, bytes]], +) -> None: + """from_key_value raises KeyError when the required store key is absent.""" + with case.raises(): + model_cls.from_key_value(case.input) + + +# --- Cluster 1: round-trips (model → json → model, parametrized) ----------- + +ROUNDTRIP_MODEL_JSON_PARAMS = [ + pytest.param( + ArrayMetadataModelV3, + ArrayMetadataModelV3.create_default( + attributes={"a": 1}, + dimension_names=("x",), + storage_transformers=(ZarrMetadataV3(name="t", configuration={}),), + extra_fields={"ext": {"must_understand": False}}, + ), + id="v3-full", + ), + pytest.param( + ArrayMetadataModelV3, + ArrayMetadataModelV3.create_default( + attributes={}, + dimension_names=None, + storage_transformers=(), + extra_fields={}, + ), + id="v3-empty-optionals", + ), + pytest.param( + ArrayMetadataModelV2, + ArrayMetadataModelV2.create_default(attributes={"a": 1}, filters=None, compressor=None), + id="v2-basic", + ), +] + + +@pytest.mark.parametrize(("model_cls", "model"), ROUNDTRIP_MODEL_JSON_PARAMS) +def test_roundtrip_model_json_model( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], + model: ArrayMetadataModelV3 | ArrayMetadataModelV2, +) -> None: + """A model round-trips through to_json/from_json back to an equal model.""" + assert model_cls.from_json(model.to_json()) == model + + +# --- Round-trips (model → key_value → model, parametrized) ----------------- + +ROUNDTRIP_KEY_VALUE_PARAMS = [ + pytest.param( + ArrayMetadataModelV3, + ArrayMetadataModelV3.create_default(attributes={"a": 1}), + id="v3", + ), + pytest.param( + ArrayMetadataModelV2, + ArrayMetadataModelV2.create_default(attributes={"a": 1}), + id="v2", + ), +] + + +@pytest.mark.parametrize(("model_cls", "model"), ROUNDTRIP_KEY_VALUE_PARAMS) +def test_roundtrip_via_key_value( + model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], + model: ArrayMetadataModelV3 | ArrayMetadataModelV2, +) -> None: + """A model round-trips through to_key_value/from_key_value back to an equal model.""" + assert model_cls.from_key_value(model.to_key_value()) == model + + +# --- Round-trips (json → model → json, direction distinct — kept direct) --- + + +def test_v3_roundtrip_json_model_json() -> None: + """A v3 document round-trips through from_json/to_json back to an equal document.""" + doc = ArrayMetadataModelV3.create_default(attributes={"a": 1}, dimension_names=("x",)).to_json() + assert ArrayMetadataModelV3.from_json(doc).to_json() == doc + + +def test_v2_roundtrip_json_model_json() -> None: + """A v2 document round-trips through from_json/to_json back to an equal document.""" + doc = ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_json() + assert ArrayMetadataModelV2.from_json(doc).to_json() == doc + + +def test_v3_parser_accepts_bare_string_data_type() -> None: + """V3 from_json accepts a bare-string data_type and re-serializes it canonically.""" + doc = ArrayMetadataModelV3.create_default().to_json() + doc["data_type"] = "int32" # bare-string form, not canonical object form + model = ArrayMetadataModelV3.from_json(doc) + # parses correctly, re-serializes to canonical object form + assert model.data_type == ZarrMetadataV3(name="int32", configuration={}) + assert model.to_json()["data_type"] == {"name": "int32", "configuration": {}} + + +def test_v2_roundtrip_with_compressor_and_filters() -> None: + # Non-None compressor/filters must round-trip; extra assertion on .compressor. + """A v2 model with non-None compressor and filters round-trips.""" + compressor: CodecMetadataV2 = {"id": "blosc", "clevel": 5} + filters: tuple[CodecMetadataV2, ...] = ({"id": "delta"},) + m = ArrayMetadataModelV2.create_default(compressor=compressor, filters=filters) + restored = ArrayMetadataModelV2.from_json(m.to_json()) + assert restored == m + assert restored.compressor == {"id": "blosc", "clevel": 5} + + +# --- ArrayMetadataModelV2.from_json ---------------------------------------- + + +def test_v2_from_json_reconstructs_fields() -> None: + """V2 from_json reconstructs the fields from a document.""" + doc = ArrayMetadataModelV2.create_default( + shape=(4,), attributes={"a": 1}, dtype=" None: + """V2 from_json defaults attributes to empty when the key is absent.""" + doc = ArrayMetadataModelV2.create_default().to_json() + del doc["attributes"] + model = ArrayMetadataModelV2.from_json(doc) + assert model.attributes == {} + + +# --- ArrayMetadataModelV2.from_key_value -------------------------------- + + +def test_v2_from_key_value_remerges_zattrs() -> None: + """V2 from_key_value re-merges .zattrs back into attributes.""" + kv = ArrayMetadataModelV2.create_default(attributes={"a": 1}, shape=(10,)).to_key_value() + model = ArrayMetadataModelV2.from_key_value(kv) + assert model.attributes == {"a": 1} + assert model.shape == (10,) + + +def test_v2_from_key_value_absent_zattrs_gives_empty_attributes() -> None: + """V2 from_key_value yields empty attributes when .zattrs is absent.""" + kv: dict[str, bytes] = dict(ArrayMetadataModelV2.create_default(attributes={}).to_key_value()) + del kv[".zattrs"] + model = ArrayMetadataModelV2.from_key_value(kv) + assert model.attributes == {} + + +def test_v2_from_json_nested_arrays_in_attributes_become_tuples() -> None: + """V2 from_json converts nested arrays in attributes into tuples.""" + doc = ArrayMetadataModelV2.create_default(attributes={"axes": [[0, 1], [2, 3]]}).to_json() + model = ArrayMetadataModelV2.from_json(doc) + assert model.attributes == {"axes": ((0, 1), (2, 3))} + + +# --- scalar wire-type guards (is_/validate_/parse_) ------------------------ +# +# Each value is modelled once as Expect[object, frozenset[tuple[str | int, ...]]] +# where `output` is the set of expected problem locs validate_* must report — +# frozenset() means VALID. Valid iff output == frozenset(). + +JSON_VALIDATE_CASES: list[Expect[object, frozenset[tuple[str | int, ...]]]] = [ + Expect("s", frozenset(), id="str"), + Expect(1, frozenset(), id="int"), + Expect(1.5, frozenset(), id="float"), + Expect(True, frozenset(), id="bool"), + Expect(None, frozenset(), id="none"), + Expect({"a": [1, {"b": None}], "c": "x"}, frozenset(), id="nested-containers"), + Expect((1, 2, 3), frozenset(), id="tuple-array"), + Expect(float("nan"), frozenset(), id="nan"), + Expect(float("inf"), frozenset(), id="inf"), + Expect(object(), frozenset({()}), id="object"), + Expect(b"abc", frozenset({()}), id="bytes"), + Expect(bytearray(b"abc"), frozenset({()}), id="bytearray"), + Expect({1: "x"}, frozenset({()}), id="non-str-key"), + Expect([1, object()], frozenset({(1,)}), id="non-json-list-item"), + Expect({"ok": object()}, frozenset({("ok",)}), id="non-json-value"), +] + + +@pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) +def test_is_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """is_json reports whether a value is JSON-serializable.""" + assert is_json(case.input) is (case.output == frozenset()) + + +@pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) +def test_validate_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """validate_json reports the problems (and their locs) for a value.""" + problems = validate_json(case.input) + assert (problems == []) is (case.output == frozenset()) + assert {p.loc for p in problems} >= case.output + + +@pytest.mark.parametrize("case", JSON_VALIDATE_CASES, ids=lambda c: c.id) +def test_parse_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """parse_json returns valid JSON values and raises on invalid ones.""" + if case.output == frozenset(): + assert parse_json(case.input) is case.input + else: + with pytest.raises(MetadataValidationError): + parse_json(case.input) + + +def test_validate_json_reports_json_in_message() -> None: + """validate_json's message for a non-JSON value mentions JSON.""" + problems = validate_json(object()) + assert problems[0].loc == () + assert "JSON" in problems[0].message + + +METADATA_FIELD_VALIDATE_CASES: list[Expect[object, frozenset[tuple[str | int, ...]]]] = [ + Expect("bytes", frozenset(), id="bare-string"), + Expect({"name": "x", "configuration": {"a": 1}}, frozenset(), id="named-config"), + Expect({"name": "bytes"}, frozenset(), id="name-only"), + Expect(5, frozenset({()}), id="not-str-or-mapping"), + Expect({"configuration": {}}, frozenset({("name",)}), id="missing-name"), + Expect({"name": 3}, frozenset({("name",)}), id="non-str-name"), + Expect( + {"name": "x", "configuration": [1]}, + frozenset({("configuration",)}), + id="config-not-mapping", + ), + Expect( + {"name": "x", "configuration": {1: "y"}}, + frozenset({("configuration",)}), + id="config-non-str-key", + ), +] + + +@pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) +def test_is_metadata_field_v3(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: + """is_metadata_field_v3 reports whether a value is a v3 metadata field.""" + assert is_metadata_field_v3(case.input) is (case.output == frozenset()) + + +@pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) +def test_validate_metadata_field_v3( + case: Expect[object, frozenset[tuple[str | int, ...]]], +) -> None: + """validate_metadata_field_v3 reports the problems for a metadata-field value.""" + problems = validate_metadata_field_v3(case.input) + assert (problems == []) is (case.output == frozenset()) + assert {p.loc for p in problems} >= case.output + + +@pytest.mark.parametrize("case", METADATA_FIELD_VALIDATE_CASES, ids=lambda c: c.id) +def test_parse_metadata_field_v3( + case: Expect[object, frozenset[tuple[str | int, ...]]], +) -> None: + """parse_metadata_field_v3 returns valid fields and raises on invalid ones.""" + if case.output == frozenset(): + assert parse_metadata_field_v3(case.input) is case.input + else: + with pytest.raises(MetadataValidationError): + parse_metadata_field_v3(case.input) + + +# --- array-document wire-type guards (is_/validate_/parse_) ---------------- +# +# Each case starts from a valid document (built by `make`) and applies a +# mutation. `expected_locs` are loc paths `validate_*` must report for the +# invalid cases (a subset check, so accumulation of OTHER problems is allowed). + + +def _build_v3(**overrides: object) -> dict[str, object]: + return dict(ArrayMetadataModelV3.create_default(**overrides).to_json()) # type: ignore[arg-type] + + +def _build_v2(**overrides: object) -> dict[str, object]: + return dict(ArrayMetadataModelV2.create_default(**overrides).to_json()) # type: ignore[arg-type] + + +def _mutate(build: Callable[[], dict], mutate: Callable[[dict], object]) -> Callable[[], dict]: + def _factory() -> dict: + doc = build() + mutate(doc) + return doc + + return _factory + + +def _del(key: str) -> Callable[[dict], object]: + return lambda doc: doc.pop(key) + + +def _set(key: str, value: object) -> Callable[[dict], object]: + return lambda doc: doc.__setitem__(key, value) + + +V3_DOC_CASES: list[Expect[Callable[[], object], frozenset[tuple[str | int, ...]]]] = [ + Expect(_build_v3, frozenset(), id="valid"), + Expect( + lambda: _build_v3(attributes={"a": 1}, dimension_names=("x",)), + frozenset(), + id="valid-with-attributes-and-dim-names", + ), + Expect( + lambda: _build_v3(extra_fields={"my_ext": {"must_understand": False}}), + frozenset(), + id="valid-with-extra-fields", + ), + Expect(_mutate(_build_v3, _del("shape")), frozenset({("shape",)}), id="missing-shape"), + Expect( + _mutate(_build_v3, _set("data_type", 5)), + frozenset({("data_type",)}), + id="bad-data-type", + ), + Expect( + _mutate(_build_v3, _set("shape", "not-a-shape")), + frozenset({("shape",)}), + id="shape-not-sequence", + ), + Expect( + _mutate(_build_v3, _set("shape", [1, "x"])), + frozenset({("shape",)}), + id="shape-non-int-item", + ), + Expect( + _mutate(_build_v3, _set("codecs", (5,))), + frozenset({("codecs", 0)}), + id="bad-codec-entry", + ), + Expect(lambda: [1, 2, 3], frozenset({()}), id="non-mapping-list"), + Expect(lambda: "nope", frozenset({()}), id="non-mapping-str"), + Expect( + _mutate(_mutate(_build_v3, _del("shape")), _set("data_type", 5)), + frozenset({("shape",), ("data_type",)}), + id="missing-shape-and-bad-data-type", + ), +] + +V2_DOC_CASES: list[Expect[Callable[[], object], frozenset[tuple[str | int, ...]]]] = [ + Expect(_build_v2, frozenset(), id="valid"), + Expect(lambda: _build_v2(attributes={"a": 1}), frozenset(), id="valid-with-attributes"), + Expect( + lambda: _build_v2(compressor=None, filters=None), + frozenset(), + id="valid-none-compressor-filters", + ), + Expect( + _mutate(_build_v2, _del("chunks")), + frozenset({("chunks",)}), + id="missing-chunks", + ), + Expect( + _mutate(_build_v2, _set("shape", [1, "x"])), + frozenset({("shape",)}), + id="bad-shape", + ), + Expect( + _mutate(_mutate(_build_v2, _del("chunks")), _set("shape", [1, "x"])), + frozenset({("chunks",), ("shape",)}), + id="missing-chunks-and-bad-shape", + ), +] + +ALL_DOC_CASES = [ + *( + pytest.param( + is_array_metadata_v3, + validate_array_metadata_v3, + parse_array_metadata_v3, + c, + id=f"v3-{c.id}", + ) + for c in V3_DOC_CASES + ), + *( + pytest.param( + is_array_metadata_v2, + validate_array_metadata_v2, + parse_array_metadata_v2, + c, + id=f"v2-{c.id}", + ) + for c in V2_DOC_CASES + ), +] + + +@pytest.mark.parametrize(("is_fn", "validate_fn", "parse_fn", "case"), ALL_DOC_CASES) +def test_array_metadata_guards( + is_fn: Callable[[object], bool], + validate_fn: Callable[[object], list[ValidationProblem]], + parse_fn: Callable[[object], object], + case: Expect[Callable[[], object], frozenset[tuple[str | int, ...]]], +) -> None: + """is_/validate_/parse_ array-metadata guards agree on validity and locs for each case.""" + doc = case.input() + valid = case.output == frozenset() + assert is_fn(doc) is valid + problems = validate_fn(doc) + assert (problems == []) is valid + assert {p.loc for p in problems} >= case.output + if valid: + assert parse_fn(doc) is doc + else: + with pytest.raises(MetadataValidationError): + parse_fn(doc) + + +# --- strict from_json validation ------------------------------------------- + + +FROM_JSON_REJECT_PARAMS = [ + pytest.param( + ArrayMetadataModelV3, + ExpectFail(lambda: {"zarr_format": 3}, MetadataValidationError, id="x"), + id="v3-missing-required", + ), + pytest.param( + ArrayMetadataModelV3, + ExpectFail(_mutate(_build_v3, _set("data_type", 5)), MetadataValidationError, id="x"), + id="v3-bad-field-type", + ), + pytest.param( + ArrayMetadataModelV2, + ExpectFail(lambda: {"zarr_format": 2}, MetadataValidationError, id="x"), + id="v2-missing-required", + ), + pytest.param( + ZarrMetadataV3, + ExpectFail(lambda: 5, MetadataValidationError, id="x"), + id="zarr-metadata-bad-input", + ), +] + + +@pytest.mark.parametrize(("model", "case"), FROM_JSON_REJECT_PARAMS) +def test_from_json_rejects_malformed( + model: type[ArrayMetadataModelV3 | ArrayMetadataModelV2 | ZarrMetadataV3], + case: ExpectFail[Callable[[], object]], +) -> None: + """from_json raises MetadataValidationError on a malformed document.""" + with case.raises(): + model.from_json(case.input()) + + +# --- ValidationProblem / MetadataValidationError / _prefix ----------------- +# Small structural tests — not "parametrize over inputs" shaped, kept direct. + + +def test_validation_problem_str_with_loc() -> None: + """ValidationProblem.__str__ renders a non-empty loc as a dotted path.""" + p = ValidationProblem(loc=("codecs", 0, "name"), message="expected str") + assert str(p) == "codecs.0.name: expected str" + + +def test_validation_problem_str_empty_loc() -> None: + """ValidationProblem.__str__ renders an empty loc as .""" + p = ValidationProblem(loc=(), message="not a mapping") + assert str(p) == ": not a mapping" + + +def test_validation_problem_is_frozen() -> None: + """ValidationProblem is immutable (frozen dataclass).""" + p = ValidationProblem(loc=("shape",), message="x") + with pytest.raises(dataclasses.FrozenInstanceError): + p.message = "y" # type: ignore[misc] + + +def test_metadata_validation_error_holds_problems() -> None: + """MetadataValidationError carries its problem list and renders them in its message.""" + problems = [ + ValidationProblem(loc=("shape",), message="missing required key"), + ValidationProblem(loc=("data_type",), message="expected a metadata field"), + ] + err = MetadataValidationError(problems) + assert err.problems == problems + assert "shape: missing required key" in str(err) + assert "data_type: expected a metadata field" in str(err) + + +def test_prefix_prepends_loc_head() -> None: + """_prefix prepends a loc head to each problem's loc.""" + problems = [ValidationProblem(loc=("name",), message="expected str")] + prefixed = _prefix(0, problems) + assert prefixed == [ValidationProblem(loc=(0, "name"), message="expected str")] From 9f32f14f75f9e97204c01e8ad28b34f6515e2a44 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 13:24:31 +0200 Subject: [PATCH 05/70] feat(zarr-metadata): add group metadata models Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/_group.py | 370 ++++++++++++++++++ .../src/zarr_metadata/model/_validation.py | 89 ++++- .../zarr-metadata/tests/model/test_group.py | 137 +++++++ 3 files changed, 595 insertions(+), 1 deletion(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/_group.py create mode 100644 packages/zarr-metadata/tests/model/test_group.py diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py new file mode 100644 index 0000000000..8eeaed1a89 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -0,0 +1,370 @@ +"""In-memory models for Zarr group and consolidated metadata documents.""" + +from __future__ import annotations + +import dataclasses +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Final, Literal, cast + +from typing_extensions import TypedDict, Unpack + +from zarr_metadata.model._array import ( + ATTRIBUTES_STORE_KEY_V2, + ArrayMetadataModelV3, +) +from zarr_metadata.model._validation import ( + GROUP_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ValidationProblem, + arrays_to_tuples, + parse_group_metadata_v2, + parse_group_metadata_v3, +) + +if TYPE_CHECKING: + from zarr_metadata import ( + ConsolidatedMetadataV3, + ExtensionFieldV3, + GroupMetadataV2, + GroupMetadataV3, + ) + from zarr_metadata._common import JSONValue + +GroupMetadataStoreKeyV3 = Literal["zarr.json"] +GROUP_METADATA_STORE_KEY_V3: Final[GroupMetadataStoreKeyV3] = "zarr.json" + +GroupMetadataStoreKeyV2 = Literal[".zgroup"] +GROUP_METADATA_STORE_KEY_V2: Final[GroupMetadataStoreKeyV2] = ".zgroup" + +ConsolidatedMetadataStoreKeyV2 = Literal[".zmetadata"] +CONSOLIDATED_METADATA_STORE_KEY_V2: Final[ConsolidatedMetadataStoreKeyV2] = ".zmetadata" + +# The key under which consolidated metadata is embedded in a v3 group document. +# This is a reference-implementation convention (not a spec artifact), stored +# as an extension field on the group's `zarr.json`. +CONSOLIDATED_METADATA_KEY_V3: Final = "consolidated_metadata" + + +class GroupMetadataModelV3Partial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `GroupMetadataModelV3`. + + Every key is optional and typed with the model's own value types, so it + describes valid keyword arguments to `GroupMetadataModelV3.update` and + `create_default`. The `init=False` fields `zarr_format` and `node_type` + are intentionally excluded, since they cannot be passed to + `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_group.py::test_group_partial_keys_match_settable_model_fields`. + """ + + attributes: dict[str, JSONValue] + consolidated_metadata: ConsolidatedMetadataModelV3 | None + extra_fields: dict[str, ExtensionFieldV3] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class GroupMetadataModelV3: + """In-memory model of a v3 group metadata document. + + A canonical, lossless representation of the `zarr.json` content for a + group. The `consolidated_metadata` reference-implementation convention is + modeled as a typed field holding thin child models; every other unknown + top-level key lands in `extra_fields` verbatim. + """ + + zarr_format: Literal[3] = field(default=3, init=False) + node_type: Literal["group"] = field(default="group", init=False) + attributes: dict[str, JSONValue] + consolidated_metadata: ConsolidatedMetadataModelV3 | None + extra_fields: dict[str, ExtensionFieldV3] + + def __post_init__(self) -> None: + reserved = GROUP_METADATA_STANDARD_KEYS_V3 | {CONSOLIDATED_METADATA_KEY_V3} + if set(self.extra_fields.keys()).intersection(reserved): + raise ValueError("Extra fields cannot overlap with standard GroupMetadataV3 fields") + + @classmethod + def create_default( + cls, **overrides: Unpack[GroupMetadataModelV3Partial] + ) -> GroupMetadataModelV3: + """ + Create a default (empty) v3 group metadata model, with optional overrides. + + The default is a structurally-valid group with no attributes — the group + analog of `list()` returning `[]`. Any field can be overridden by keyword + (the same fields accepted by `update`). + """ + default = cls(attributes={}, consolidated_metadata=None, extra_fields={}) + return default.update(**overrides) + + def update(self, **kwargs: Unpack[GroupMetadataModelV3Partial]) -> GroupMetadataModelV3: + """ + Return a new `GroupMetadataModelV3` with the given fields updated. + + Only the constructor-settable fields listed in + `GroupMetadataModelV3Partial` can be updated; the fixed `zarr_format` / + `node_type` are rejected at the type level. Each given field fully + replaces its previous value, including `extra_fields`. + """ + return dataclasses.replace(self, **kwargs) + + def to_json(self) -> GroupMetadataV3: + out: GroupMetadataV3 = { + "zarr_format": self.zarr_format, + "node_type": self.node_type, + } + if len(self.attributes) > 0: + out["attributes"] = self.attributes + if self.consolidated_metadata is not None: + # The consolidated-metadata shape ({kind, must_understand, metadata}, + # no `name`) predates the strict v3.1 extension-field rules, so it is + # not assignable to `ExtensionFieldV3`; see the discussion on + # `zarr_metadata.v3.consolidated`. + out[CONSOLIDATED_METADATA_KEY_V3] = cast( + "ExtensionFieldV3", self.consolidated_metadata.to_json() + ) + for key, value in self.extra_fields.items(): + out[key] = value + return out + + @classmethod + def from_json(cls, data: object) -> GroupMetadataModelV3: + parsed = parse_group_metadata_v3(arrays_to_tuples(data)) + consolidated_raw = parsed.get(CONSOLIDATED_METADATA_KEY_V3) + consolidated = ( + None + if consolidated_raw is None + else ConsolidatedMetadataModelV3.from_json(consolidated_raw) + ) + extra_fields: dict[str, ExtensionFieldV3] = { + k: v # type: ignore[misc] + for k, v in parsed.items() + if k not in GROUP_METADATA_STANDARD_KEYS_V3 and k != CONSOLIDATED_METADATA_KEY_V3 + } + return cls( + attributes=dict(parsed.get("attributes", {})), + consolidated_metadata=consolidated, + extra_fields=extra_fields, + ) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV3: + return cls.from_json(json.loads(mapping[GROUP_METADATA_STORE_KEY_V3])) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + return { + GROUP_METADATA_STORE_KEY_V3: json.dumps(self.to_json(), indent=indent).encode("utf-8") + } + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ConsolidatedMetadataModelV3: + """In-memory model of v3 inline consolidated metadata. + + Models the reference-implementation convention where consolidated metadata + is embedded as an extension field on a group's `zarr.json`. Each entry in + `metadata` is a complete child document, held as a thin array or group + model. `must_understand` is typed permissively as `bool` to mirror the + document shape, but only `False` is valid; this is enforced at runtime. + """ + + kind: Literal["inline"] = field(default="inline", init=False) + must_understand: bool = False + metadata: dict[str, ArrayMetadataModelV3 | GroupMetadataModelV3] + + def __post_init__(self) -> None: + if self.must_understand is not False: + raise ValueError( + f"Invalid value for 'must_understand'. Expected False. " + f"Got {self.must_understand!r}." + ) + + def to_json(self) -> ConsolidatedMetadataV3: + # `must_understand` is emitted as the literal False: the field is typed + # permissively as `bool`, but `__post_init__` guarantees the value. + return { + "kind": self.kind, + "must_understand": False, + "metadata": {key: node.to_json() for key, node in self.metadata.items()}, + } + + @classmethod + def from_json(cls, data: object) -> ConsolidatedMetadataModelV3: + if not isinstance(data, Mapping): + raise MetadataValidationError([ValidationProblem((), "expected a mapping")]) + doc = cast("Mapping[str, object]", data) + entries_raw = doc.get("metadata") + if not isinstance(entries_raw, Mapping): + raise MetadataValidationError([ValidationProblem(("metadata",), "expected a mapping")]) + entries: dict[str, ArrayMetadataModelV3 | GroupMetadataModelV3] = {} + problems: list[ValidationProblem] = [] + for key, entry in cast("Mapping[object, object]", entries_raw).items(): + if not isinstance(key, str): + problems.append(ValidationProblem(("metadata",), f"non-string key {key!r}")) + continue + entry_obj: object = entry + node_type: object = None + if isinstance(entry, Mapping): + node_type = cast("Mapping[str, object]", entry).get("node_type") + if node_type == "array": + entries[key] = ArrayMetadataModelV3.from_json(entry_obj) + elif node_type == "group": + entries[key] = GroupMetadataModelV3.from_json(entry_obj) + else: + problems.append( + ValidationProblem(("metadata", key, "node_type"), "expected 'array' or 'group'") + ) + if problems: + raise MetadataValidationError(problems) + must_understand = doc.get("must_understand", False) + if must_understand is not False: + raise MetadataValidationError( + [ValidationProblem(("must_understand",), "expected False")] + ) + return cls(must_understand=must_understand, metadata=entries) + + +class GroupMetadataModelV2Partial(TypedDict, total=False): + """ + Partial form of the constructor-settable fields of `GroupMetadataModelV2`. + + Every key is optional and typed with the model's own value types, so it + describes valid keyword arguments to `GroupMetadataModelV2.update` and + `create_default`. The `init=False` field `zarr_format` is intentionally + excluded, since it cannot be passed to `dataclasses.replace`. + + Drift between this type and the model's settable fields is prevented by + `tests/model/test_group.py::test_group_partial_keys_match_settable_model_fields`. + """ + + attributes: dict[str, JSONValue] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class GroupMetadataModelV2: + """In-memory model of a v2 group metadata document. + + A canonical, lossless representation of the `.zgroup` content plus the + sibling `.zattrs` attributes, folded into a single in-memory value + (mirroring the merged `GroupMetadataV2` document form). + """ + + zarr_format: Literal[2] = field(default=2, init=False) + attributes: dict[str, JSONValue] + + @classmethod + def create_default( + cls, **overrides: Unpack[GroupMetadataModelV2Partial] + ) -> GroupMetadataModelV2: + """ + Create a default (empty) v2 group metadata model, with optional overrides. + + The default is a structurally-valid group with no attributes — the group + analog of `list()` returning `[]`. Any field can be overridden by keyword + (the same fields accepted by `update`). + """ + default = cls(attributes={}) + return default.update(**overrides) + + def update(self, **kwargs: Unpack[GroupMetadataModelV2Partial]) -> GroupMetadataModelV2: + """ + Return a new `GroupMetadataModelV2` with the given fields updated. + + Only the constructor-settable fields listed in + `GroupMetadataModelV2Partial` can be updated; the fixed `zarr_format` + is rejected at the type level. Each given field fully replaces its + previous value. + """ + return dataclasses.replace(self, **kwargs) + + def to_json(self) -> GroupMetadataV2: + out: GroupMetadataV2 = {"zarr_format": self.zarr_format} + if len(self.attributes) > 0: + out["attributes"] = self.attributes + return out + + @classmethod + def from_json(cls, data: object) -> GroupMetadataModelV2: + parsed = parse_group_metadata_v2(arrays_to_tuples(data)) + return cls(attributes=dict(parsed.get("attributes", {}))) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV2: + zgroup = json.loads(mapping[GROUP_METADATA_STORE_KEY_V2]) + zattrs: dict[str, JSONValue] = ( + json.loads(mapping[ATTRIBUTES_STORE_KEY_V2]) + if ATTRIBUTES_STORE_KEY_V2 in mapping + else {} + ) + return cls.from_json({**zgroup, "attributes": zattrs}) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + # Attributes live only in the sibling `.zattrs` file; the `.zgroup` + # document must exclude them. + zgroup = {k: v for k, v in self.to_json().items() if k != "attributes"} + return { + GROUP_METADATA_STORE_KEY_V2: json.dumps(zgroup, indent=indent).encode("utf-8"), + ATTRIBUTES_STORE_KEY_V2: json.dumps(self.attributes, indent=indent).encode("utf-8"), + } + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ConsolidatedMetadataModelV2: + """In-memory model of a v2 `.zmetadata` document. + + The `metadata` map holds the flat file-keyed entries (`"path/.zarray"`, + `"path/.zattrs"`, ...) verbatim, preserving byte-faithful round-tripping. + Entries are deliberately NOT merged into per-node models: which nodes had + a `.zattrs` file at all is information the canonical representation must + keep. Interpreting entries into node models is consumer work. + """ + + zarr_consolidated_format: Literal[1] = field(default=1, init=False) + metadata: dict[str, JSONValue] + + def to_json(self) -> dict[str, JSONValue]: + return { + "zarr_consolidated_format": self.zarr_consolidated_format, + "metadata": self.metadata, + } + + @classmethod + def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: + if not isinstance(data, Mapping): + raise MetadataValidationError([ValidationProblem((), "expected a mapping")]) + doc = cast("Mapping[str, object]", data) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key") + for key in ("zarr_consolidated_format", "metadata") + if key not in doc + ] + if "metadata" in doc: + entries = doc["metadata"] + if not isinstance(entries, Mapping) or not all( + isinstance(k, str) for k in cast("Mapping[object, object]", entries) + ): + problems.append( + ValidationProblem(("metadata",), "expected a mapping with string keys") + ) + if problems: + raise MetadataValidationError(problems) + entries_tupled = cast( + "dict[str, JSONValue]", + arrays_to_tuples(dict(cast("Mapping[str, object]", doc["metadata"]))), + ) + return cls(metadata=entries_tupled) + + @classmethod + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ConsolidatedMetadataModelV2: + return cls.from_json(json.loads(mapping[CONSOLIDATED_METADATA_STORE_KEY_V2])) + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + return { + CONSOLIDATED_METADATA_STORE_KEY_V2: json.dumps(self.to_json(), indent=indent).encode( + "utf-8" + ) + } diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 25f3422c50..c69a80c144 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -14,7 +14,13 @@ from typing_extensions import TypeIs -from zarr_metadata import ArrayMetadataV2, ArrayMetadataV3, MetadataV3 +from zarr_metadata import ( + ArrayMetadataV2, + ArrayMetadataV3, + GroupMetadataV2, + GroupMetadataV3, + MetadataV3, +) from zarr_metadata._common import JSONValue @@ -99,6 +105,22 @@ def parse_json(value: object) -> JSONValue: ArrayMetadataV2.__required_keys__ ) +# The standard top-level keys of a v3 group metadata document. Anything outside +# this set is an extension field. +GROUP_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( + GroupMetadataV3.__required_keys__ +) +GROUP_METADATA_OPTIONAL_KEYS_V3: Final[frozenset[str]] = frozenset( + GroupMetadataV3.__optional_keys__ +) +GROUP_METADATA_STANDARD_KEYS_V3: Final[frozenset[str]] = ( + GROUP_METADATA_REQUIRED_KEYS_V3 | GROUP_METADATA_OPTIONAL_KEYS_V3 +) + +GROUP_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( + GroupMetadataV2.__required_keys__ +) + def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: """Return every reason `value` is not a v3 metadata field. @@ -259,6 +281,71 @@ def parse_array_metadata_v2(value: object) -> ArrayMetadataV2: return cast(ArrayMetadataV2, value) +def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v3 group doc. + + Checks structure, not domain validity. Unknown top-level keys are allowed + (they map to `extra_fields`); a `consolidated_metadata` key, if present, + must be a mapping (its entries are validated by the consolidated model). + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key") + for key in sorted(GROUP_METADATA_REQUIRED_KEYS_V3 - doc.keys()) + ] + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + if "consolidated_metadata" in doc and not isinstance(doc["consolidated_metadata"], Mapping): + problems.append(ValidationProblem(("consolidated_metadata",), "expected a mapping")) + return problems + + +def is_group_metadata_v3(value: object) -> TypeIs[GroupMetadataV3]: + """Whether `value` is a structurally-valid v3 group metadata document.""" + return not validate_group_metadata_v3(value) + + +def parse_group_metadata_v3(value: object) -> GroupMetadataV3: + """Return `value` narrowed to `GroupMetadataV3`, or raise `MetadataValidationError`.""" + problems = validate_group_metadata_v3(value) + if problems: + raise MetadataValidationError(problems) + return cast(GroupMetadataV3, value) + + +def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a structurally-valid v2 group doc. + + Validates the in-memory merged form: the `.zgroup` fields plus an + optional `attributes` mapping folded in from `.zattrs`. + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping")] + doc = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key") + for key in sorted(GROUP_METADATA_REQUIRED_KEYS_V2 - doc.keys()) + ] + if "attributes" in doc: + problems.extend(_validate_attributes(doc["attributes"])) + return problems + + +def is_group_metadata_v2(value: object) -> TypeIs[GroupMetadataV2]: + """Whether `value` is a structurally-valid v2 group metadata document.""" + return not validate_group_metadata_v2(value) + + +def parse_group_metadata_v2(value: object) -> GroupMetadataV2: + """Return `value` narrowed to `GroupMetadataV2`, or raise `MetadataValidationError`.""" + problems = validate_group_metadata_v2(value) + if problems: + raise MetadataValidationError(problems) + return cast(GroupMetadataV2, value) + + def arrays_to_tuples(obj: object) -> object: """Recursively convert every list in a JSON-decoded structure to a tuple.""" if isinstance(obj, list): diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py new file mode 100644 index 0000000000..1e98cf3e47 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -0,0 +1,137 @@ +"""Tests for the group and consolidated metadata models in `zarr_metadata.model`.""" + +import json + +import pytest + +from zarr_metadata.model._group import ( + GroupMetadataModelV2, + GroupMetadataModelV3, +) +from zarr_metadata.model._validation import ( + MetadataValidationError, + parse_group_metadata_v2, + parse_group_metadata_v3, +) + +# --- GroupMetadataModelV3 --------------------------------------------------- + + +def test_group_v3_roundtrip() -> None: + """A v3 group document round-trips through the model unchanged.""" + doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": (1, 2)}} + model = GroupMetadataModelV3.from_json(doc) + assert model.to_json() == doc + + +def test_group_v3_omits_empty_attributes() -> None: + """to_json omits the attributes key when attributes is empty.""" + model = GroupMetadataModelV3.create_default() + assert "attributes" not in model.to_json() + + +def test_group_v3_lists_become_tuples() -> None: + """from_json converts JSON arrays in attributes to tuples.""" + doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": [1, 2]}} + model = GroupMetadataModelV3.from_json(doc) + assert model.attributes == {"a": (1, 2)} + + +def test_group_v3_extra_fields_roundtrip() -> None: + """Unknown top-level keys land in extra_fields and reappear in to_json.""" + doc = { + "zarr_format": 3, + "node_type": "group", + "my_extension": {"name": "thing", "must_understand": False}, + } + model = GroupMetadataModelV3.from_json(doc) + assert model.extra_fields == {"my_extension": {"name": "thing", "must_understand": False}} + assert model.to_json() == doc + + +def test_group_v3_extra_fields_overlap_rejected() -> None: + """Constructing a v3 group model with extra_fields shadowing a standard key raises.""" + with pytest.raises(ValueError, match="Extra fields"): + GroupMetadataModelV3( + attributes={}, + consolidated_metadata=None, + extra_fields={"node_type": {"name": "x", "must_understand": False}}, + ) + + +def test_group_v3_consolidated_extra_field_rejected() -> None: + """extra_fields may not shadow the consolidated_metadata convention key.""" + with pytest.raises(ValueError, match="Extra fields"): + GroupMetadataModelV3( + attributes={}, + consolidated_metadata=None, + extra_fields={"consolidated_metadata": {"name": "x", "must_understand": False}}, + ) + + +def test_group_v3_missing_required_key() -> None: + """parse_group_metadata_v3 reports each missing required key.""" + with pytest.raises(MetadataValidationError, match="node_type"): + parse_group_metadata_v3({"zarr_format": 3}) + + +def test_group_v3_bad_attributes() -> None: + """parse_group_metadata_v3 rejects a non-mapping attributes value.""" + with pytest.raises(MetadataValidationError, match="attributes"): + parse_group_metadata_v3({"zarr_format": 3, "node_type": "group", "attributes": 5}) + + +def test_group_v3_key_value_roundtrip() -> None: + """from_key_value(to_key_value()) is the identity for v3 groups.""" + model = GroupMetadataModelV3.create_default(attributes={"a": 1}) + assert GroupMetadataModelV3.from_key_value(model.to_key_value()) == model + + +def test_group_v3_update() -> None: + """update replaces the given fields and returns a new instance.""" + base = GroupMetadataModelV3.create_default() + updated = base.update(attributes={"a": 1}) + assert updated.attributes == {"a": 1} + assert base.attributes == {} + + +# --- GroupMetadataModelV2 --------------------------------------------------- + + +def test_group_v2_key_value_split() -> None: + """v2 to_key_value writes .zgroup and .zattrs; from_key_value merges them.""" + model = GroupMetadataModelV2.create_default(attributes={"a": 1}) + kv = model.to_key_value() + assert set(kv) == {".zgroup", ".zattrs"} + assert json.loads(kv[".zgroup"]) == {"zarr_format": 2} + assert GroupMetadataModelV2.from_key_value(kv) == model + + +def test_group_v2_from_key_value_without_zattrs() -> None: + """A v2 group with no .zattrs file parses with empty attributes.""" + model = GroupMetadataModelV2.from_key_value({".zgroup": b'{"zarr_format": 2}'}) + assert model.attributes == {} + + +def test_group_v2_json_roundtrip() -> None: + """A merged-form v2 group document round-trips through the model unchanged.""" + doc = {"zarr_format": 2, "attributes": {"a": 1}} + model = GroupMetadataModelV2.from_json(doc) + assert model.to_json() == doc + + +def test_group_v2_omits_empty_attributes() -> None: + """to_json omits the attributes key when attributes is empty.""" + assert "attributes" not in GroupMetadataModelV2.create_default().to_json() + + +def test_group_v2_not_a_mapping() -> None: + """parse_group_metadata_v2 rejects a non-mapping document.""" + with pytest.raises(MetadataValidationError, match="expected a mapping"): + parse_group_metadata_v2([1, 2, 3]) + + +def test_group_v2_missing_required_key() -> None: + """parse_group_metadata_v2 reports a missing zarr_format key.""" + with pytest.raises(MetadataValidationError, match="zarr_format"): + parse_group_metadata_v2({}) From a85319f69d381d648fe28e9411e093228789a022 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 13:27:15 +0200 Subject: [PATCH 06/70] test(zarr-metadata): add consolidated metadata model tests Assisted-by: ClaudeCode:claude-fable-5 --- .../zarr-metadata/tests/model/test_group.py | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 1e98cf3e47..909346586b 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -1,12 +1,18 @@ """Tests for the group and consolidated metadata models in `zarr_metadata.model`.""" +import dataclasses import json import pytest +from zarr_metadata.model._array import ArrayMetadataModelV3 from zarr_metadata.model._group import ( + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, GroupMetadataModelV2, + GroupMetadataModelV2Partial, GroupMetadataModelV3, + GroupMetadataModelV3Partial, ) from zarr_metadata.model._validation import ( MetadataValidationError, @@ -135,3 +141,127 @@ def test_group_v2_missing_required_key() -> None: """parse_group_metadata_v2 reports a missing zarr_format key.""" with pytest.raises(MetadataValidationError, match="zarr_format"): parse_group_metadata_v2({}) + + +# --- Partial TypedDict drift guards ----------------------------------------- + + +def test_group_partial_keys_match_settable_model_fields() -> None: + """Each group partial TypedDict must list exactly the settable model fields. + + Guards against drift: adding/removing a settable field on a group model + without updating its `*Partial` TypedDict fails here. + """ + for model_cls, partial_cls in ( + (GroupMetadataModelV3, GroupMetadataModelV3Partial), + (GroupMetadataModelV2, GroupMetadataModelV2Partial), + ): + settable = {f.name for f in dataclasses.fields(model_cls) if f.init} + assert set(partial_cls.__annotations__) == settable + + +# --- ConsolidatedMetadataModelV3 -------------------------------------------- + + +def test_consolidated_v3_roundtrip() -> None: + """A v3 group with inline consolidated metadata round-trips, with child + entries parsed into array/group models.""" + child = ArrayMetadataModelV3.create_default(shape=(2,)).to_json() + doc = { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": child, "g": {"zarr_format": 3, "node_type": "group"}}, + }, + } + model = GroupMetadataModelV3.from_json(doc) + assert isinstance(model.consolidated_metadata, ConsolidatedMetadataModelV3) + assert isinstance(model.consolidated_metadata.metadata["a"], ArrayMetadataModelV3) + assert isinstance(model.consolidated_metadata.metadata["g"], GroupMetadataModelV3) + assert model.to_json() == doc + + +def test_consolidated_v3_must_understand_true_rejected() -> None: + """ConsolidatedMetadataModelV3 enforces must_understand=False at runtime.""" + with pytest.raises(ValueError, match="must_understand"): + ConsolidatedMetadataModelV3(must_understand=True, metadata={}) + + +def test_consolidated_v3_from_json_must_understand_true_rejected() -> None: + """from_json rejects a consolidated document carrying must_understand=true.""" + with pytest.raises(MetadataValidationError, match="must_understand"): + ConsolidatedMetadataModelV3.from_json( + {"kind": "inline", "must_understand": True, "metadata": {}} + ) + + +def test_consolidated_v3_entry_without_node_type_rejected() -> None: + """from_json rejects a consolidated entry lacking a recognizable node_type.""" + with pytest.raises(MetadataValidationError, match="node_type"): + ConsolidatedMetadataModelV3.from_json( + {"kind": "inline", "must_understand": False, "metadata": {"a": {"zarr_format": 3}}} + ) + + +def test_consolidated_v3_not_a_mapping() -> None: + """from_json rejects a non-mapping consolidated document.""" + with pytest.raises(MetadataValidationError, match="expected a mapping"): + ConsolidatedMetadataModelV3.from_json(5) + + +# --- ConsolidatedMetadataModelV2 -------------------------------------------- + + +def test_consolidated_v2_verbatim_roundtrip() -> None: + """The v2 .zmetadata model holds the flat file-keyed map verbatim, + including nodes that have no .zattrs entry.""" + doc = { + "zarr_consolidated_format": 1, + "metadata": { + ".zgroup": {"zarr_format": 2}, + "a/.zarray": { + "zarr_format": 2, + "shape": (2,), + "chunks": (2,), + "dtype": "|u1", + "fill_value": 0, + "order": "C", + "compressor": None, + "filters": None, + }, + }, + } + model = ConsolidatedMetadataModelV2.from_json(doc) + assert model.to_json() == doc + + +def test_consolidated_v2_key_value_roundtrip() -> None: + """from_key_value(to_key_value()) is the identity for .zmetadata documents.""" + model = ConsolidatedMetadataModelV2.from_json( + {"zarr_consolidated_format": 1, "metadata": {".zgroup": {"zarr_format": 2}}} + ) + assert ConsolidatedMetadataModelV2.from_key_value(model.to_key_value()) == model + + +def test_consolidated_v2_lists_become_tuples() -> None: + """from_json converts JSON arrays inside entries to tuples.""" + doc = { + "zarr_consolidated_format": 1, + "metadata": {"a/.zarray": {"shape": [2, 3]}}, + } + model = ConsolidatedMetadataModelV2.from_json(doc) + assert model.metadata == {"a/.zarray": {"shape": (2, 3)}} + + +def test_consolidated_v2_envelope_validation() -> None: + """from_json rejects a .zmetadata document missing the metadata key.""" + with pytest.raises(MetadataValidationError, match="metadata"): + ConsolidatedMetadataModelV2.from_json({"zarr_consolidated_format": 1}) + + +def test_consolidated_v2_not_a_mapping() -> None: + """from_json rejects a non-mapping .zmetadata document.""" + with pytest.raises(MetadataValidationError, match="expected a mapping"): + ConsolidatedMetadataModelV2.from_json([1]) From fbfb425ad913cdc2424f51db029c57ec7377207d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 13:30:18 +0200 Subject: [PATCH 07/70] feat(zarr-metadata): export model layer from package front door Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/__init__.py | 28 +++++++++++++ .../src/zarr_metadata/model/__init__.py | 42 +++++++++++++++++++ .../src/zarr_metadata/model/_array.py | 12 +++--- .../src/zarr_metadata/model/_group.py | 10 ++--- .../src/zarr_metadata/model/_validation.py | 12 +++--- .../zarr-metadata/tests/test_public_api.py | 14 +++++++ 6 files changed, 99 insertions(+), 19 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index 46949570a2..d522c89cd2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -1,6 +1,21 @@ from importlib.metadata import version from zarr_metadata._common import JSONValue, NamedConfigV3 +from zarr_metadata.model import ( + ArrayMetadataModelV2, + ArrayMetadataModelV2Partial, + ArrayMetadataModelV3, + ArrayMetadataModelV3Partial, + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, + GroupMetadataModelV2, + GroupMetadataModelV2Partial, + GroupMetadataModelV3, + GroupMetadataModelV3Partial, + MetadataValidationError, + ValidationProblem, + ZarrMetadataV3, +) from zarr_metadata.v2.array import ( ARRAY_DIMENSION_SEPARATOR_V2, ARRAY_ORDER_V2, @@ -235,6 +250,10 @@ "V2_CHUNK_KEY_ENCODING_SEPARATOR", "ZSTD_CODEC_NAME", "ArrayDimensionSeparatorV2", + "ArrayMetadataModelV2", + "ArrayMetadataModelV2Partial", + "ArrayMetadataModelV3", + "ArrayMetadataModelV3Partial", "ArrayMetadataV2", "ArrayMetadataV2Partial", "ArrayMetadataV3", @@ -259,6 +278,8 @@ "Complex64FillValue", "Complex128DataTypeName", "Complex128FillValue", + "ConsolidatedMetadataModelV2", + "ConsolidatedMetadataModelV3", "ConsolidatedMetadataV2", "ConsolidatedMetadataV3", "Crc32cCodecMetadata", @@ -275,6 +296,10 @@ "Float32FillValue", "Float64DataTypeName", "Float64FillValue", + "GroupMetadataModelV2", + "GroupMetadataModelV2Partial", + "GroupMetadataModelV3", + "GroupMetadataModelV3Partial", "GroupMetadataV2", "GroupMetadataV2Partial", "GroupMetadataV3", @@ -291,6 +316,7 @@ "Int64FillValue", "JSONValue", "MetadataV3", + "MetadataValidationError", "NamedConfigV3", "NumpyDatetime64DataTypeName", "NumpyDatetime64FillValue", @@ -325,9 +351,11 @@ "V2ChunkKeyEncodingMetadata", "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingSeparator", + "ValidationProblem", "ZArrayMetadata", "ZAttrsMetadata", "ZGroupMetadata", + "ZarrMetadataV3", "ZstdCodecMetadata", "ZstdCodecName", "__version__", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 8776b4fab3..68e36be0a7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -10,23 +10,45 @@ ArrayMetadataModelV3Partial, ZarrMetadataV3, ) +from zarr_metadata.model._group import ( + CONSOLIDATED_METADATA_KEY_V3, + CONSOLIDATED_METADATA_STORE_KEY_V2, + GROUP_METADATA_STORE_KEY_V2, + GROUP_METADATA_STORE_KEY_V3, + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, + GroupMetadataModelV2, + GroupMetadataModelV2Partial, + GroupMetadataModelV3, + GroupMetadataModelV3Partial, +) from zarr_metadata.model._validation import ( ARRAY_METADATA_OPTIONAL_KEYS_V3, ARRAY_METADATA_REQUIRED_KEYS_V2, ARRAY_METADATA_REQUIRED_KEYS_V3, ARRAY_METADATA_STANDARD_KEYS_V3, + GROUP_METADATA_OPTIONAL_KEYS_V3, + GROUP_METADATA_REQUIRED_KEYS_V2, + GROUP_METADATA_REQUIRED_KEYS_V3, + GROUP_METADATA_STANDARD_KEYS_V3, MetadataValidationError, ValidationProblem, is_array_metadata_v2, is_array_metadata_v3, + is_group_metadata_v2, + is_group_metadata_v3, is_json, is_metadata_field_v3, parse_array_metadata_v2, parse_array_metadata_v3, + parse_group_metadata_v2, + parse_group_metadata_v3, parse_json, parse_metadata_field_v3, validate_array_metadata_v2, validate_array_metadata_v3, + validate_group_metadata_v2, + validate_group_metadata_v3, validate_json, validate_metadata_field_v3, ) @@ -39,23 +61,43 @@ "ARRAY_METADATA_STORE_KEY_V2", "ARRAY_METADATA_STORE_KEY_V3", "ATTRIBUTES_STORE_KEY_V2", + "CONSOLIDATED_METADATA_KEY_V3", + "CONSOLIDATED_METADATA_STORE_KEY_V2", + "GROUP_METADATA_OPTIONAL_KEYS_V3", + "GROUP_METADATA_REQUIRED_KEYS_V2", + "GROUP_METADATA_REQUIRED_KEYS_V3", + "GROUP_METADATA_STANDARD_KEYS_V3", + "GROUP_METADATA_STORE_KEY_V2", + "GROUP_METADATA_STORE_KEY_V3", "ArrayMetadataModelV2", "ArrayMetadataModelV2Partial", "ArrayMetadataModelV3", "ArrayMetadataModelV3Partial", + "ConsolidatedMetadataModelV2", + "ConsolidatedMetadataModelV3", + "GroupMetadataModelV2", + "GroupMetadataModelV2Partial", + "GroupMetadataModelV3", + "GroupMetadataModelV3Partial", "MetadataValidationError", "ValidationProblem", "ZarrMetadataV3", "is_array_metadata_v2", "is_array_metadata_v3", + "is_group_metadata_v2", + "is_group_metadata_v3", "is_json", "is_metadata_field_v3", "parse_array_metadata_v2", "parse_array_metadata_v3", + "parse_group_metadata_v2", + "parse_group_metadata_v3", "parse_json", "parse_metadata_field_v3", "validate_array_metadata_v2", "validate_array_metadata_v3", + "validate_group_metadata_v2", + "validate_group_metadata_v3", "validate_json", "validate_metadata_field_v3", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index ea2745284d..e5b23918d4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -20,16 +20,16 @@ if TYPE_CHECKING: from collections.abc import Mapping - from zarr_metadata import ( + from zarr_metadata._common import JSONValue + from zarr_metadata.v2.array import ( ArrayDimensionSeparatorV2, ArrayMetadataV2, - ArrayMetadataV3, ArrayOrderV2, - ExtensionFieldV3, - MetadataV3, + DataTypeMetadataV2, ) - from zarr_metadata._common import JSONValue - from zarr_metadata.v2 import CodecMetadataV2, DataTypeMetadataV2 + from zarr_metadata.v2.codec import CodecMetadataV2 + from zarr_metadata.v3._common import MetadataV3 + from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 ArrayMetadataStoreKeyV3 = Literal["zarr.json"] ARRAY_METADATA_STORE_KEY_V3: Final[ArrayMetadataStoreKeyV3] = "zarr.json" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 8eeaed1a89..a061ed5d67 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -24,13 +24,11 @@ ) if TYPE_CHECKING: - from zarr_metadata import ( - ConsolidatedMetadataV3, - ExtensionFieldV3, - GroupMetadataV2, - GroupMetadataV3, - ) from zarr_metadata._common import JSONValue + from zarr_metadata.v2.group import GroupMetadataV2 + from zarr_metadata.v3.array import ExtensionFieldV3 + from zarr_metadata.v3.consolidated import ConsolidatedMetadataV3 + from zarr_metadata.v3.group import GroupMetadataV3 GroupMetadataStoreKeyV3 = Literal["zarr.json"] GROUP_METADATA_STORE_KEY_V3: Final[GroupMetadataStoreKeyV3] = "zarr.json" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index c69a80c144..a38258316b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -14,14 +14,12 @@ from typing_extensions import TypeIs -from zarr_metadata import ( - ArrayMetadataV2, - ArrayMetadataV3, - GroupMetadataV2, - GroupMetadataV3, - MetadataV3, -) from zarr_metadata._common import JSONValue +from zarr_metadata.v2.array import ArrayMetadataV2 +from zarr_metadata.v2.group import GroupMetadataV2 +from zarr_metadata.v3._common import MetadataV3 +from zarr_metadata.v3.array import ArrayMetadataV3 +from zarr_metadata.v3.group import GroupMetadataV3 @dataclass(frozen=True, slots=True) diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index d3270579c3..7a6db7ab30 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -39,6 +39,20 @@ def _group_rank(s: str) -> int: "NamedConfigV3", "MetadataV3", "JSONValue", + # Category A' — metadata models (in-memory dataclasses over the documents) + "ArrayMetadataModelV2", + "ArrayMetadataModelV2Partial", + "ArrayMetadataModelV3", + "ArrayMetadataModelV3Partial", + "GroupMetadataModelV2", + "GroupMetadataModelV2Partial", + "GroupMetadataModelV3", + "GroupMetadataModelV3Partial", + "ConsolidatedMetadataModelV2", + "ConsolidatedMetadataModelV3", + "ZarrMetadataV3", + "ValidationProblem", + "MetadataValidationError", # v2 data-type encoding union "DataTypeMetadataV2", # Category B — codec canonical unions From 99608ce38789fcd9c677d34cdcfd9fa6cce17fb0 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 13:31:40 +0200 Subject: [PATCH 08/70] docs(zarr-metadata): changelog entry for the model layer Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/zarr-metadata/changes/210.feature.md diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md new file mode 100644 index 0000000000..35835c9ebe --- /dev/null +++ b/packages/zarr-metadata/changes/210.feature.md @@ -0,0 +1,7 @@ +Added `zarr_metadata.model`: frozen-dataclass models (`ArrayMetadataModelV2`, +`ArrayMetadataModelV3`, `GroupMetadataModelV2`, `GroupMetadataModelV3`, +`ConsolidatedMetadataModelV2`, `ConsolidatedMetadataModelV3`, `ZarrMetadataV3`) +that are canonical, lossless representations of Zarr metadata documents, plus +structural validators (`validate_*` / `is_*` / `parse_*`). Every v3 extension +point (data type, chunk grid, chunk key encoding, codecs, storage transformers) +is held as a name + configuration pair; nothing is interpreted. From c98a1220493813935a2e451725117952dbe01eed Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 14:58:41 +0200 Subject: [PATCH 09/70] feat(zarr-metadata): harden model validation and error reporting Findings from an API-ergonomics exercise (a fresh agent consuming defective metadata documents): - ValidationProblem gains a machine-readable kind (missing_key / invalid_type / invalid_value / invalid_json), ending message string-matching in consumers. - The v2 array validator now enforces what its types declare (dtype, order, compressor, filters, dimension_separator), and all four document validators check the fixed zarr_format / node_type literals. - All ingestion failures surface as MetadataValidationError: missing store keys and undecodable bytes in from_key_value (previously KeyError / JSONDecodeError) and constructor invariants (previously bare ValueError). - ZarrMetadataV3 is renamed NamedConfigModelV3: it models a name + configuration pair, and the old name read as a whole-document type. - Discoverability: the validate_*/is_*/parse_* contract is documented on zarr_metadata.model itself; update() documents that it does not re-validate; the v2 to_json/to_key_value attributes split is documented on both. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 11 +- .../src/zarr_metadata/__init__.py | 6 +- .../src/zarr_metadata/model/__init__.py | 19 +- .../src/zarr_metadata/model/_array.py | 76 ++++-- .../src/zarr_metadata/model/_group.py | 67 +++-- .../src/zarr_metadata/model/_validation.py | 237 +++++++++++++++--- .../zarr-metadata/tests/model/test_array.py | 187 +++++++++++--- .../zarr-metadata/tests/model/test_group.py | 19 ++ .../zarr-metadata/tests/test_public_api.py | 3 +- 9 files changed, 501 insertions(+), 124 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 35835c9ebe..13517ce38a 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -1,7 +1,16 @@ Added `zarr_metadata.model`: frozen-dataclass models (`ArrayMetadataModelV2`, `ArrayMetadataModelV3`, `GroupMetadataModelV2`, `GroupMetadataModelV3`, -`ConsolidatedMetadataModelV2`, `ConsolidatedMetadataModelV3`, `ZarrMetadataV3`) +`ConsolidatedMetadataModelV2`, `ConsolidatedMetadataModelV3`, `NamedConfigModelV3`) that are canonical, lossless representations of Zarr metadata documents, plus structural validators (`validate_*` / `is_*` / `parse_*`). Every v3 extension point (data type, chunk grid, chunk key encoding, codecs, storage transformers) is held as a name + configuration pair; nothing is interpreted. + +Validation is strict about what the types declare: v2 `dtype` / `order` / +`compressor` / `filters` / `dimension_separator` shapes and the fixed +`zarr_format` / `node_type` literals are all enforced. Every +`ValidationProblem` carries a machine-readable `kind` +(`missing_key` / `invalid_type` / `invalid_value` / `invalid_json`) so +consumers can dispatch on the failure mode without matching message strings, +and every ingestion failure — including missing store keys and undecodable +bytes in `from_key_value` — surfaces as `MetadataValidationError`. diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index d522c89cd2..ecd7e342cf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -13,8 +13,9 @@ GroupMetadataModelV3, GroupMetadataModelV3Partial, MetadataValidationError, + NamedConfigModelV3, + ProblemKind, ValidationProblem, - ZarrMetadataV3, ) from zarr_metadata.v2.array import ( ARRAY_DIMENSION_SEPARATOR_V2, @@ -317,12 +318,14 @@ "JSONValue", "MetadataV3", "MetadataValidationError", + "NamedConfigModelV3", "NamedConfigV3", "NumpyDatetime64DataTypeName", "NumpyDatetime64FillValue", "NumpyTimeUnit", "NumpyTimedelta64DataTypeName", "NumpyTimedelta64FillValue", + "ProblemKind", "RawBytesDataTypeName", "RawBytesFillValue", "RectilinearChunkGridMetadata", @@ -355,7 +358,6 @@ "ZArrayMetadata", "ZAttrsMetadata", "ZGroupMetadata", - "ZarrMetadataV3", "ZstdCodecMetadata", "ZstdCodecName", "__version__", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 68e36be0a7..aa8555d623 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -1,4 +1,15 @@ -"""In-memory models for Zarr metadata documents.""" +"""In-memory models for Zarr metadata documents. + +Models are frozen dataclasses that hold a canonical, lossless representation +of the JSON documents; they never interpret extension points (codecs, chunk +grids, data types). Validators check JSON structure, not domain validity. +Each document concept gets a `validate_*` function returning every problem +found (a `list[ValidationProblem]`, each with a machine-readable `kind`), an +`is_*` type guard, and a `parse_*` function that narrows or raises +`MetadataValidationError`. Model `from_json` / `from_key_value` constructors +raise `MetadataValidationError` for every ingestion failure, including +missing store keys and undecodable bytes. +""" from zarr_metadata.model._array import ( ARRAY_METADATA_STORE_KEY_V2, @@ -8,7 +19,7 @@ ArrayMetadataModelV2Partial, ArrayMetadataModelV3, ArrayMetadataModelV3Partial, - ZarrMetadataV3, + NamedConfigModelV3, ) from zarr_metadata.model._group import ( CONSOLIDATED_METADATA_KEY_V3, @@ -32,6 +43,7 @@ GROUP_METADATA_REQUIRED_KEYS_V3, GROUP_METADATA_STANDARD_KEYS_V3, MetadataValidationError, + ProblemKind, ValidationProblem, is_array_metadata_v2, is_array_metadata_v3, @@ -80,8 +92,9 @@ "GroupMetadataModelV3", "GroupMetadataModelV3Partial", "MetadataValidationError", + "NamedConfigModelV3", + "ProblemKind", "ValidationProblem", - "ZarrMetadataV3", "is_array_metadata_v2", "is_array_metadata_v3", "is_group_metadata_v2", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index e5b23918d4..4ec02137ec 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -11,7 +11,10 @@ from zarr_metadata.model._validation import ( ARRAY_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ValidationProblem, arrays_to_tuples, + load_store_json, parse_array_metadata_v2, parse_array_metadata_v3, parse_metadata_field_v3, @@ -42,7 +45,7 @@ @dataclass(frozen=True, slots=True, kw_only=True) -class ZarrMetadataV3: +class NamedConfigModelV3: """A v3 metadata field in normalized form: a name plus a configuration. This is the in-memory model of `MetadataV3` (a bare name string or a @@ -57,7 +60,7 @@ def to_json(self) -> MetadataV3: return {"name": self.name, "configuration": self.configuration} @classmethod - def from_json(cls, data: object) -> ZarrMetadataV3: + def from_json(cls, data: object) -> NamedConfigModelV3: field = parse_metadata_field_v3(data) if isinstance(field, str): return cls(name=field, configuration={}) @@ -81,13 +84,13 @@ class ArrayMetadataModelV3Partial(TypedDict, total=False): shape: tuple[int, ...] fill_value: JSONValue - data_type: ZarrMetadataV3 - chunk_grid: ZarrMetadataV3 - codecs: tuple[ZarrMetadataV3, ...] - chunk_key_encoding: ZarrMetadataV3 + data_type: NamedConfigModelV3 + chunk_grid: NamedConfigModelV3 + codecs: tuple[NamedConfigModelV3, ...] + chunk_key_encoding: NamedConfigModelV3 dimension_names: tuple[str | None, ...] | None attributes: dict[str, JSONValue] - storage_transformers: tuple[ZarrMetadataV3, ...] + storage_transformers: tuple[NamedConfigModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @@ -97,7 +100,7 @@ class ArrayMetadataModelV3: A canonical, lossless representation of the `zarr.json` content for an array. Extension points (`data_type`, `chunk_grid`, `chunk_key_encoding`, - `codecs`, `storage_transformers`) are held as `ZarrMetadataV3` name + + `codecs`, `storage_transformers`) are held as `NamedConfigModelV3` name + configuration pairs and are never interpreted; `fill_value` is held verbatim in its JSON form. """ @@ -106,13 +109,13 @@ class ArrayMetadataModelV3: node_type: Literal["array"] = field(default="array", init=False) shape: tuple[int, ...] fill_value: JSONValue - data_type: ZarrMetadataV3 - chunk_grid: ZarrMetadataV3 - codecs: tuple[ZarrMetadataV3, ...] - chunk_key_encoding: ZarrMetadataV3 + data_type: NamedConfigModelV3 + chunk_grid: NamedConfigModelV3 + codecs: tuple[NamedConfigModelV3, ...] + chunk_key_encoding: NamedConfigModelV3 dimension_names: tuple[str | None, ...] | None attributes: dict[str, JSONValue] - storage_transformers: tuple[ZarrMetadataV3, ...] + storage_transformers: tuple[NamedConfigModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @classmethod @@ -129,10 +132,10 @@ def create_default( default = cls( shape=(), fill_value=0, - data_type=ZarrMetadataV3(name="uint8", configuration={}), - chunk_grid=ZarrMetadataV3(name="regular", configuration={"chunk_shape": ()}), - codecs=(ZarrMetadataV3(name="bytes", configuration={}),), - chunk_key_encoding=ZarrMetadataV3(name="default", configuration={}), + data_type=NamedConfigModelV3(name="uint8", configuration={}), + chunk_grid=NamedConfigModelV3(name="regular", configuration={"chunk_shape": ()}), + codecs=(NamedConfigModelV3(name="bytes", configuration={}),), + chunk_key_encoding=NamedConfigModelV3(name="default", configuration={}), dimension_names=None, attributes={}, storage_transformers=(), @@ -152,12 +155,25 @@ def update(self, **kwargs: Unpack[ArrayMetadataModelV3Partial]) -> ArrayMetadata This is useful for test fixtures that want to override a few fields of a base template without having to re-specify the entire document. + + No re-validation is performed (`update` is `dataclasses.replace`), so + a repair or edit can produce an invalid document; validity is checked + on `from_json`, not on field replacement. """ return dataclasses.replace(self, **kwargs) def __post_init__(self) -> None: - if set(self.extra_fields.keys()).intersection(ARRAY_METADATA_STANDARD_KEYS_V3): - raise ValueError("Extra fields cannot overlap with standard ArrayMetadataV3 fields") + overlap = set(self.extra_fields.keys()).intersection(ARRAY_METADATA_STANDARD_KEYS_V3) + if overlap: + raise MetadataValidationError( + [ + ValidationProblem( + ("extra_fields",), + "Extra fields cannot overlap with standard ArrayMetadataV3 fields", + "invalid_value", + ) + ] + ) def to_json(self) -> ArrayMetadataV3: out: ArrayMetadataV3 = { @@ -197,21 +213,21 @@ def from_json(cls, data: object) -> ArrayMetadataModelV3: return cls( shape=parsed["shape"], fill_value=parsed["fill_value"], # type: ignore[arg-type] # fill_value: object in upstream TypedDict - data_type=ZarrMetadataV3.from_json(parsed["data_type"]), - chunk_grid=ZarrMetadataV3.from_json(parsed["chunk_grid"]), - codecs=tuple(ZarrMetadataV3.from_json(c) for c in parsed["codecs"]), - chunk_key_encoding=ZarrMetadataV3.from_json(parsed["chunk_key_encoding"]), + data_type=NamedConfigModelV3.from_json(parsed["data_type"]), + chunk_grid=NamedConfigModelV3.from_json(parsed["chunk_grid"]), + codecs=tuple(NamedConfigModelV3.from_json(c) for c in parsed["codecs"]), + chunk_key_encoding=NamedConfigModelV3.from_json(parsed["chunk_key_encoding"]), dimension_names=parsed.get("dimension_names"), attributes=dict(parsed.get("attributes", {})), storage_transformers=tuple( - ZarrMetadataV3.from_json(t) for t in parsed.get("storage_transformers", ()) + NamedConfigModelV3.from_json(t) for t in parsed.get("storage_transformers", ()) ), extra_fields=extra_fields, ) @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV3: - return cls.from_json(json.loads(mapping[ARRAY_METADATA_STORE_KEY_V3])) + return cls.from_json(load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V3)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: return { @@ -299,6 +315,12 @@ def create_default( return default.update(**overrides) def to_json(self) -> ArrayMetadataV2: + """Return the merged in-memory document form, INCLUDING `attributes`. + + This is not the on-disk `.zarray` content: a conforming `.zarray` must + exclude `attributes` (they live in the sibling `.zattrs` file). Use + `to_key_value` to produce the spec-conforming split for storage. + """ out: ArrayMetadataV2 = { "zarr_format": self.zarr_format, "shape": self.shape, @@ -330,9 +352,9 @@ def from_json(cls, data: object) -> ArrayMetadataModelV2: @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV2: - zarray = json.loads(mapping[ARRAY_METADATA_STORE_KEY_V2]) + zarray = load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V2) zattrs: dict[str, JSONValue] = ( - json.loads(mapping[ATTRIBUTES_STORE_KEY_V2]) + load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) if ATTRIBUTES_STORE_KEY_V2 in mapping else {} ) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index a061ed5d67..6128b0306b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -19,6 +19,7 @@ MetadataValidationError, ValidationProblem, arrays_to_tuples, + load_store_json, parse_group_metadata_v2, parse_group_metadata_v3, ) @@ -83,7 +84,15 @@ class GroupMetadataModelV3: def __post_init__(self) -> None: reserved = GROUP_METADATA_STANDARD_KEYS_V3 | {CONSOLIDATED_METADATA_KEY_V3} if set(self.extra_fields.keys()).intersection(reserved): - raise ValueError("Extra fields cannot overlap with standard GroupMetadataV3 fields") + raise MetadataValidationError( + [ + ValidationProblem( + ("extra_fields",), + "Extra fields cannot overlap with standard GroupMetadataV3 fields", + "invalid_value", + ) + ] + ) @classmethod def create_default( @@ -151,7 +160,7 @@ def from_json(cls, data: object) -> GroupMetadataModelV3: @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV3: - return cls.from_json(json.loads(mapping[GROUP_METADATA_STORE_KEY_V3])) + return cls.from_json(load_store_json(mapping, GROUP_METADATA_STORE_KEY_V3)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: return { @@ -176,9 +185,15 @@ class ConsolidatedMetadataModelV3: def __post_init__(self) -> None: if self.must_understand is not False: - raise ValueError( - f"Invalid value for 'must_understand'. Expected False. " - f"Got {self.must_understand!r}." + raise MetadataValidationError( + [ + ValidationProblem( + ("must_understand",), + f"Invalid value for 'must_understand'. Expected False. " + f"Got {self.must_understand!r}.", + "invalid_value", + ) + ] ) def to_json(self) -> ConsolidatedMetadataV3: @@ -193,16 +208,22 @@ def to_json(self) -> ConsolidatedMetadataV3: @classmethod def from_json(cls, data: object) -> ConsolidatedMetadataModelV3: if not isinstance(data, Mapping): - raise MetadataValidationError([ValidationProblem((), "expected a mapping")]) + raise MetadataValidationError( + [ValidationProblem((), "expected a mapping", "invalid_type")] + ) doc = cast("Mapping[str, object]", data) entries_raw = doc.get("metadata") if not isinstance(entries_raw, Mapping): - raise MetadataValidationError([ValidationProblem(("metadata",), "expected a mapping")]) + raise MetadataValidationError( + [ValidationProblem(("metadata",), "expected a mapping", "invalid_type")] + ) entries: dict[str, ArrayMetadataModelV3 | GroupMetadataModelV3] = {} problems: list[ValidationProblem] = [] for key, entry in cast("Mapping[object, object]", entries_raw).items(): if not isinstance(key, str): - problems.append(ValidationProblem(("metadata",), f"non-string key {key!r}")) + problems.append( + ValidationProblem(("metadata",), f"non-string key {key!r}", "invalid_type") + ) continue entry_obj: object = entry node_type: object = None @@ -214,14 +235,18 @@ def from_json(cls, data: object) -> ConsolidatedMetadataModelV3: entries[key] = GroupMetadataModelV3.from_json(entry_obj) else: problems.append( - ValidationProblem(("metadata", key, "node_type"), "expected 'array' or 'group'") + ValidationProblem( + ("metadata", key, "node_type"), + "expected 'array' or 'group'", + "invalid_value", + ) ) if problems: raise MetadataValidationError(problems) must_understand = doc.get("must_understand", False) if must_understand is not False: raise MetadataValidationError( - [ValidationProblem(("must_understand",), "expected False")] + [ValidationProblem(("must_understand",), "expected False", "invalid_value")] ) return cls(must_understand=must_understand, metadata=entries) @@ -280,6 +305,12 @@ def update(self, **kwargs: Unpack[GroupMetadataModelV2Partial]) -> GroupMetadata return dataclasses.replace(self, **kwargs) def to_json(self) -> GroupMetadataV2: + """Return the merged in-memory document form, INCLUDING `attributes`. + + This is not the on-disk `.zgroup` content: a conforming `.zgroup` must + exclude `attributes` (they live in the sibling `.zattrs` file). Use + `to_key_value` to produce the spec-conforming split for storage. + """ out: GroupMetadataV2 = {"zarr_format": self.zarr_format} if len(self.attributes) > 0: out["attributes"] = self.attributes @@ -292,9 +323,9 @@ def from_json(cls, data: object) -> GroupMetadataModelV2: @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV2: - zgroup = json.loads(mapping[GROUP_METADATA_STORE_KEY_V2]) + zgroup = load_store_json(mapping, GROUP_METADATA_STORE_KEY_V2) zattrs: dict[str, JSONValue] = ( - json.loads(mapping[ATTRIBUTES_STORE_KEY_V2]) + load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) if ATTRIBUTES_STORE_KEY_V2 in mapping else {} ) @@ -333,10 +364,12 @@ def to_json(self) -> dict[str, JSONValue]: @classmethod def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: if not isinstance(data, Mapping): - raise MetadataValidationError([ValidationProblem((), "expected a mapping")]) + raise MetadataValidationError( + [ValidationProblem((), "expected a mapping", "invalid_type")] + ) doc = cast("Mapping[str, object]", data) problems: list[ValidationProblem] = [ - ValidationProblem((key,), "missing required key") + ValidationProblem((key,), "missing required key", "missing_key") for key in ("zarr_consolidated_format", "metadata") if key not in doc ] @@ -346,7 +379,9 @@ def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: isinstance(k, str) for k in cast("Mapping[object, object]", entries) ): problems.append( - ValidationProblem(("metadata",), "expected a mapping with string keys") + ValidationProblem( + ("metadata",), "expected a mapping with string keys", "invalid_type" + ) ) if problems: raise MetadataValidationError(problems) @@ -358,7 +393,7 @@ def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> ConsolidatedMetadataModelV2: - return cls.from_json(json.loads(mapping[CONSOLIDATED_METADATA_STORE_KEY_V2])) + return cls.from_json(load_store_json(mapping, CONSOLIDATED_METADATA_STORE_KEY_V2)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: return { diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index a38258316b..304a80d4ab 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -1,16 +1,22 @@ """Structural validation for Zarr metadata documents. -Validators check JSON structure (shapes, key presence, primitive kinds), -not domain validity. Each concept gets a `validate_*` function returning -every problem found, an `is_*` type guard, and a `parse_*` function that -narrows or raises `MetadataValidationError`. +Validators check JSON structure (key presence, value shapes, and fixed +literals like `zarr_format`), not domain validity. Each concept gets a +`validate_*` function returning every problem found, an `is_*` type guard, +and a `parse_*` function that narrows or raises `MetadataValidationError`. + +Every `ValidationProblem` carries a machine-readable `kind` alongside its +human-readable `message`, so consumers can dispatch on the failure mode +(`missing_key`, `invalid_type`, `invalid_value`, `invalid_json`) without +string-matching messages. """ from __future__ import annotations +import json from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import Final, cast +from typing import Any, Final, Literal, cast from typing_extensions import TypeIs @@ -21,6 +27,17 @@ from zarr_metadata.v3.array import ArrayMetadataV3 from zarr_metadata.v3.group import GroupMetadataV3 +ProblemKind = Literal["missing_key", "invalid_type", "invalid_value", "invalid_json"] +"""Machine-readable classification of a `ValidationProblem`. + +- `missing_key`: a required key (document key or store key) is absent. +- `invalid_type`: a value has the wrong structural type (e.g. a string where + a mapping is required, a non-JSON-serializable object). +- `invalid_value`: a value has an acceptable type but an invalid content + (e.g. `zarr_format: 2` in a v3 document, `order: "Q"`). +- `invalid_json`: bytes that do not decode as JSON. +""" + @dataclass(frozen=True, slots=True) class ValidationProblem: @@ -28,10 +45,13 @@ class ValidationProblem: `loc` is the path from the document root to the offending value, e.g. `("codecs", 0, "name")`. An empty `loc` refers to the document as a whole. + `kind` classifies the failure mode for programmatic dispatch; `message` + is the human-readable description. """ loc: tuple[str | int, ...] message: str + kind: ProblemKind def __str__(self) -> str: location = ".".join(str(part) for part in self.loc) if self.loc else "" @@ -51,7 +71,7 @@ def __init__(self, problems: list[ValidationProblem]) -> None: def _prefix(loc_head: str | int, problems: list[ValidationProblem]) -> list[ValidationProblem]: """Prepend `loc_head` to the `loc` of every problem (for nested validators).""" - return [ValidationProblem((loc_head, *p.loc), p.message) for p in problems] + return [ValidationProblem((loc_head, *p.loc), p.message, p.kind) for p in problems] def validate_json(value: object) -> list[ValidationProblem]: @@ -62,7 +82,9 @@ def validate_json(value: object) -> list[ValidationProblem]: if isinstance(value, Mapping): for key, item in cast("Mapping[object, object]", value).items(): if not isinstance(key, str): - problems.append(ValidationProblem((), f"non-string key {key!r} in JSON object")) + problems.append( + ValidationProblem((), f"non-string key {key!r} in JSON object", "invalid_type") + ) continue problems.extend(_prefix(key, validate_json(item))) return problems @@ -70,7 +92,7 @@ def validate_json(value: object) -> list[ValidationProblem]: for index, item in enumerate(cast("Sequence[object]", value)): problems.extend(_prefix(index, validate_json(item))) return problems - return [ValidationProblem((), f"not a JSON-serializable value: {value!r}")] + return [ValidationProblem((), f"not a JSON-serializable value: {value!r}", "invalid_type")] def is_json(value: object) -> TypeIs[JSONValue]: @@ -120,6 +142,25 @@ def parse_json(value: object) -> JSONValue: ) +def _missing_keys(required: frozenset[str], doc: Mapping[str, object]) -> list[ValidationProblem]: + """One `missing_key` problem per required key absent from `doc`.""" + return [ + ValidationProblem((key,), "missing required key", "missing_key") + for key in sorted(required - doc.keys()) + ] + + +def _check_literal( + doc: Mapping[str, object], key: str, expected: object +) -> list[ValidationProblem]: + """One `invalid_value` problem if `doc[key]` is present but not `expected`.""" + if key in doc and doc[key] != expected: + return [ + ValidationProblem((key,), f"expected {expected!r}, got {doc[key]!r}", "invalid_value") + ] + return [] + + def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: """Return every reason `value` is not a v3 metadata field. @@ -129,18 +170,26 @@ def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: return [] if not isinstance(value, Mapping): return [ - ValidationProblem((), "expected a metadata field (string or {name, configuration})") + ValidationProblem( + (), + "expected a metadata field (string or {name, configuration})", + "invalid_type", + ) ] field = cast("Mapping[object, object]", value) problems: list[ValidationProblem] = [] if not isinstance(field.get("name"), str): - problems.append(ValidationProblem(("name",), "expected a string name")) + problems.append(ValidationProblem(("name",), "expected a string name", "invalid_type")) if "configuration" in field: configuration = field["configuration"] if not isinstance(configuration, Mapping): - problems.append(ValidationProblem(("configuration",), "expected a mapping")) + problems.append( + ValidationProblem(("configuration",), "expected a mapping", "invalid_type") + ) elif not all(isinstance(k, str) for k in cast("Mapping[object, object]", configuration)): - problems.append(ValidationProblem(("configuration",), "expected string keys")) + problems.append( + ValidationProblem(("configuration",), "expected string keys", "invalid_type") + ) return problems @@ -166,6 +215,40 @@ def _is_int_sequence(value: object) -> bool: ) +def _is_dtype_v2(value: object) -> bool: + """Whether `value` is shaped like a v2 dtype: a string or field records. + + A field record is a `(name, dtype)` or `(name, dtype, shape)` sequence, + where `dtype` is itself a string or nested field records and `shape` is a + sequence of int. The string content is NOT interpreted — whether the + string names a real dtype is domain validity, not structure. + """ + if isinstance(value, str): + return True + if not isinstance(value, Sequence): + return False + for record in cast("Sequence[object]", value): + if isinstance(record, str) or not isinstance(record, Sequence): + return False + fields = cast("Sequence[object]", record) + if len(fields) not in (2, 3): + return False + if not isinstance(fields[0], str): + return False + if not _is_dtype_v2(fields[1]): + return False + if len(fields) == 3 and not _is_int_sequence(fields[2]): + return False + return True + + +def _is_codec_v2(value: object) -> bool: + """Whether `value` is shaped like a v2 codec config: a mapping with a string `id`.""" + return isinstance(value, Mapping) and isinstance( + cast("Mapping[object, object]", value).get("id"), str + ) + + def _validate_attributes(value: object) -> list[ValidationProblem]: """Validate an `attributes` value: a mapping with string keys. @@ -178,7 +261,11 @@ def _validate_attributes(value: object) -> list[ValidationProblem]: if not isinstance(value, Mapping) or not all( isinstance(k, str) for k in cast("Mapping[object, object]", value) ): - return [ValidationProblem(("attributes",), "expected a mapping with string keys")] + return [ + ValidationProblem( + ("attributes",), "expected a mapping with string keys", "invalid_type" + ) + ] return [] @@ -189,14 +276,13 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: (they map to `extra_fields`). """ if not isinstance(value, Mapping): - return [ValidationProblem((), "expected a mapping")] + return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) - problems: list[ValidationProblem] = [ - ValidationProblem((key,), "missing required key") - for key in sorted(ARRAY_METADATA_REQUIRED_KEYS_V3 - doc.keys()) - ] + problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V3, doc) + problems.extend(_check_literal(doc, "zarr_format", 3)) + problems.extend(_check_literal(doc, "node_type", "array")) if "shape" in doc and not _is_int_sequence(doc["shape"]): - problems.append(ValidationProblem(("shape",), "expected a sequence of int")) + problems.append(ValidationProblem(("shape",), "expected a sequence of int", "invalid_type")) if "fill_value" in doc: problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) for key in ("data_type", "chunk_grid", "chunk_key_encoding"): @@ -206,7 +292,7 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: if key in doc: entries = doc[key] if isinstance(entries, str) or not isinstance(entries, Sequence): - problems.append(ValidationProblem((key,), "expected a sequence")) + problems.append(ValidationProblem((key,), "expected a sequence", "invalid_type")) else: for index, entry in enumerate(cast("Sequence[object]", entries)): problems.extend(_prefix(key, _prefix(index, validate_metadata_field_v3(entry)))) @@ -218,12 +304,16 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: # the metadata-field lists (codecs, storage_transformers). names = doc["dimension_names"] if isinstance(names, str) or not isinstance(names, Sequence): - problems.append(ValidationProblem(("dimension_names",), "expected a sequence")) + problems.append( + ValidationProblem(("dimension_names",), "expected a sequence", "invalid_type") + ) elif not all( item is None or isinstance(item, str) for item in cast("Sequence[object]", names) ): problems.append( - ValidationProblem(("dimension_names",), "expected items of str or None") + ValidationProblem( + ("dimension_names",), "expected items of str or None", "invalid_type" + ) ) return problems @@ -244,21 +334,67 @@ def parse_array_metadata_v3(value: object) -> ArrayMetadataV3: def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: """Return every reason `value` is not a structurally-valid v2 array doc. - Checks structure, not domain validity. `compressor`/`filters` are required - keys but may be `None`. + Checks structure, not domain validity: `dtype` must be a string or field + records, but the string content is not interpreted; `compressor` and + `filters` are required keys that may be `None`, and otherwise must be + codec configurations (mappings with a string `id`). """ if not isinstance(value, Mapping): - return [ValidationProblem((), "expected a mapping")] + return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) - problems: list[ValidationProblem] = [ - ValidationProblem((key,), "missing required key") - for key in sorted(ARRAY_METADATA_REQUIRED_KEYS_V2 - doc.keys()) - ] + problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V2, doc) + problems.extend(_check_literal(doc, "zarr_format", 2)) problems.extend( - ValidationProblem((key,), "expected a sequence of int") + ValidationProblem((key,), "expected a sequence of int", "invalid_type") for key in ("shape", "chunks") if key in doc and not _is_int_sequence(doc[key]) ) + if "dtype" in doc and not _is_dtype_v2(doc["dtype"]): + problems.append( + ValidationProblem( + ("dtype",), + "expected a v2 dtype string or a sequence of field records", + "invalid_type", + ) + ) + if "order" in doc and doc["order"] not in ("C", "F"): + problems.append( + ValidationProblem( + ("order",), f"expected 'C' or 'F', got {doc['order']!r}", "invalid_value" + ) + ) + if "compressor" in doc: + compressor = doc["compressor"] + if compressor is not None and not _is_codec_v2(compressor): + problems.append( + ValidationProblem( + ("compressor",), + "expected null or a codec configuration with a string 'id'", + "invalid_type", + ) + ) + if "filters" in doc: + filters = doc["filters"] + if filters is not None and ( + isinstance(filters, str) + or not isinstance(filters, Sequence) + or not all(_is_codec_v2(item) for item in cast("Sequence[object]", filters)) + ): + problems.append( + ValidationProblem( + ("filters",), + "expected null or a sequence of codec configurations with string 'id's", + "invalid_type", + ) + ) + if "dimension_separator" in doc and doc["dimension_separator"] not in (".", "/"): + problems.append( + ValidationProblem( + ("dimension_separator",), + f"expected '.' or '/', got {doc['dimension_separator']!r}", + "invalid_value", + ) + ) if "fill_value" in doc: problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) if "attributes" in doc: @@ -287,16 +423,17 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: must be a mapping (its entries are validated by the consolidated model). """ if not isinstance(value, Mapping): - return [ValidationProblem((), "expected a mapping")] + return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) - problems: list[ValidationProblem] = [ - ValidationProblem((key,), "missing required key") - for key in sorted(GROUP_METADATA_REQUIRED_KEYS_V3 - doc.keys()) - ] + problems: list[ValidationProblem] = _missing_keys(GROUP_METADATA_REQUIRED_KEYS_V3, doc) + problems.extend(_check_literal(doc, "zarr_format", 3)) + problems.extend(_check_literal(doc, "node_type", "group")) if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) if "consolidated_metadata" in doc and not isinstance(doc["consolidated_metadata"], Mapping): - problems.append(ValidationProblem(("consolidated_metadata",), "expected a mapping")) + problems.append( + ValidationProblem(("consolidated_metadata",), "expected a mapping", "invalid_type") + ) return problems @@ -320,12 +457,10 @@ def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: optional `attributes` mapping folded in from `.zattrs`. """ if not isinstance(value, Mapping): - return [ValidationProblem((), "expected a mapping")] + return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) - problems: list[ValidationProblem] = [ - ValidationProblem((key,), "missing required key") - for key in sorted(GROUP_METADATA_REQUIRED_KEYS_V2 - doc.keys()) - ] + problems: list[ValidationProblem] = _missing_keys(GROUP_METADATA_REQUIRED_KEYS_V2, doc) + problems.extend(_check_literal(doc, "zarr_format", 2)) if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) return problems @@ -344,6 +479,26 @@ def parse_group_metadata_v2(value: object) -> GroupMetadataV2: return cast(GroupMetadataV2, value) +def load_store_json(mapping: Mapping[str, bytes], key: str) -> Any: + """Decode the JSON document stored at `key` in `mapping`. + + Every ingestion failure surfaces as `MetadataValidationError`: a missing + store key is a `missing_key` problem and undecodable bytes are an + `invalid_json` problem, rather than leaking `KeyError` / + `json.JSONDecodeError` to callers. + """ + if key not in mapping: + raise MetadataValidationError( + [ValidationProblem((key,), "missing store key", "missing_key")] + ) + try: + return json.loads(mapping[key]) + except json.JSONDecodeError as exc: + raise MetadataValidationError( + [ValidationProblem((key,), f"invalid JSON: {exc}", "invalid_json")] + ) from exc + + def arrays_to_tuples(obj: object) -> object: """Recursively convert every list in a JSON-decoded structure to a tuple.""" if isinstance(obj, list): diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 80dac709de..904e0b12fa 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -17,8 +17,8 @@ ArrayMetadataModelV3, ArrayMetadataModelV3Partial, MetadataValidationError, + NamedConfigModelV3, ValidationProblem, - ZarrMetadataV3, is_array_metadata_v2, is_array_metadata_v3, is_json, @@ -119,16 +119,16 @@ def test_string_nan_fill_value_roundtrips() -> None: assert ArrayMetadataModelV3.from_json(m.to_json()).fill_value == "NaN" -# --- ZarrMetadataV3.to_json ------------------------------------------------ +# --- NamedConfigModelV3.to_json ------------------------------------------------ ZARR_TO_JSON_CASES = [ Expect( - ZarrMetadataV3(name="regular", configuration={"chunk_shape": [1]}), + NamedConfigModelV3(name="regular", configuration={"chunk_shape": [1]}), {"name": "regular", "configuration": {"chunk_shape": [1]}}, id="with-configuration", ), Expect( - ZarrMetadataV3(name="bytes", configuration={}), + NamedConfigModelV3(name="bytes", configuration={}), {"name": "bytes", "configuration": {}}, id="without-configuration", ), @@ -136,32 +136,32 @@ def test_string_nan_fill_value_roundtrips() -> None: @pytest.mark.parametrize("case", ZARR_TO_JSON_CASES, ids=lambda c: c.id) -def test_zarr_metadata_v3_to_json(case: Expect[ZarrMetadataV3, dict[str, object]]) -> None: - """ZarrMetadataV3.to_json emits the canonical object form.""" +def test_zarr_metadata_v3_to_json(case: Expect[NamedConfigModelV3, dict[str, object]]) -> None: + """NamedConfigModelV3.to_json emits the canonical object form.""" assert case.input.to_json() == case.output -# --- ZarrMetadataV3.from_json ----------------------------------------------- +# --- NamedConfigModelV3.from_json ----------------------------------------------- ZARR_FROM_JSON_CASES = [ - Expect("bytes", ZarrMetadataV3(name="bytes", configuration={}), id="bare-string"), + Expect("bytes", NamedConfigModelV3(name="bytes", configuration={}), id="bare-string"), Expect( {"name": "regular", "configuration": {"chunk_shape": [1]}}, - ZarrMetadataV3(name="regular", configuration={"chunk_shape": (1,)}), + NamedConfigModelV3(name="regular", configuration={"chunk_shape": (1,)}), id="object-with-config", ), Expect( {"name": "bytes"}, - ZarrMetadataV3(name="bytes", configuration={}), + NamedConfigModelV3(name="bytes", configuration={}), id="object-without-config", ), ] @pytest.mark.parametrize("case", ZARR_FROM_JSON_CASES, ids=lambda c: c.id) -def test_zarr_metadata_v3_from_json(case: Expect[object, ZarrMetadataV3]) -> None: - """ZarrMetadataV3.from_json parses both the bare-string and object forms.""" - assert ZarrMetadataV3.from_json(case.input) == case.output +def test_zarr_metadata_v3_from_json(case: Expect[object, NamedConfigModelV3]) -> None: + """NamedConfigModelV3.from_json parses both the bare-string and object forms.""" + assert NamedConfigModelV3.from_json(case.input) == case.output # --- V3 baseline ----------------------------------------------------------- @@ -170,7 +170,7 @@ def test_zarr_metadata_v3_from_json(case: Expect[object, ZarrMetadataV3]) -> Non def test_v3_to_json_includes_required_fields() -> None: """V3 to_json emits all required fields with the expected values.""" out = ArrayMetadataModelV3.create_default( - shape=(10,), data_type=ZarrMetadataV3(name="int32", configuration={}) + shape=(10,), data_type=NamedConfigModelV3(name="int32", configuration={}) ).to_json() assert out["zarr_format"] == 3 assert out["node_type"] == "array" @@ -221,7 +221,7 @@ def test_v3_single_storage_transformer_included() -> None: Regression: the guard used ``> 1`` instead of ``> 0``, dropping a lone storage transformer. """ - st = ZarrMetadataV3(name="some_transformer", configuration={}) + st = NamedConfigModelV3(name="some_transformer", configuration={}) out: dict[str, object] = dict( ArrayMetadataModelV3.create_default(storage_transformers=(st,)).to_json() ) @@ -292,7 +292,7 @@ def test_v3_create_default_is_valid_empty_array() -> None: """V3 create_default builds a structurally valid empty array that round-trips.""" m = ArrayMetadataModelV3.create_default() assert m.shape == () - assert m.data_type == ZarrMetadataV3(name="uint8", configuration={}) + assert m.data_type == NamedConfigModelV3(name="uint8", configuration={}) assert m.fill_value == 0 assert m.attributes == {} assert m.extra_fields == {} @@ -307,7 +307,7 @@ def test_v3_create_default_applies_overrides() -> None: assert m.shape == (4, 4) assert m.attributes == {"a": 1} # un-overridden fields keep their defaults - assert m.data_type == ZarrMetadataV3(name="uint8", configuration={}) + assert m.data_type == NamedConfigModelV3(name="uint8", configuration={}) def test_v2_create_default_is_valid_empty_array() -> None: @@ -464,11 +464,11 @@ def test_v3_from_json_reconstructs_required_fields() -> None: doc = ArrayMetadataModelV3.create_default( shape=(7,), attributes={"a": 1}, - data_type=ZarrMetadataV3(name="int32", configuration={}), + data_type=NamedConfigModelV3(name="int32", configuration={}), ).to_json() model = ArrayMetadataModelV3.from_json(doc) assert model.shape == (7,) - assert model.data_type == ZarrMetadataV3(name="int32", configuration={}) + assert model.data_type == NamedConfigModelV3(name="int32", configuration={}) assert model.attributes == {"a": 1} @@ -522,12 +522,12 @@ def test_v3_from_key_value_parses_zarr_json() -> None: FROM_KEY_VALUE_MISSING_PARAMS = [ pytest.param( ArrayMetadataModelV3, - ExpectFail({}, KeyError, id="v3-missing-zarr-json"), + ExpectFail({}, MetadataValidationError, id="v3-missing-zarr-json", msg="missing store key"), id="v3-missing-zarr-json", ), pytest.param( ArrayMetadataModelV2, - ExpectFail({}, KeyError, id="v2-missing-zarray"), + ExpectFail({}, MetadataValidationError, id="v2-missing-zarray", msg="missing store key"), id="v2-missing-zarray", ), ] @@ -538,7 +538,7 @@ def test_from_key_value_missing_key_raises( model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], case: ExpectFail[dict[str, bytes]], ) -> None: - """from_key_value raises KeyError when the required store key is absent.""" + """from_key_value raises MetadataValidationError when the required store key is absent.""" with case.raises(): model_cls.from_key_value(case.input) @@ -551,7 +551,7 @@ def test_from_key_value_missing_key_raises( ArrayMetadataModelV3.create_default( attributes={"a": 1}, dimension_names=("x",), - storage_transformers=(ZarrMetadataV3(name="t", configuration={}),), + storage_transformers=(NamedConfigModelV3(name="t", configuration={}),), extra_fields={"ext": {"must_understand": False}}, ), id="v3-full", @@ -629,7 +629,7 @@ def test_v3_parser_accepts_bare_string_data_type() -> None: doc["data_type"] = "int32" # bare-string form, not canonical object form model = ArrayMetadataModelV3.from_json(doc) # parses correctly, re-serializes to canonical object form - assert model.data_type == ZarrMetadataV3(name="int32", configuration={}) + assert model.data_type == NamedConfigModelV3(name="int32", configuration={}) assert model.to_json()["data_type"] == {"name": "int32", "configuration": {}} @@ -960,7 +960,7 @@ def test_array_metadata_guards( id="v2-missing-required", ), pytest.param( - ZarrMetadataV3, + NamedConfigModelV3, ExpectFail(lambda: 5, MetadataValidationError, id="x"), id="zarr-metadata-bad-input", ), @@ -969,7 +969,7 @@ def test_array_metadata_guards( @pytest.mark.parametrize(("model", "case"), FROM_JSON_REJECT_PARAMS) def test_from_json_rejects_malformed( - model: type[ArrayMetadataModelV3 | ArrayMetadataModelV2 | ZarrMetadataV3], + model: type[ArrayMetadataModelV3 | ArrayMetadataModelV2 | NamedConfigModelV3], case: ExpectFail[Callable[[], object]], ) -> None: """from_json raises MetadataValidationError on a malformed document.""" @@ -983,19 +983,19 @@ def test_from_json_rejects_malformed( def test_validation_problem_str_with_loc() -> None: """ValidationProblem.__str__ renders a non-empty loc as a dotted path.""" - p = ValidationProblem(loc=("codecs", 0, "name"), message="expected str") + p = ValidationProblem(loc=("codecs", 0, "name"), message="expected str", kind="invalid_type") assert str(p) == "codecs.0.name: expected str" def test_validation_problem_str_empty_loc() -> None: """ValidationProblem.__str__ renders an empty loc as .""" - p = ValidationProblem(loc=(), message="not a mapping") + p = ValidationProblem(loc=(), message="not a mapping", kind="invalid_type") assert str(p) == ": not a mapping" def test_validation_problem_is_frozen() -> None: """ValidationProblem is immutable (frozen dataclass).""" - p = ValidationProblem(loc=("shape",), message="x") + p = ValidationProblem(loc=("shape",), message="x", kind="invalid_type") with pytest.raises(dataclasses.FrozenInstanceError): p.message = "y" # type: ignore[misc] @@ -1003,8 +1003,10 @@ def test_validation_problem_is_frozen() -> None: def test_metadata_validation_error_holds_problems() -> None: """MetadataValidationError carries its problem list and renders them in its message.""" problems = [ - ValidationProblem(loc=("shape",), message="missing required key"), - ValidationProblem(loc=("data_type",), message="expected a metadata field"), + ValidationProblem(loc=("shape",), message="missing required key", kind="missing_key"), + ValidationProblem( + loc=("data_type",), message="expected a metadata field", kind="invalid_type" + ), ] err = MetadataValidationError(problems) assert err.problems == problems @@ -1014,6 +1016,125 @@ def test_metadata_validation_error_holds_problems() -> None: def test_prefix_prepends_loc_head() -> None: """_prefix prepends a loc head to each problem's loc.""" - problems = [ValidationProblem(loc=("name",), message="expected str")] + problems = [ValidationProblem(loc=("name",), message="expected str", kind="invalid_type")] prefixed = _prefix(0, problems) - assert prefixed == [ValidationProblem(loc=(0, "name"), message="expected str")] + assert prefixed == [ + ValidationProblem(loc=(0, "name"), message="expected str", kind="invalid_type") + ] + + +# --- Stricter v2/v3 field validation and error kinds ------------------------- + + +def test_v2_dtype_must_be_string_or_records() -> None: + """A non-string, non-records v2 dtype is rejected with an invalid_type problem.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dtype": 42} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dtype",), "invalid_type")] + + +def test_v2_structured_dtype_records_accepted() -> None: + """A structured v2 dtype (field records, optionally nested/shaped) validates.""" + dtype = (("a", " None: + """A field record with the wrong arity is rejected.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dtype": (("a",),)} + problems = validate_array_metadata_v2(doc) + assert [p.loc for p in problems] == [("dtype",)] + + +def test_v2_order_literal_enforced() -> None: + """An order other than 'C' or 'F' is rejected with an invalid_value problem.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"order": "Q"} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("order",), "invalid_value")] + + +def test_v2_compressor_must_be_codec_or_none() -> None: + """A compressor that is not null or a codec config mapping is rejected.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"compressor": "zlib"} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("compressor",), "invalid_type")] + + +def test_v2_compressor_requires_string_id() -> None: + """A compressor mapping without a string id is rejected.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"compressor": {"level": 3}} + problems = validate_array_metadata_v2(doc) + assert [p.loc for p in problems] == [("compressor",)] + + +def test_v2_filters_must_be_codec_sequence_or_none() -> None: + """Filters that are not null or a sequence of codec configs are rejected.""" + for bad in (7, (5,), "gzip"): + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"filters": bad} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("filters",), "invalid_type")], bad + + +def test_v2_dimension_separator_literal_enforced() -> None: + """A dimension_separator other than '.' or '/' is rejected.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dimension_separator": "-"} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] + + +def test_v2_zarr_format_literal_enforced() -> None: + """A v2 document claiming zarr_format 3 is rejected with an invalid_value problem.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"zarr_format": 3} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] + + +def test_v3_zarr_format_literal_enforced() -> None: + """A v3 document claiming zarr_format 2 is rejected with an invalid_value problem.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"zarr_format": 2} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] + + +def test_v3_node_type_literal_enforced() -> None: + """A v3 array document claiming node_type 'group' is rejected.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"node_type": "group"} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("node_type",), "invalid_value")] + + +def test_missing_key_kind_is_machine_readable() -> None: + """A missing required key is distinguishable by kind, without message matching.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) + del doc["chunk_key_encoding"] + problems = validate_array_metadata_v3(doc) + assert problems == [ + ValidationProblem(("chunk_key_encoding",), "missing required key", "missing_key") + ] + + +# --- Unified error channels --------------------------------------------------- + + +def test_from_key_value_invalid_json_raises_metadata_error() -> None: + """Undecodable store bytes raise MetadataValidationError (kind invalid_json), not JSONDecodeError.""" + with pytest.raises(MetadataValidationError) as exc_info: + ArrayMetadataModelV3.from_key_value({"zarr.json": b"{not json"}) + assert [p.kind for p in exc_info.value.problems] == ["invalid_json"] + + +def test_from_key_value_missing_key_kind() -> None: + """A missing store key surfaces as a missing_key problem at the store-key loc.""" + with pytest.raises(MetadataValidationError) as exc_info: + ArrayMetadataModelV2.from_key_value({}) + assert exc_info.value.problems == [ + ValidationProblem((".zarray",), "missing store key", "missing_key") + ] + + +def test_extra_fields_overlap_raises_metadata_error() -> None: + """The extra-fields overlap invariant raises MetadataValidationError (a ValueError).""" + with pytest.raises(MetadataValidationError, match="Extra fields") as exc_info: + ArrayMetadataModelV3.create_default(extra_fields={"shape": {"must_understand": False}}) + assert [p.kind for p in exc_info.value.problems] == ["invalid_value"] diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 909346586b..829d09b11a 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -18,6 +18,8 @@ MetadataValidationError, parse_group_metadata_v2, parse_group_metadata_v3, + validate_group_metadata_v2, + validate_group_metadata_v3, ) # --- GroupMetadataModelV3 --------------------------------------------------- @@ -265,3 +267,20 @@ def test_consolidated_v2_not_a_mapping() -> None: """from_json rejects a non-mapping .zmetadata document.""" with pytest.raises(MetadataValidationError, match="expected a mapping"): ConsolidatedMetadataModelV2.from_json([1]) + + +# --- Literal-value enforcement ----------------------------------------------- + + +def test_group_v3_literals_enforced() -> None: + """A v3 group doc with wrong zarr_format or node_type is rejected with invalid_value.""" + base = GroupMetadataModelV3.create_default().to_json() + for key, bad in (("zarr_format", 2), ("node_type", "array")): + problems = validate_group_metadata_v3(dict(base) | {key: bad}) + assert [(p.loc, p.kind) for p in problems] == [((key,), "invalid_value")], key + + +def test_group_v2_zarr_format_literal_enforced() -> None: + """A v2 group doc claiming zarr_format 3 is rejected with invalid_value.""" + problems = validate_group_metadata_v2({"zarr_format": 3}) + assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 7a6db7ab30..cd0196d47b 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -50,9 +50,10 @@ def _group_rank(s: str) -> int: "GroupMetadataModelV3Partial", "ConsolidatedMetadataModelV2", "ConsolidatedMetadataModelV3", - "ZarrMetadataV3", + "NamedConfigModelV3", "ValidationProblem", "MetadataValidationError", + "ProblemKind", # v2 data-type encoding union "DataTypeMetadataV2", # Category B — codec canonical unions From be736dabd317fda8da16075c697450393aa4c906 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 15:10:26 +0200 Subject: [PATCH 10/70] feat(zarr-metadata): annotate metadata fields with role alias MetadataFieldModelV3 Model fields and consumer signatures should convey the logical meaning of the type (a metadata-document field), not the form it takes when JSON-serialized (a named configuration). MetadataFieldModelV3 is today exactly NamedConfigModelV3; if a future spec revision adds a field form that cannot normalize to name + configuration, the alias widens to a union and annotation sites do not move. Mirrors the raw-layer split between NamedConfigV3 (shape) and MetadataV3 (field union). Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 5 ++- .../src/zarr_metadata/__init__.py | 2 + .../src/zarr_metadata/model/__init__.py | 2 + .../src/zarr_metadata/model/_array.py | 41 +++++++++++++------ .../zarr-metadata/tests/model/test_array.py | 13 ++++++ .../zarr-metadata/tests/test_public_api.py | 1 + 6 files changed, 50 insertions(+), 14 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 13517ce38a..326aee1b42 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -4,7 +4,10 @@ Added `zarr_metadata.model`: frozen-dataclass models (`ArrayMetadataModelV2`, that are canonical, lossless representations of Zarr metadata documents, plus structural validators (`validate_*` / `is_*` / `parse_*`). Every v3 extension point (data type, chunk grid, chunk key encoding, codecs, storage transformers) -is held as a name + configuration pair; nothing is interpreted. +is held as a name + configuration pair; nothing is interpreted. Model fields +are annotated with the role alias `MetadataFieldModelV3` (today exactly +`NamedConfigModelV3`), so the annotations convey the logical meaning of the +field and stay put if the spec ever adds a new field form. Validation is strict about what the types declare: v2 `dtype` / `order` / `compressor` / `filters` / `dimension_separator` shapes and the fixed diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index ecd7e342cf..354f2a97f5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -12,6 +12,7 @@ GroupMetadataModelV2Partial, GroupMetadataModelV3, GroupMetadataModelV3Partial, + MetadataFieldModelV3, MetadataValidationError, NamedConfigModelV3, ProblemKind, @@ -316,6 +317,7 @@ "Int64DataTypeName", "Int64FillValue", "JSONValue", + "MetadataFieldModelV3", "MetadataV3", "MetadataValidationError", "NamedConfigModelV3", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index aa8555d623..42fbc233d8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -19,6 +19,7 @@ ArrayMetadataModelV2Partial, ArrayMetadataModelV3, ArrayMetadataModelV3Partial, + MetadataFieldModelV3, NamedConfigModelV3, ) from zarr_metadata.model._group import ( @@ -91,6 +92,7 @@ "GroupMetadataModelV2Partial", "GroupMetadataModelV3", "GroupMetadataModelV3Partial", + "MetadataFieldModelV3", "MetadataValidationError", "NamedConfigModelV3", "ProblemKind", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 4ec02137ec..33682c2008 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -5,7 +5,7 @@ import dataclasses import json from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Final, Literal +from typing import TYPE_CHECKING, Final, Literal, TypeAlias from typing_extensions import TypedDict, Unpack @@ -68,6 +68,20 @@ def from_json(cls, data: object) -> NamedConfigModelV3: return cls(name=field["name"], configuration=configuration) # type: ignore[arg-type] +MetadataFieldModelV3: TypeAlias = NamedConfigModelV3 +"""The in-memory model of one field of a v3 metadata document. + +This is the role-named alias for annotation positions: model fields and +consumer signatures should say `MetadataFieldModelV3` (the logical meaning) +rather than `NamedConfigModelV3` (the serialized form the field currently +takes). Today every metadata field normalizes to a named configuration, so +the alias is exactly `NamedConfigModelV3`; if a future spec revision adds a +field form that cannot be normalized to name + configuration, this alias +widens to a union and annotation sites do not change. Mirrors the raw-layer +split between `NamedConfigV3` (shape) and `MetadataV3` (field union). +""" + + class ArrayMetadataModelV3Partial(TypedDict, total=False): """ Partial form of the constructor-settable fields of `ArrayMetadataModelV3`. @@ -84,13 +98,13 @@ class ArrayMetadataModelV3Partial(TypedDict, total=False): shape: tuple[int, ...] fill_value: JSONValue - data_type: NamedConfigModelV3 - chunk_grid: NamedConfigModelV3 - codecs: tuple[NamedConfigModelV3, ...] - chunk_key_encoding: NamedConfigModelV3 + data_type: MetadataFieldModelV3 + chunk_grid: MetadataFieldModelV3 + codecs: tuple[MetadataFieldModelV3, ...] + chunk_key_encoding: MetadataFieldModelV3 dimension_names: tuple[str | None, ...] | None attributes: dict[str, JSONValue] - storage_transformers: tuple[NamedConfigModelV3, ...] + storage_transformers: tuple[MetadataFieldModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @@ -100,8 +114,9 @@ class ArrayMetadataModelV3: A canonical, lossless representation of the `zarr.json` content for an array. Extension points (`data_type`, `chunk_grid`, `chunk_key_encoding`, - `codecs`, `storage_transformers`) are held as `NamedConfigModelV3` name + - configuration pairs and are never interpreted; `fill_value` is held + `codecs`, `storage_transformers`) are held as `MetadataFieldModelV3` + values (currently always `NamedConfigModelV3` name + configuration pairs) + and are never interpreted; `fill_value` is held verbatim in its JSON form. """ @@ -109,13 +124,13 @@ class ArrayMetadataModelV3: node_type: Literal["array"] = field(default="array", init=False) shape: tuple[int, ...] fill_value: JSONValue - data_type: NamedConfigModelV3 - chunk_grid: NamedConfigModelV3 - codecs: tuple[NamedConfigModelV3, ...] - chunk_key_encoding: NamedConfigModelV3 + data_type: MetadataFieldModelV3 + chunk_grid: MetadataFieldModelV3 + codecs: tuple[MetadataFieldModelV3, ...] + chunk_key_encoding: MetadataFieldModelV3 dimension_names: tuple[str | None, ...] | None attributes: dict[str, JSONValue] - storage_transformers: tuple[NamedConfigModelV3, ...] + storage_transformers: tuple[MetadataFieldModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @classmethod diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 904e0b12fa..e825f0211f 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -16,6 +16,7 @@ ArrayMetadataModelV2Partial, ArrayMetadataModelV3, ArrayMetadataModelV3Partial, + MetadataFieldModelV3, MetadataValidationError, NamedConfigModelV3, ValidationProblem, @@ -1138,3 +1139,15 @@ def test_extra_fields_overlap_raises_metadata_error() -> None: with pytest.raises(MetadataValidationError, match="Extra fields") as exc_info: ArrayMetadataModelV3.create_default(extra_fields={"shape": {"must_understand": False}}) assert [p.kind for p in exc_info.value.problems] == ["invalid_value"] + + +def test_extension_point_fields_annotated_with_role_alias() -> None: + """Extension-point fields are annotated with MetadataFieldModelV3 (the + logical role), not NamedConfigModelV3 (the current serialized form), so a + future widening of the field union does not move annotation sites.""" + assert MetadataFieldModelV3 is NamedConfigModelV3 + annotations = ArrayMetadataModelV3.__annotations__ + for field_name in ("data_type", "chunk_grid", "chunk_key_encoding"): + assert annotations[field_name] == "MetadataFieldModelV3" + for field_name in ("codecs", "storage_transformers"): + assert annotations[field_name] == "tuple[MetadataFieldModelV3, ...]" diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index cd0196d47b..fb2f574a4f 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -51,6 +51,7 @@ def _group_rank(s: str) -> int: "ConsolidatedMetadataModelV2", "ConsolidatedMetadataModelV3", "NamedConfigModelV3", + "MetadataFieldModelV3", "ValidationProblem", "MetadataValidationError", "ProblemKind", From 7fc0211e4ca329ddbab3b4b3ee718fc173719072 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 15:15:18 +0200 Subject: [PATCH 11/70] test(zarr-metadata): assert required-key coverage via the typed constant test_v3_to_json_includes_required_fields hand-enumerated keys with chained asserts, restating what ARRAY_METADATA_REQUIRED_KEYS_V3 already defines. Now: one coverage assert driven by the constant (tracks the TypedDict automatically) and one whole-document equality for the values. Assisted-by: ClaudeCode:claude-fable-5 --- .../zarr-metadata/tests/model/test_array.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index e825f0211f..8806eb8490 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -169,16 +169,22 @@ def test_zarr_metadata_v3_from_json(case: Expect[object, NamedConfigModelV3]) -> def test_v3_to_json_includes_required_fields() -> None: - """V3 to_json emits all required fields with the expected values.""" + """V3 to_json emits every spec-required key (per the typed constant), + and the whole document matches the model's values.""" out = ArrayMetadataModelV3.create_default( shape=(10,), data_type=NamedConfigModelV3(name="int32", configuration={}) ).to_json() - assert out["zarr_format"] == 3 - assert out["node_type"] == "array" - assert out["shape"] == (10,) - assert out["fill_value"] == 0 - assert out["data_type"] == {"name": "int32", "configuration": {}} - assert out["codecs"] == ({"name": "bytes", "configuration": {}},) + assert out.keys() >= ARRAY_METADATA_REQUIRED_KEYS_V3 + assert out == { + "zarr_format": 3, + "node_type": "array", + "shape": (10,), + "fill_value": 0, + "data_type": {"name": "int32", "configuration": {}}, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": ()}}, + "codecs": ({"name": "bytes", "configuration": {}},), + "chunk_key_encoding": {"name": "default", "configuration": {}}, + } def test_v3_dimension_names_included_when_present() -> None: From 15e02189def66f60923209aac44368c60535349b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 15:16:12 +0200 Subject: [PATCH 12/70] test(zarr-metadata): single whole-document comparison for v3 to_json The subset assert against ARRAY_METADATA_REQUIRED_KEYS_V3 was redundant: equality with a literal that spells out the full document already covers every required key. One dict, one assert. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/tests/model/test_array.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 8806eb8490..0df935144b 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -168,13 +168,12 @@ def test_zarr_metadata_v3_from_json(case: Expect[object, NamedConfigModelV3]) -> # --- V3 baseline ----------------------------------------------------------- -def test_v3_to_json_includes_required_fields() -> None: - """V3 to_json emits every spec-required key (per the typed constant), - and the whole document matches the model's values.""" +def test_v3_to_json_emits_canonical_document() -> None: + """V3 to_json emits exactly the expected document (which covers every + spec-required key by construction).""" out = ArrayMetadataModelV3.create_default( shape=(10,), data_type=NamedConfigModelV3(name="int32", configuration={}) ).to_json() - assert out.keys() >= ARRAY_METADATA_REQUIRED_KEYS_V3 assert out == { "zarr_format": 3, "node_type": "array", From c0a40cf6bd1c3e9d2b355a86279ddb25c0db557b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 16:52:10 +0200 Subject: [PATCH 13/70] fix(zarr-metadata): close validation holes found by adversarial review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Invalid documents that previously passed validation: - shape/chunks containing JSON booleans (bool is an int subclass in Python but not an integer in a metadata document) or negative values - dimension_names whose length does not match shape - attributes and configuration values that are not JSON-serializable — now checked recursively like fill_value, so an int-keyed dict cannot be silently rewritten by json.dumps on round-trip and a set() cannot escape as a TypeError from to_key_value - consolidated_metadata envelopes: the group validator now deep-validates the envelope and its entries via the shared validate_consolidated_metadata_v3, which ConsolidatedMetadataModelV3 .from_json also uses, so is_group_metadata_v3 never vouches for a document the model constructor would reject Three pre-existing test fixtures paired dimension_names=('x',) with the default scalar shape () and were themselves spec-invalid; they now use a matching 1-d shape. Deliberately unchanged, pending a design decision: unknown extension fields with must_understand: true still pass (which layer owns the spec's refusal duty), and empty v2 dtype records / empty codec names still pass (domain territory). Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 8 +- .../src/zarr_metadata/model/_group.py | 49 ++------ .../src/zarr_metadata/model/_validation.py | 119 +++++++++++++++--- .../zarr-metadata/tests/model/test_array.py | 65 +++++++++- .../zarr-metadata/tests/model/test_group.py | 51 ++++++++ 5 files changed, 235 insertions(+), 57 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 326aee1b42..fbecb7b4a4 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -16,4 +16,10 @@ Validation is strict about what the types declare: v2 `dtype` / `order` / (`missing_key` / `invalid_type` / `invalid_value` / `invalid_json`) so consumers can dispatch on the failure mode without matching message strings, and every ingestion failure — including missing store keys and undecodable -bytes in `from_key_value` — surfaces as `MetadataValidationError`. +bytes in `from_key_value` — surfaces as `MetadataValidationError`. An +adversarial review added further structural checks: JSON booleans are not +accepted as dimension lengths, dimensions are non-negative, +`dimension_names` must have one entry per dimension of `shape`, `attributes` +and `configuration` values are JSON-checked recursively (like `fill_value`), +and the inline consolidated-metadata envelope and entries are deep-validated +so the group validator's verdict always agrees with the model constructor. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 6128b0306b..b138a7d0f5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -22,6 +22,7 @@ load_store_json, parse_group_metadata_v2, parse_group_metadata_v3, + validate_consolidated_metadata_v3, ) if TYPE_CHECKING: @@ -207,48 +208,18 @@ def to_json(self) -> ConsolidatedMetadataV3: @classmethod def from_json(cls, data: object) -> ConsolidatedMetadataModelV3: - if not isinstance(data, Mapping): - raise MetadataValidationError( - [ValidationProblem((), "expected a mapping", "invalid_type")] - ) - doc = cast("Mapping[str, object]", data) - entries_raw = doc.get("metadata") - if not isinstance(entries_raw, Mapping): - raise MetadataValidationError( - [ValidationProblem(("metadata",), "expected a mapping", "invalid_type")] - ) + problems = validate_consolidated_metadata_v3(data) + if problems: + raise MetadataValidationError(problems) + env = cast("Mapping[str, object]", data) entries: dict[str, ArrayMetadataModelV3 | GroupMetadataModelV3] = {} - problems: list[ValidationProblem] = [] - for key, entry in cast("Mapping[object, object]", entries_raw).items(): - if not isinstance(key, str): - problems.append( - ValidationProblem(("metadata",), f"non-string key {key!r}", "invalid_type") - ) - continue - entry_obj: object = entry - node_type: object = None - if isinstance(entry, Mapping): - node_type = cast("Mapping[str, object]", entry).get("node_type") + for key, entry in cast("Mapping[str, object]", env["metadata"]).items(): + node_type = cast("Mapping[str, object]", entry).get("node_type") if node_type == "array": - entries[key] = ArrayMetadataModelV3.from_json(entry_obj) - elif node_type == "group": - entries[key] = GroupMetadataModelV3.from_json(entry_obj) + entries[key] = ArrayMetadataModelV3.from_json(entry) else: - problems.append( - ValidationProblem( - ("metadata", key, "node_type"), - "expected 'array' or 'group'", - "invalid_value", - ) - ) - if problems: - raise MetadataValidationError(problems) - must_understand = doc.get("must_understand", False) - if must_understand is not False: - raise MetadataValidationError( - [ValidationProblem(("must_understand",), "expected False", "invalid_value")] - ) - return cls(must_understand=must_understand, metadata=entries) + entries[key] = GroupMetadataModelV3.from_json(entry) + return cls(metadata=entries) class GroupMetadataModelV2Partial(TypedDict, total=False): diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 304a80d4ab..fc781c7899 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -190,6 +190,9 @@ def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: problems.append( ValidationProblem(("configuration",), "expected string keys", "invalid_type") ) + else: + for key, item in cast("Mapping[str, object]", configuration).items(): + problems.extend(_prefix("configuration", _prefix(key, validate_json(item)))) return problems @@ -207,14 +210,36 @@ def parse_metadata_field_v3(value: object) -> MetadataV3: def _is_int_sequence(value: object) -> bool: - """Whether `value` is a non-string sequence of integers.""" + """Whether `value` is a non-string sequence of integers. + + JSON booleans decode to `bool`, which is an `int` subclass in Python but + is not an integer in a metadata document, so booleans are excluded. + """ return ( not isinstance(value, str) and isinstance(value, Sequence) - and all(isinstance(item, int) for item in cast("Sequence[object]", value)) + and all( + isinstance(item, int) and not isinstance(item, bool) + for item in cast("Sequence[object]", value) + ) ) +def _validate_dim_sequence(doc: Mapping[str, object], key: str) -> list[ValidationProblem]: + """Validate a dimension sequence (`shape` / `chunks`) if present in `doc`. + + Dimension lengths are non-negative integers. + """ + if key not in doc: + return [] + value = doc[key] + if not _is_int_sequence(value): + return [ValidationProblem((key,), "expected a sequence of int", "invalid_type")] + if any(item < 0 for item in cast("Sequence[int]", value)): + return [ValidationProblem((key,), "expected non-negative integers", "invalid_value")] + return [] + + def _is_dtype_v2(value: object) -> bool: """Whether `value` is shaped like a v2 dtype: a string or field records. @@ -266,7 +291,10 @@ def _validate_attributes(value: object) -> list[ValidationProblem]: ("attributes",), "expected a mapping with string keys", "invalid_type" ) ] - return [] + problems: list[ValidationProblem] = [] + for key, item in cast("Mapping[str, object]", value).items(): + problems.extend(_prefix("attributes", _prefix(key, validate_json(item)))) + return problems def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: @@ -281,8 +309,7 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V3, doc) problems.extend(_check_literal(doc, "zarr_format", 3)) problems.extend(_check_literal(doc, "node_type", "array")) - if "shape" in doc and not _is_int_sequence(doc["shape"]): - problems.append(ValidationProblem(("shape",), "expected a sequence of int", "invalid_type")) + problems.extend(_validate_dim_sequence(doc, "shape")) if "fill_value" in doc: problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) for key in ("data_type", "chunk_grid", "chunk_key_encoding"): @@ -315,6 +342,16 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: ("dimension_names",), "expected items of str or None", "invalid_type" ) ) + elif _is_int_sequence(doc.get("shape")) and len(cast("Sequence[object]", names)) != len( + cast("Sequence[int]", doc["shape"]) + ): + problems.append( + ValidationProblem( + ("dimension_names",), + "expected one name per dimension of shape", + "invalid_value", + ) + ) return problems @@ -344,11 +381,8 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: doc = cast("Mapping[str, object]", value) problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V2, doc) problems.extend(_check_literal(doc, "zarr_format", 2)) - problems.extend( - ValidationProblem((key,), "expected a sequence of int", "invalid_type") - for key in ("shape", "chunks") - if key in doc and not _is_int_sequence(doc[key]) - ) + problems.extend(_validate_dim_sequence(doc, "shape")) + problems.extend(_validate_dim_sequence(doc, "chunks")) if "dtype" in doc and not _is_dtype_v2(doc["dtype"]): problems.append( ValidationProblem( @@ -415,12 +449,66 @@ def parse_array_metadata_v2(value: object) -> ArrayMetadataV2: return cast(ArrayMetadataV2, value) +def validate_consolidated_metadata_v3(value: object) -> list[ValidationProblem]: + """Return every reason `value` is not a valid inline consolidated envelope. + + Locs are value-relative (the caller prefixes with `consolidated_metadata` + where appropriate). Entries recurse into the array and group document + validators, so a validator verdict always agrees with what + `ConsolidatedMetadataModelV3.from_json` accepts. + """ + if not isinstance(value, Mapping): + return [ValidationProblem((), "expected a mapping", "invalid_type")] + env = cast("Mapping[str, object]", value) + problems: list[ValidationProblem] = [ + ValidationProblem((key,), "missing required key", "missing_key") + for key in ("kind", "must_understand", "metadata") + if key not in env + ] + problems.extend(_check_literal(env, "kind", "inline")) + if "must_understand" in env and env["must_understand"] is not False: + problems.append(ValidationProblem(("must_understand",), "expected False", "invalid_value")) + if "metadata" in env: + entries = env["metadata"] + if not isinstance(entries, Mapping): + problems.append(ValidationProblem(("metadata",), "expected a mapping", "invalid_type")) + else: + for key, entry in cast("Mapping[object, object]", entries).items(): + if not isinstance(key, str): + problems.append( + ValidationProblem(("metadata",), f"non-string key {key!r}", "invalid_type") + ) + continue + entry_obj: object = entry + node_type: object = None + if isinstance(entry, Mapping): + node_type = cast("Mapping[str, object]", entry).get("node_type") + if node_type == "array": + problems.extend( + _prefix("metadata", _prefix(key, validate_array_metadata_v3(entry_obj))) + ) + elif node_type == "group": + problems.extend( + _prefix("metadata", _prefix(key, validate_group_metadata_v3(entry_obj))) + ) + else: + problems.append( + ValidationProblem( + ("metadata", key, "node_type"), + "expected 'array' or 'group'", + "invalid_value", + ) + ) + return problems + + def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: """Return every reason `value` is not a structurally-valid v3 group doc. Checks structure, not domain validity. Unknown top-level keys are allowed (they map to `extra_fields`); a `consolidated_metadata` key, if present, - must be a mapping (its entries are validated by the consolidated model). + is deep-validated (envelope and entries) via + `validate_consolidated_metadata_v3`. """ if not isinstance(value, Mapping): return [ValidationProblem((), "expected a mapping", "invalid_type")] @@ -430,9 +518,12 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: problems.extend(_check_literal(doc, "node_type", "group")) if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) - if "consolidated_metadata" in doc and not isinstance(doc["consolidated_metadata"], Mapping): - problems.append( - ValidationProblem(("consolidated_metadata",), "expected a mapping", "invalid_type") + if "consolidated_metadata" in doc: + problems.extend( + _prefix( + "consolidated_metadata", + validate_consolidated_metadata_v3(doc["consolidated_metadata"]), + ) ) return problems diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 0df935144b..9032f18638 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -501,7 +501,9 @@ def test_v3_from_json_routes_unknown_keys_to_extra_fields() -> None: def test_v3_from_json_standard_keys_not_in_extra_fields() -> None: """V3 from_json keeps standard keys out of extra_fields.""" - doc = ArrayMetadataModelV3.create_default(attributes={"a": 1}, dimension_names=("x",)).to_json() + doc = ArrayMetadataModelV3.create_default( + shape=(10,), attributes={"a": 1}, dimension_names=("x",) + ).to_json() model = ArrayMetadataModelV3.from_json(doc) assert model.extra_fields == {} @@ -555,6 +557,7 @@ def test_from_key_value_missing_key_raises( pytest.param( ArrayMetadataModelV3, ArrayMetadataModelV3.create_default( + shape=(10,), attributes={"a": 1}, dimension_names=("x",), storage_transformers=(NamedConfigModelV3(name="t", configuration={}),), @@ -619,7 +622,9 @@ def test_roundtrip_via_key_value( def test_v3_roundtrip_json_model_json() -> None: """A v3 document round-trips through from_json/to_json back to an equal document.""" - doc = ArrayMetadataModelV3.create_default(attributes={"a": 1}, dimension_names=("x",)).to_json() + doc = ArrayMetadataModelV3.create_default( + shape=(10,), attributes={"a": 1}, dimension_names=("x",) + ).to_json() assert ArrayMetadataModelV3.from_json(doc).to_json() == doc @@ -837,7 +842,7 @@ def _set(key: str, value: object) -> Callable[[dict], object]: V3_DOC_CASES: list[Expect[Callable[[], object], frozenset[tuple[str | int, ...]]]] = [ Expect(_build_v3, frozenset(), id="valid"), Expect( - lambda: _build_v3(attributes={"a": 1}, dimension_names=("x",)), + lambda: _build_v3(shape=(10,), attributes={"a": 1}, dimension_names=("x",)), frozenset(), id="valid-with-attributes-and-dim-names", ), @@ -1156,3 +1161,57 @@ def test_extension_point_fields_annotated_with_role_alias() -> None: assert annotations[field_name] == "MetadataFieldModelV3" for field_name in ("codecs", "storage_transformers"): assert annotations[field_name] == "tuple[MetadataFieldModelV3, ...]" + + +# --- Adversarial-probe fixes: documents that used to pass validation --------- + + +def test_shape_rejects_json_booleans() -> None: + """JSON booleans are not integers: shape/chunks containing true/false are + rejected (bool is an int subclass in Python, so isinstance alone passes).""" + v3 = dict(ArrayMetadataModelV3.create_default().to_json()) | {"shape": (True, True)} + assert [p.loc for p in validate_array_metadata_v3(v3)] == [("shape",)] + v2 = dict(ArrayMetadataModelV2.create_default().to_json()) | {"chunks": (True,)} + assert [p.loc for p in validate_array_metadata_v2(v2)] == [("chunks",)] + + +def test_shape_rejects_negative_dimensions() -> None: + """Dimension lengths must be non-negative; a negative entry is invalid_value.""" + v3 = dict(ArrayMetadataModelV3.create_default().to_json()) | {"shape": (-1,)} + assert [(p.loc, p.kind) for p in validate_array_metadata_v3(v3)] == [ + (("shape",), "invalid_value") + ] + v2 = dict(ArrayMetadataModelV2.create_default().to_json()) | {"chunks": (-5,)} + assert [(p.loc, p.kind) for p in validate_array_metadata_v2(v2)] == [ + (("chunks",), "invalid_value") + ] + + +def test_dimension_names_length_must_match_shape() -> None: + """dimension_names must have one entry per dimension of shape.""" + doc = dict(ArrayMetadataModelV3.create_default(shape=(10,)).to_json()) | { + "dimension_names": ("x", "y", "z") + } + assert [(p.loc, p.kind) for p in validate_array_metadata_v3(doc)] == [ + (("dimension_names",), "invalid_value") + ] + + +def test_attributes_values_must_be_json() -> None: + """Attribute values are JSON-checked recursively (like fill_value), so a + non-serializable value is a validation problem, not a later TypeError.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"attributes": {"a": {1, 2}}} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("attributes", "a"), "invalid_type")] + + +def test_configuration_values_must_be_json() -> None: + """Configuration values are JSON-checked recursively, so an int-keyed dict + cannot pass validation and be silently rewritten by json.dumps.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | { + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": {1: 2}}} + } + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [ + (("chunk_grid", "configuration", "chunk_shape"), "invalid_type") + ] diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 829d09b11a..329471cd94 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -284,3 +284,54 @@ def test_group_v2_zarr_format_literal_enforced() -> None: """A v2 group doc claiming zarr_format 3 is rejected with invalid_value.""" problems = validate_group_metadata_v2({"zarr_format": 3}) assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] + + +# --- Consolidated envelope validated by the group validator ------------------ + + +def test_group_v3_validator_agrees_with_from_json_on_consolidated() -> None: + """The group validator validates the consolidated envelope and entries, so + is_group_metadata_v3 never vouches for a document from_json would reject.""" + bad_docs = ( + # empty envelope: missing kind/must_understand/metadata + {"zarr_format": 3, "node_type": "group", "consolidated_metadata": {}}, + # entry without a recognizable node_type + { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": {"zarr_format": 3}}, + }, + }, + # must_understand: true + { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": True, + "metadata": {}, + }, + }, + ) + for doc in bad_docs: + assert validate_group_metadata_v3(doc) != [], doc + with pytest.raises(MetadataValidationError): + GroupMetadataModelV3.from_json(doc) + + +def test_group_v3_valid_consolidated_passes_validator() -> None: + """A well-formed consolidated group validates cleanly (control case).""" + child = ArrayMetadataModelV3.create_default(shape=(2,)).to_json() + doc = { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": child, "g": {"zarr_format": 3, "node_type": "group"}}, + }, + } + assert validate_group_metadata_v3(doc) == [] From b6e53ce02b8db5e95d48683ee4e3974081abffce Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 17:09:31 +0200 Subject: [PATCH 14/70] feat(zarr-metadata): expose must_understand_fields on the v3 models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v3 core spec: 'An implementation MUST fail to open Zarr groups or arrays if any metadata fields are present which (a) the implementation does not recognize and (b) are not explicitly set to "must_understand": false' — and fields are implicitly must-understand unless waived. The model layer cannot discharge this itself: recognition is reader-specific (consolidated_metadata is itself an extension field one reader understands and another does not), and a document carrying a must-understand extension is still a valid document. So the models partition by obligation: must_understand_fields is the subset of extra_fields not explicitly waived, and a compliant reader fails to open when must_understand_fields.keys() - recognized is non-empty. The design spec pins that duty on the part-2 resolve layer, matching what zarr-python's parse_extra_fields enforces today. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 6 +++ .../src/zarr_metadata/model/_array.py | 44 +++++++++++++++++-- .../src/zarr_metadata/model/_group.py | 13 ++++++ .../zarr-metadata/tests/model/test_array.py | 29 ++++++++++++ .../zarr-metadata/tests/model/test_group.py | 15 +++++++ 5 files changed, 104 insertions(+), 3 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index fbecb7b4a4..f35625558b 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -23,3 +23,9 @@ accepted as dimension lengths, dimensions are non-negative, and `configuration` values are JSON-checked recursively (like `fill_value`), and the inline consolidated-metadata envelope and entries are deep-validated so the group validator's verdict always agrees with the model constructor. + +The v3 models expose `must_understand_fields`: the subset of `extra_fields` +not explicitly waived with `must_understand: false` (fields are implicitly +must-understand per the spec). Readers discharge the spec's fail-to-open +duty by subtracting the extension names they recognize; the model only +partitions by obligation, since recognition is reader-specific. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 33682c2008..7a4487b7f4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -4,8 +4,9 @@ import dataclasses import json +from collections.abc import Mapping from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Final, Literal, TypeAlias +from typing import TYPE_CHECKING, Final, Literal, TypeAlias, cast from typing_extensions import TypedDict, Unpack @@ -21,8 +22,6 @@ ) if TYPE_CHECKING: - from collections.abc import Mapping - from zarr_metadata._common import JSONValue from zarr_metadata.v2.array import ( ArrayDimensionSeparatorV2, @@ -82,6 +81,33 @@ def from_json(cls, data: object) -> NamedConfigModelV3: """ +def must_understand_subset( + extra_fields: Mapping[str, ExtensionFieldV3], +) -> dict[str, ExtensionFieldV3]: + """The subset of `extra_fields` the reader is obligated to understand. + + Per the v3 spec, an extension field is implicitly `must_understand: True` + unless it explicitly says otherwise, and an implementation MUST fail to + open a group or array carrying fields it does not recognize that are not + explicitly `must_understand: false`. A non-mapping field value cannot + carry the explicit waiver, so it always requires understanding (the + runtime isinstance check defends against values looser than the declared + `ExtensionFieldV3`). + """ + fields = cast("Mapping[str, object]", extra_fields) + return cast( + "dict[str, ExtensionFieldV3]", + { + name: value + for name, value in fields.items() + if not ( + isinstance(value, Mapping) + and cast("Mapping[str, object]", value).get("must_understand") is False + ) + }, + ) + + class ArrayMetadataModelV3Partial(TypedDict, total=False): """ Partial form of the constructor-settable fields of `ArrayMetadataModelV3`. @@ -240,6 +266,18 @@ def from_json(cls, data: object) -> ArrayMetadataModelV3: extra_fields=extra_fields, ) + @property + def must_understand_fields(self) -> dict[str, ExtensionFieldV3]: + """Extra fields the reader is obligated to understand. + + Everything in `extra_fields` not explicitly waived with + `must_understand: false` (the spec's implicit-true rule). A compliant + reader MUST fail to open the array if this contains any field it does + not recognize; the model layer only partitions by obligation, since + recognition is reader-specific. + """ + return must_understand_subset(self.extra_fields) + @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV3: return cls.from_json(load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V3)) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index b138a7d0f5..b8cca3849d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -13,6 +13,7 @@ from zarr_metadata.model._array import ( ATTRIBUTES_STORE_KEY_V2, ArrayMetadataModelV3, + must_understand_subset, ) from zarr_metadata.model._validation import ( GROUP_METADATA_STANDARD_KEYS_V3, @@ -159,6 +160,18 @@ def from_json(cls, data: object) -> GroupMetadataModelV3: extra_fields=extra_fields, ) + @property + def must_understand_fields(self) -> dict[str, ExtensionFieldV3]: + """Extra fields the reader is obligated to understand. + + Everything in `extra_fields` not explicitly waived with + `must_understand: false` (the spec's implicit-true rule). A compliant + reader MUST fail to open the group if this contains any field it does + not recognize; the model layer only partitions by obligation, since + recognition is reader-specific. + """ + return must_understand_subset(self.extra_fields) + @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV3: return cls.from_json(load_store_json(mapping, GROUP_METADATA_STORE_KEY_V3)) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 9032f18638..e73739929a 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1215,3 +1215,32 @@ def test_configuration_values_must_be_json() -> None: assert [(p.loc, p.kind) for p in problems] == [ (("chunk_grid", "configuration", "chunk_shape"), "invalid_type") ] + + +# --- must_understand partition (spec: MUST fail to open unrecognized fields) -- + + +def test_must_understand_fields_partition() -> None: + """must_understand_fields contains every extra field not explicitly waived + with must_understand: false, including implicitly-true and non-mapping + fields, so a reader can discharge the spec's fail-to-open duty by + subtracting the extensions it recognizes.""" + model = ArrayMetadataModelV3.create_default( + extra_fields={ + "ext_a": {"name": "a", "must_understand": False}, + "ext_b": {"name": "b"}, + "ext_c": {"name": "c", "must_understand": True}, + "ext_d": 123, + } + ) + assert set(model.must_understand_fields) == {"ext_b", "ext_c", "ext_d"} + recognized = {"ext_b"} + assert model.must_understand_fields.keys() - recognized == {"ext_c", "ext_d"} + + +def test_must_understand_fields_empty_when_all_waived() -> None: + """must_understand_fields is empty when every extra field is explicitly waived.""" + model = ArrayMetadataModelV3.create_default( + extra_fields={"ext_a": {"name": "a", "must_understand": False}} + ) + assert model.must_understand_fields == {} diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 329471cd94..d2399220be 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -335,3 +335,18 @@ def test_group_v3_valid_consolidated_passes_validator() -> None: }, } assert validate_group_metadata_v3(doc) == [] + + +# --- must_understand partition ------------------------------------------------ + + +def test_group_must_understand_fields_partition() -> None: + """The group model partitions extra fields by the spec's implicit-true rule, + like the array model.""" + model = GroupMetadataModelV3.create_default( + extra_fields={ + "waived": {"name": "w", "must_understand": False}, + "implicit": {"name": "i"}, + } + ) + assert set(model.must_understand_fields) == {"implicit"} From 22188c9fddb703450049ae335c039de600fc5c11 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 17:18:27 +0200 Subject: [PATCH 15/70] test(zarr-metadata): executable example of pydantic integration Delegate wholesale rather than letting pydantic introspect the dataclass: InstanceOf (is-instance core schema) + BeforeValidator(from_json) + PlainSerializer(to_json, return_type=dict). Field-by-field validation is impossible anyway (the models' annotation-only imports live behind TYPE_CHECKING, so pydantic raises class-not-fully-defined) and would diverge from the library's structural validation via coercion if it weren't. MetadataValidationError subclasses ValueError, so failed parses surface as pydantic ValidationError with the loc-annotated messages. pydantic is already in the package's test dependency group. Assisted-by: ClaudeCode:claude-fable-5 --- .../tests/model/test_pydantic.py | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 packages/zarr-metadata/tests/model/test_pydantic.py diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py new file mode 100644 index 0000000000..23f69aaf52 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -0,0 +1,130 @@ +"""Executable example: integrating the metadata models with pydantic (v2). + +Do NOT hand the dataclass to pydantic for field-by-field validation: the +models keep their annotation-only imports behind `TYPE_CHECKING` (so the +string annotations are unresolvable at runtime, and pydantic raises +`class-not-fully-defined`), and even if they resolved, pydantic's coercion +rules would diverge from the library's structural validation. Delegate +wholesale instead — treat the model as an opaque value: + +- `InstanceOf` makes pydantic's core schema an is-instance check (no field + introspection), +- validation goes through `from_json` (the single source of truth for what + a well-formed document is, including normalization: bare-string metadata + fields, arrays-to-tuples), +- serialization goes through `to_json` (the canonical document form). + +`MetadataValidationError` subclasses `ValueError`, so pydantic converts a +failed parse into its own `ValidationError` with the loc-annotated problem +messages intact. +""" + +from typing import Annotated + +import pytest +from pydantic import ( + BaseModel, + BeforeValidator, + InstanceOf, + PlainSerializer, + TypeAdapter, + ValidationError, +) + +from zarr_metadata.model import ArrayMetadataModelV3 + +# --- the integration (this is the example) ----------------------------------- + + +def _as_array_metadata_v3(value: object) -> ArrayMetadataModelV3: + """Accept an existing model instance or a raw metadata document.""" + if isinstance(value, ArrayMetadataModelV3): + return value + return ArrayMetadataModelV3.from_json(value) + + +# return_type is explicit because to_json's own annotation (`ArrayMetadataV3`) +# is a TYPE_CHECKING-only name pydantic cannot resolve at runtime. +ArrayMetadataV3Field = Annotated[ + InstanceOf[ArrayMetadataModelV3], + BeforeValidator(_as_array_metadata_v3), + PlainSerializer(ArrayMetadataModelV3.to_json, return_type=dict), +] +"""A pydantic-ready field type for v3 array metadata. + +Validates raw documents via `from_json`, passes model instances through, +and serializes to the canonical document form via `to_json`. +""" + + +class ArrayManifest(BaseModel): + """Example consumer model: a named array with its metadata document.""" + + path: str + metadata: ArrayMetadataV3Field + + +# --- tests pinning the example ------------------------------------------------ + +VALID_DOC = { + "zarr_format": 3, + "node_type": "array", + "shape": [10], + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}}, + "chunk_key_encoding": {"name": "default"}, + "codecs": [{"name": "bytes"}], +} + + +def test_raw_document_is_validated_into_a_model() -> None: + """A raw metadata document on a pydantic field is parsed by from_json, + with the library's normalization applied (tuples, canonical field form).""" + manifest = ArrayManifest(path="a/b", metadata=VALID_DOC) # type: ignore[arg-type] + assert isinstance(manifest.metadata, ArrayMetadataModelV3) + assert manifest.metadata.shape == (10,) + assert manifest.metadata.data_type.name == "uint8" + + +def test_model_instance_passes_through() -> None: + """An already-constructed model instance is accepted unchanged.""" + model = ArrayMetadataModelV3.from_json(VALID_DOC) + manifest = ArrayManifest(path="a/b", metadata=model) + assert manifest.metadata is model + + +def test_invalid_document_surfaces_problems_in_validation_error() -> None: + """A structurally-invalid document fails pydantic validation, carrying the + loc-annotated problem messages from MetadataValidationError.""" + doc = dict(VALID_DOC) + del doc["chunk_key_encoding"] + with pytest.raises(ValidationError) as exc_info: + ArrayManifest(path="a/b", metadata=doc) # type: ignore[arg-type] + assert "chunk_key_encoding: missing required key" in str(exc_info.value) + + +def test_dump_emits_canonical_document() -> None: + """model_dump serializes the field via to_json — the canonical document, + not pydantic's field-by-field view of the dataclass.""" + manifest = ArrayManifest(path="a/b", metadata=VALID_DOC) # type: ignore[arg-type] + dumped = manifest.model_dump() + assert dumped["metadata"] == manifest.metadata.to_json() + # the bare-string data_type was normalized to the canonical object form + assert dumped["metadata"]["data_type"] == {"name": "uint8", "configuration": {}} + + +def test_json_roundtrip_through_pydantic() -> None: + """model_dump_json output re-validates to an equal manifest (JSON emits + tuples as arrays; from_json converts them back).""" + manifest = ArrayManifest(path="a/b", metadata=VALID_DOC) # type: ignore[arg-type] + revived = ArrayManifest.model_validate_json(manifest.model_dump_json()) + assert revived == manifest + + +def test_type_adapter_standalone() -> None: + """The annotated alias also works without a BaseModel, via TypeAdapter.""" + adapter = TypeAdapter(ArrayMetadataV3Field) + model = adapter.validate_python(VALID_DOC) + assert isinstance(model, ArrayMetadataModelV3) + assert adapter.dump_python(model) == model.to_json() From 6ea15335d2ce732ba49efd614308c568ad1b212e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 17:24:03 +0200 Subject: [PATCH 16/70] test(zarr-metadata): document pydantic's native dataclass path and why not MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correcting the previous commit's too-strong claim: pydantic CAN introspect the model dataclass — TypeAdapter(...).rebuild() with the TYPE_CHECKING-only names supplied as _types_namespace resolves the schema, and __post_init__ invariants still run. A new test exercises that path and pins why it is not the recommended integration: it validates the model shape, not the document (bare-string data_type rejected — no from_json normalization), and pydantic's lax coercion silently re-opens holes the library validators close (shape=[True, -5] coerces to (1, -5); a wrong dimension_names count passes). Assisted-by: ClaudeCode:claude-fable-5 --- .../tests/model/test_pydantic.py | 74 +++++++++++++++++-- 1 file changed, 68 insertions(+), 6 deletions(-) diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index 23f69aaf52..cfd077651a 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -1,11 +1,16 @@ """Executable example: integrating the metadata models with pydantic (v2). -Do NOT hand the dataclass to pydantic for field-by-field validation: the -models keep their annotation-only imports behind `TYPE_CHECKING` (so the -string annotations are unresolvable at runtime, and pydantic raises -`class-not-fully-defined`), and even if they resolved, pydantic's coercion -rules would diverge from the library's structural validation. Delegate -wholesale instead — treat the model as an opaque value: +Pydantic's native dataclass introspection CAN be made to work (see +`test_native_dataclass_introspection_is_possible_but_diverges`): the models +keep their annotation-only imports behind `TYPE_CHECKING`, so a bare +`TypeAdapter(ArrayMetadataModelV3)` raises `class-not-fully-defined`, but +`rebuild(_types_namespace=...)` with the names supplied resolves the schema. +It is still the wrong tool: it validates the MODEL SHAPE, not the DOCUMENT — +no `from_json` normalization (a bare-string `data_type` is rejected), and +pydantic's lax coercion silently re-opens holes the library's validators +close (`shape=[True, -5]` coerces to `(1, -5)`; a wrong `dimension_names` +count passes). The recommended integration delegates wholesale — treat the +model as an opaque value: - `InstanceOf` makes pydantic's core schema an is-instance check (no field introspection), @@ -128,3 +133,60 @@ def test_type_adapter_standalone() -> None: model = adapter.validate_python(VALID_DOC) assert isinstance(model, ArrayMetadataModelV3) assert adapter.dump_python(model) == model.to_json() + + +# --- the road not taken: native dataclass introspection ---------------------- + + +def test_native_dataclass_introspection_is_possible_but_diverges() -> None: + """Pydantic CAN introspect the model dataclass after a namespace rebuild, + but that path validates the model shape, not the document: it rejects the + document form, coerces booleans into dimensions, and skips the library's + cross-field checks. This test documents why the delegation pattern above + is the recommended integration.""" + from zarr_metadata._common import JSONValue + from zarr_metadata.model import NamedConfigModelV3 + from zarr_metadata.v3._common import MetadataV3 + from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 + + adapter = TypeAdapter(ArrayMetadataModelV3) + adapter.rebuild( + force=True, + _types_namespace={ + "JSONValue": JSONValue, + "ExtensionFieldV3": ExtensionFieldV3, + "MetadataV3": MetadataV3, + "ArrayMetadataV3": ArrayMetadataV3, + "NamedConfigModelV3": NamedConfigModelV3, + "MetadataFieldModelV3": NamedConfigModelV3, + }, + ) + model_shaped = { + "shape": [10], + "fill_value": 0, + "data_type": {"name": "uint8", "configuration": {}}, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}}, + "codecs": [{"name": "bytes", "configuration": {}}], + "chunk_key_encoding": {"name": "default", "configuration": {}}, + "dimension_names": None, + "attributes": {}, + "storage_transformers": [], + "extra_fields": {}, + } + # model-shaped data validates, nested named configs and all + model = adapter.validate_python(model_shaped) + assert isinstance(model.data_type, NamedConfigModelV3) + + # divergence 1: the DOCUMENT form is rejected — no from_json normalization + with pytest.raises(ValidationError): + adapter.validate_python(model_shaped | {"data_type": "uint8"}) + + # divergence 2: lax coercion re-opens holes the library validators close + coerced = adapter.validate_python(model_shaped | {"shape": [True, -5]}) + assert coerced.shape == (1, -5) # from_json would reject both entries + + # __post_init__ invariants DO still run under pydantic construction + with pytest.raises(ValidationError, match="Extra fields"): + adapter.validate_python( + model_shaped | {"extra_fields": {"shape": {"must_understand": False}}} + ) From 03ee0b15fa8b821f2f326e003add95c009b67483 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 17:31:15 +0200 Subject: [PATCH 17/70] test(zarr-metadata): engine-backed pydantic BaseModel example (pydantic-zarr pattern) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For consumers that want a first-class BaseModel — JSON schema generation and generics for typed attributes, as in pydantic-zarr's ArraySpec — the example adds a third pattern: pydantic-native fields as the user-facing surface, with the library as the engine. A mode='before' validator canonicalizes every input via from_json(...).to_json(), so structural validation and normalization run before pydantic parses fields (the [True, -5] coercion divergence cannot occur), and to_metadata_model / to_document bridge both ways through the document form. One translation noted at the bridge: the document spells 'no dimension names' as key absence, the pydantic side as None. Assisted-by: ClaudeCode:claude-fable-5 --- .../tests/model/test_pydantic.py | 123 +++++++++++++++++- 1 file changed, 120 insertions(+), 3 deletions(-) diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index cfd077651a..fa4fb5f590 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -24,19 +24,23 @@ messages intact. """ -from typing import Annotated +from collections.abc import Mapping +from typing import Annotated, Generic, TypeVar import pytest from pydantic import ( BaseModel, BeforeValidator, + ConfigDict, InstanceOf, PlainSerializer, TypeAdapter, ValidationError, + model_validator, ) -from zarr_metadata.model import ArrayMetadataModelV3 +from zarr_metadata import JSONValue +from zarr_metadata.model import ArrayMetadataModelV3, NamedConfigModelV3 # --- the integration (this is the example) ----------------------------------- @@ -145,7 +149,6 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: cross-field checks. This test documents why the delegation pattern above is the recommended integration.""" from zarr_metadata._common import JSONValue - from zarr_metadata.model import NamedConfigModelV3 from zarr_metadata.v3._common import MetadataV3 from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 @@ -190,3 +193,117 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: adapter.validate_python( model_shaped | {"extra_fields": {"shape": {"must_understand": False}}} ) + + +# --- a first-class pydantic model, engine-backed (the pydantic-zarr pattern) -- +# +# When a consumer wants a real BaseModel — JSON schema generation, and +# generics for typed attributes, as in pydantic-zarr's ArraySpec — the model +# fields are pydantic-native, but validation and serialization still route +# through the library: a mode="before" validator canonicalizes every input +# document with from_json(...).to_json(), so the structural validators and +# normalization run BEFORE pydantic parses fields (no coercion divergence), +# and the document form is the bridge in both directions. + +AttrsT = TypeVar("AttrsT") + + +class NamedConfig(BaseModel): + """Pydantic mirror of a normalized metadata field (name + configuration).""" + + name: str + configuration: dict[str, JSONValue] = {} + + +class ArrayMetadataV3Spec(BaseModel, Generic[AttrsT]): + """A pydantic-native, attribute-typed view of a v3 array metadata document. + + The library is the engine: every input is canonicalized and structurally + validated by `ArrayMetadataModelV3.from_json` before pydantic sees the + fields, and `to_document` / `to_metadata_model` emit through the library. + """ + + model_config = ConfigDict(frozen=True) + + zarr_format: int = 3 + node_type: str = "array" + shape: tuple[int, ...] + data_type: NamedConfig + chunk_grid: NamedConfig + chunk_key_encoding: NamedConfig + fill_value: JSONValue + codecs: tuple[NamedConfig, ...] + attributes: AttrsT + dimension_names: tuple[str | None, ...] | None = None + storage_transformers: tuple[NamedConfig, ...] = () + + @model_validator(mode="before") + @classmethod + def _canonicalize(cls, data: object) -> object: + """Route every input document through the library's validation and + normalization; pydantic then parses only canonical documents.""" + if isinstance(data, Mapping): + doc = dict(ArrayMetadataModelV3.from_json(data).to_json()) + doc.setdefault("attributes", {}) + return doc + return data + + def to_metadata_model(self) -> ArrayMetadataModelV3: + """Bridge back to the canonical model, via the document form. + + In the document, "no dimension names" is key-absence, not null; the + pydantic-side None translates to dropping the key. + """ + doc = self.model_dump() + if doc["dimension_names"] is None: + del doc["dimension_names"] + return ArrayMetadataModelV3.from_json(doc) + + def to_document(self) -> dict[str, object]: + """The canonical document (omit-empty conventions applied).""" + return dict(self.to_metadata_model().to_json()) + + +class MicroscopyAttrs(BaseModel): + """Example of consumer-typed attributes, pydantic-zarr style.""" + + resolution_um: float + + +def test_spec_typed_attributes() -> None: + """The generic parameter types the attributes, so consumers get validated, + attribute-level access — the pydantic-zarr ArraySpec pattern.""" + doc = dict(VALID_DOC) | {"attributes": {"resolution_um": 0.5}} + spec = ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(doc) + assert spec.attributes.resolution_um == 0.5 + assert spec.data_type == NamedConfig(name="uint8") + + +def test_spec_engine_validates_before_pydantic() -> None: + """The library's structural validation runs before pydantic's parsing, so + coercion cannot re-open validator holes (contrast with the native + introspection test above, where [True, -5] coerced to (1, -5)).""" + with pytest.raises(ValidationError, match="shape"): + ArrayMetadataV3Spec[MicroscopyAttrs].model_validate( + dict(VALID_DOC) | {"shape": [True, -5], "attributes": {"resolution_um": 0.5}} + ) + + +def test_spec_bridges_to_canonical_model_and_document() -> None: + """to_metadata_model / to_document round-trip through the document form, + and the emitted document matches what the library itself would emit.""" + doc = dict(VALID_DOC) | {"attributes": {"resolution_um": 0.5}} + spec = ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(doc) + model = spec.to_metadata_model() + assert isinstance(model, ArrayMetadataModelV3) + assert spec.to_document() == dict(model.to_json()) + # and back: the document revalidates to an equal spec + assert ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(spec.to_document()) == spec + + +def test_spec_json_schema_generation() -> None: + """A real BaseModel means model_json_schema works — the capability the + opaque InstanceOf pattern cannot provide.""" + schema = ArrayMetadataV3Spec[MicroscopyAttrs].model_json_schema() + assert schema["properties"]["shape"]["type"] == "array" + assert "MicroscopyAttrs" in schema["$defs"] From 9193bf47bbf5b596550db6e3a75e754855aef378 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 18:14:40 +0200 Subject: [PATCH 18/70] test(zarr-metadata): pin that a null dimension_names field is invalid Spec: 'If specified, must be an array of strings or null objects... If dimension_names is not specified, all dimensions are unnamed.' The null object is a permitted element (an unnamed dimension), never the field value; key absence is the only spelling of 'not specified'. Pins the validator's existing rejection so it is not later 'fixed' to accept null-as-absence, and documents that in-memory None maps to key absence on serialization. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/tests/model/test_array.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index e73739929a..3a13aa6bea 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1244,3 +1244,17 @@ def test_must_understand_fields_empty_when_all_waived() -> None: extra_fields={"ext_a": {"name": "a", "must_understand": False}} ) assert model.must_understand_fields == {} + + +def test_dimension_names_null_field_rejected() -> None: + """A dimension_names field whose VALUE is null is invalid: the spec permits + null as an element (an unnamed dimension), never as the field value — "not + specified" is spelled by omitting the key. Consumers bridging from an + in-memory None sentinel must drop the key, not write null.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"dimension_names": None} + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dimension_names",), "invalid_type")] + # and the model's own None spelling correctly maps to key absence + assert ( + "dimension_names" not in ArrayMetadataModelV3.create_default(dimension_names=None).to_json() + ) From eaf2587280f0e2aed664d68bed96e5dfcdfeb601 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 18:34:41 +0200 Subject: [PATCH 19/70] feat(zarr-metadata): optional pydantic integration as zarr_metadata.pydantic Gamed out three shapes with prototypes before choosing: - dunders on the core classes (works, verified pydantic 2.0-2.13, but puts a framework protocol in the dependency-free layer); - pydantic-aware SUBCLASSES in a namespace (rejected on empirical failures: identity split breaks equality, core instances are rejected by subclass-typed fields, and nested construction produces core-class children unless every cross-reference is overridden); - Annotated field types over the CORE classes in an opt-in module (chosen): instances are the core classes so interop is free, pydantic imports eagerly at the module (loud failure when absent), core stays framework-free, and pydantic-protocol risk is quarantined to one clearly-labeled module. The module exports one field type per model. Validation delegates to from_json (structural validation and normalization cannot be bypassed by pydantic coercion), instances pass through, serialization emits the canonical document, and WithJsonSchema describes the accepted document form so model_json_schema works. Tests cover all seven field types, core-instance interop, error quality, JSON schema, roundtrip, and that importing zarr_metadata does not import pydantic. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 7 + .../src/zarr_metadata/pydantic.py | 131 ++++++++++++++++++ .../tests/model/test_pydantic_module.py | 125 +++++++++++++++++ 3 files changed, 263 insertions(+) create mode 100644 packages/zarr-metadata/src/zarr_metadata/pydantic.py create mode 100644 packages/zarr-metadata/tests/model/test_pydantic_module.py diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index f35625558b..44b7c1d892 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -29,3 +29,10 @@ not explicitly waived with `must_understand: false` (fields are implicitly must-understand per the spec). Readers discharge the spec's fail-to-open duty by subtracting the extension names they recognize; the model only partitions by obligation, since recognition is reader-specific. + +Optional pydantic integration ships as `zarr_metadata.pydantic` (importing it +requires pydantic v2; the core package does not depend on it): one `Annotated` +field type per model, validating raw documents through `from_json`, passing +core-model instances through unchanged, and serializing via `to_json`. The +instances are the core model classes, so values interoperate freely with +non-pydantic code. diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py new file mode 100644 index 0000000000..5ec837ee23 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -0,0 +1,131 @@ +"""Optional pydantic (v2) integration: field types over the core models. + +Importing this module requires pydantic; the core package deliberately does +not depend on it, so this module is never imported by `zarr_metadata` itself. + +Each exported name is an `Annotated` field type over the corresponding core +model class — the instances ARE the core classes, so values interoperate +freely with non-pydantic code (equality, isinstance, nesting). Validation +delegates to the library: a raw document routes through `from_json` (the +single source of truth for structural validation and normalization, so +pydantic's field-level coercion can never bypass it), an existing model +instance passes through unchanged, and serialization emits the canonical +document via `to_json`. `MetadataValidationError` subclasses `ValueError`, +so a failed parse surfaces as a pydantic `ValidationError` carrying the +loc-annotated problem messages. + +Usage: + + import zarr_metadata.pydantic as zmp + + class ArrayManifest(BaseModel): + path: str + metadata: zmp.ArrayMetadataV3 + +Static type checkers see each field type as its core model class, so +`manifest.metadata` is an `ArrayMetadataModelV3`. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Annotated, TypeVar + +from pydantic import BeforeValidator, InstanceOf, PlainSerializer, WithJsonSchema + +from zarr_metadata.model import ( + ArrayMetadataModelV2, + ArrayMetadataModelV3, + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, + GroupMetadataModelV2, + GroupMetadataModelV3, + NamedConfigModelV3, +) + +if TYPE_CHECKING: + from collections.abc import Callable + +_M = TypeVar("_M") + + +def _coerce_to(cls: type[_M]) -> Callable[[object], _M]: + """A validator that passes instances through and parses anything else.""" + + def coerce(value: object) -> _M: + if isinstance(value, cls): + return value + return cls.from_json(value) # type: ignore[attr-defined, no-any-return] + + return coerce + + +# JSON schemas describe the DOCUMENT form each field accepts (the validation +# input), not the in-memory model shape. +_DOCUMENT_SCHEMA = {"type": "object"} +_FIELD_SCHEMA = {"anyOf": [{"type": "string"}, {"type": "object"}]} + +ArrayMetadataV3 = Annotated[ + InstanceOf[ArrayMetadataModelV3], + BeforeValidator(_coerce_to(ArrayMetadataModelV3)), + PlainSerializer(ArrayMetadataModelV3.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ArrayMetadataV3"}), +] +"""Field type for a v3 array metadata document (`zarr.json` content).""" + +ArrayMetadataV2 = Annotated[ + InstanceOf[ArrayMetadataModelV2], + BeforeValidator(_coerce_to(ArrayMetadataModelV2)), + PlainSerializer(ArrayMetadataModelV2.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ArrayMetadataV2"}), +] +"""Field type for a v2 array metadata document (merged `.zarray` + `.zattrs` form).""" + +GroupMetadataV3 = Annotated[ + InstanceOf[GroupMetadataModelV3], + BeforeValidator(_coerce_to(GroupMetadataModelV3)), + PlainSerializer(GroupMetadataModelV3.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "GroupMetadataV3"}), +] +"""Field type for a v3 group metadata document (`zarr.json` content).""" + +GroupMetadataV2 = Annotated[ + InstanceOf[GroupMetadataModelV2], + BeforeValidator(_coerce_to(GroupMetadataModelV2)), + PlainSerializer(GroupMetadataModelV2.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "GroupMetadataV2"}), +] +"""Field type for a v2 group metadata document (merged `.zgroup` + `.zattrs` form).""" + +ConsolidatedMetadataV3 = Annotated[ + InstanceOf[ConsolidatedMetadataModelV3], + BeforeValidator(_coerce_to(ConsolidatedMetadataModelV3)), + PlainSerializer(ConsolidatedMetadataModelV3.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ConsolidatedMetadataV3"}), +] +"""Field type for v3 inline consolidated metadata.""" + +ConsolidatedMetadataV2 = Annotated[ + InstanceOf[ConsolidatedMetadataModelV2], + BeforeValidator(_coerce_to(ConsolidatedMetadataModelV2)), + PlainSerializer(ConsolidatedMetadataModelV2.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ConsolidatedMetadataV2"}), +] +"""Field type for a v2 `.zmetadata` document.""" + +MetadataFieldV3 = Annotated[ + InstanceOf[NamedConfigModelV3], + BeforeValidator(_coerce_to(NamedConfigModelV3)), + PlainSerializer(NamedConfigModelV3.to_json, return_type=dict), + WithJsonSchema(_FIELD_SCHEMA | {"title": "MetadataFieldV3"}), +] +"""Field type for one v3 metadata field (bare name string or name + configuration).""" + +__all__ = [ + "ArrayMetadataV2", + "ArrayMetadataV3", + "ConsolidatedMetadataV2", + "ConsolidatedMetadataV3", + "GroupMetadataV2", + "GroupMetadataV3", + "MetadataFieldV3", +] diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py new file mode 100644 index 0000000000..6202020dd6 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -0,0 +1,125 @@ +"""Tests for `zarr_metadata.pydantic`, the optional pydantic field-type module. + +The hand-rolled recipes in `test_pydantic.py` document how the integration +works; this module ships it. Instances are the CORE model classes (no +parallel hierarchy), so values interoperate freely with non-pydantic code. +""" + +import pytest +from pydantic import BaseModel, TypeAdapter, ValidationError + +import zarr_metadata.pydantic as zmp +from zarr_metadata.model import ( + ArrayMetadataModelV2, + ArrayMetadataModelV3, + ConsolidatedMetadataModelV2, + ConsolidatedMetadataModelV3, + GroupMetadataModelV2, + GroupMetadataModelV3, + NamedConfigModelV3, +) + +V3_ARRAY_DOC = dict(ArrayMetadataModelV3.create_default(shape=(4,)).to_json()) +V2_ARRAY_DOC = dict(ArrayMetadataModelV2.create_default(shape=(4,), chunks=(2,)).to_json()) +V3_GROUP_DOC = {"zarr_format": 3, "node_type": "group", "attributes": {"a": 1}} +V2_GROUP_DOC = {"zarr_format": 2, "attributes": {"a": 1}} +V3_CONSOLIDATED_DOC = { + "kind": "inline", + "must_understand": False, + "metadata": {"a": dict(V3_ARRAY_DOC)}, +} +V2_CONSOLIDATED_DOC = { + "zarr_consolidated_format": 1, + "metadata": {".zgroup": {"zarr_format": 2}}, +} + +FIELD_CASES = [ + pytest.param(zmp.ArrayMetadataV3, ArrayMetadataModelV3, V3_ARRAY_DOC, id="array-v3"), + pytest.param(zmp.ArrayMetadataV2, ArrayMetadataModelV2, V2_ARRAY_DOC, id="array-v2"), + pytest.param(zmp.GroupMetadataV3, GroupMetadataModelV3, V3_GROUP_DOC, id="group-v3"), + pytest.param(zmp.GroupMetadataV2, GroupMetadataModelV2, V2_GROUP_DOC, id="group-v2"), + pytest.param( + zmp.ConsolidatedMetadataV3, + ConsolidatedMetadataModelV3, + V3_CONSOLIDATED_DOC, + id="consolidated-v3", + ), + pytest.param( + zmp.ConsolidatedMetadataV2, + ConsolidatedMetadataModelV2, + V2_CONSOLIDATED_DOC, + id="consolidated-v2", + ), + pytest.param(zmp.MetadataFieldV3, NamedConfigModelV3, {"name": "bytes"}, id="field-v3"), +] + + +@pytest.mark.parametrize(("field_type", "model_cls", "doc"), FIELD_CASES) +def test_field_type_validates_and_dumps_canonically( + field_type: object, model_cls: type, doc: dict[str, object] +) -> None: + """Each field type parses its raw document into the CORE model class, + passes existing instances through unchanged, and dumps the canonical + document via to_json.""" + adapter = TypeAdapter(field_type) + model = adapter.validate_python(doc) + assert type(model) is model_cls + assert adapter.validate_python(model) is model + assert adapter.dump_python(model) == dict(model.to_json()) + + +def test_core_instances_interoperate() -> None: + """A core model instance (e.g. handed out by zarr-python) drops straight + into a pydantic field — the reason the module ships Annotated aliases over + the core classes rather than pydantic-aware subclasses.""" + + class Manifest(BaseModel): + metadata: zmp.ArrayMetadataV3 + + core = ArrayMetadataModelV3.from_json(V3_ARRAY_DOC) + manifest = Manifest(metadata=core) + assert manifest.metadata is core + + +def test_validation_error_carries_problems() -> None: + """A defective document fails with the library's loc-annotated messages.""" + + class Manifest(BaseModel): + metadata: zmp.ArrayMetadataV3 + + doc = dict(V3_ARRAY_DOC) + del doc["chunk_key_encoding"] + with pytest.raises(ValidationError, match="chunk_key_encoding: missing required key"): + Manifest(metadata=doc) # type: ignore[arg-type] + + +def test_json_schema_generation() -> None: + """model_json_schema works, describing the document form each field accepts.""" + + class Manifest(BaseModel): + metadata: zmp.ArrayMetadataV3 + codec: zmp.MetadataFieldV3 + + schema = Manifest.model_json_schema() + assert schema["properties"]["metadata"] == {"type": "object", "title": "ArrayMetadataV3"} + assert schema["properties"]["codec"]["anyOf"] == [{"type": "string"}, {"type": "object"}] + + +def test_json_roundtrip() -> None: + """model_dump_json output re-validates to an equal pydantic model.""" + + class Manifest(BaseModel): + metadata: zmp.ArrayMetadataV3 + + manifest = Manifest(metadata=V3_ARRAY_DOC) # type: ignore[arg-type] + assert Manifest.model_validate_json(manifest.model_dump_json()) == manifest + + +def test_core_package_does_not_import_pydantic() -> None: + """Importing zarr_metadata (in a fresh interpreter) must not import + pydantic: the integration is opt-in via zarr_metadata.pydantic.""" + import subprocess + import sys + + code = "import sys, zarr_metadata; assert 'pydantic' not in sys.modules, 'leaked'" + subprocess.run([sys.executable, "-c", code], check=True) From bd80d7dfaf754628c95ac33206e9955adcc47df1 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 18:46:23 +0200 Subject: [PATCH 20/70] fix(zarr-metadata): create_default derives the chunk grid from shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit create_default(shape=(100, 100)) silently kept the scalar default's 0-d chunk grid (chunk_shape: ()), producing a structurally-valid but semantically inconsistent document — a footgun for every test fixture built on it. When shape is overridden and the grid is not, the default is now one regular chunk covering the array (v3 chunk_shape == shape, v2 chunks == shape); an explicit chunk_grid/chunks override still wins. update() stays a dumb dataclasses.replace, per its documented contract. One existing whole-document test literal carried exactly this inconsistency (shape (10,) with chunk_shape ()) and was updated. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 5 +++ .../src/zarr_metadata/model/_array.py | 14 ++++++-- .../zarr-metadata/tests/model/test_array.py | 34 ++++++++++++++++++- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 44b7c1d892..088d6e11a6 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -36,3 +36,8 @@ field type per model, validating raw documents through `from_json`, passing core-model instances through unchanged, and serializing via `to_json`. The instances are the core model classes, so values interoperate freely with non-pydantic code. + +`create_default` keeps its output self-consistent: overriding `shape` without +a chunk grid derives one regular chunk covering the array (v3 +`chunk_shape == shape`; v2 `chunks == shape`) instead of silently keeping the +scalar default's 0-d grid. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 7a4487b7f4..af680e8d3f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -168,8 +168,14 @@ def create_default( The default is a structurally-valid scalar `uint8` array — the array analog of `list()` returning `[]`. Any field can be overridden by keyword - (the same fields accepted by `update`). + (the same fields accepted by `update`). Overriding `shape` without + `chunk_grid` derives a consistent default grid: one regular chunk + covering the array (`chunk_shape` equal to `shape`). """ + if "shape" in overrides and "chunk_grid" not in overrides: + overrides["chunk_grid"] = NamedConfigModelV3( + name="regular", configuration={"chunk_shape": tuple(overrides["shape"])} + ) default = cls( shape=(), fill_value=0, @@ -353,8 +359,12 @@ def create_default( The default is a structurally-valid scalar `uint8` (`"|u1"`) array — the array analog of `list()` returning `[]`. Any field can be overridden by - keyword (the same fields accepted by `update`). + keyword (the same fields accepted by `update`). Overriding `shape` + without `chunks` derives `chunks` equal to `shape` (one chunk covering + the array). """ + if "shape" in overrides and "chunks" not in overrides: + overrides["chunks"] = tuple(overrides["shape"]) default = cls( shape=(), dtype="|u1", diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 3a13aa6bea..535541d180 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -180,7 +180,7 @@ def test_v3_to_json_emits_canonical_document() -> None: "shape": (10,), "fill_value": 0, "data_type": {"name": "int32", "configuration": {}}, - "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": ()}}, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (10,)}}, "codecs": ({"name": "bytes", "configuration": {}},), "chunk_key_encoding": {"name": "default", "configuration": {}}, } @@ -1258,3 +1258,35 @@ def test_dimension_names_null_field_rejected() -> None: assert ( "dimension_names" not in ArrayMetadataModelV3.create_default(dimension_names=None).to_json() ) + + +# --- create_default derives the chunk grid from shape ------------------------ + + +def test_v3_create_default_chunk_grid_follows_shape() -> None: + """Overriding shape without chunk_grid derives a consistent default grid: + one chunk covering the array (chunk_shape == shape), instead of silently + keeping the scalar default's 0-d grid.""" + model = ArrayMetadataModelV3.create_default(shape=(100, 100)) + assert model.chunk_grid == NamedConfigModelV3( + name="regular", configuration={"chunk_shape": (100, 100)} + ) + + +def test_v3_create_default_explicit_chunk_grid_respected() -> None: + """An explicit chunk_grid override wins over the shape-derived default.""" + grid = NamedConfigModelV3(name="regular", configuration={"chunk_shape": (10, 10)}) + model = ArrayMetadataModelV3.create_default(shape=(100, 100), chunk_grid=grid) + assert model.chunk_grid == grid + + +def test_v2_create_default_chunks_follow_shape() -> None: + """Overriding shape without chunks derives chunks == shape.""" + model = ArrayMetadataModelV2.create_default(shape=(100, 100)) + assert model.chunks == (100, 100) + + +def test_v2_create_default_explicit_chunks_respected() -> None: + """An explicit chunks override wins over the shape-derived default.""" + model = ArrayMetadataModelV2.create_default(shape=(100, 100), chunks=(10, 10)) + assert model.chunks == (10, 10) From bdec803ac47707e69eb2121b26ca07238c8a1d3f Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 18:49:31 +0200 Subject: [PATCH 21/70] test(zarr-metadata): pin zero-length-dimension case of the derived chunk grid The spec's constraint is conditional ('non-zero when the corresponding dimensions of the arrays have non-zero length'), so chunk_shape == shape is sound for every shape, including empty dimensions. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/tests/model/test_array.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 535541d180..a5ffbb7080 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1290,3 +1290,12 @@ def test_v2_create_default_explicit_chunks_respected() -> None: """An explicit chunks override wins over the shape-derived default.""" model = ArrayMetadataModelV2.create_default(shape=(100, 100), chunks=(10, 10)) assert model.chunks == (10, 10) + + +def test_v3_create_default_zero_length_dimensions() -> None: + """chunk_shape == shape is spec-sound even with zero-length dimensions: + 'The chunk shape elements are non-zero when the corresponding dimensions + of the arrays have non-zero length' — the constraint is conditional, so a + zero chunk length is permitted exactly where the dimension is empty.""" + model = ArrayMetadataModelV3.create_default(shape=(0, 3)) + assert model.chunk_grid.configuration["chunk_shape"] == (0, 3) From 965ea029cd0d5e5421c5b34a6e01b0d65f1db854 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 18:53:20 +0200 Subject: [PATCH 22/70] docs(zarr-metadata): document that create_default's derivation is one-way MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Overriding shape without a grid derives the grid; the reverse does not hold. A user-supplied chunk_grid is an extension point taken verbatim — deriving shape from it would require interpreting grid configurations, which the model layer never does and cannot do for unrecognized grid names. Pinned by test so the asymmetry reads as a decision, not an oversight; the v2 model documents the same one-way rule for chunks for cross-version consistency. Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/_array.py | 13 +++++++++++++ packages/zarr-metadata/tests/model/test_array.py | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index af680e8d3f..cfd43571ec 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -171,6 +171,14 @@ def create_default( (the same fields accepted by `update`). Overriding `shape` without `chunk_grid` derives a consistent default grid: one regular chunk covering the array (`chunk_shape` equal to `shape`). + + The derivation is deliberately one-way. A user-supplied `chunk_grid` + is an extension point and is taken verbatim — deriving `shape` from + it would require interpreting the grid's configuration, which this + layer never does (and cannot do for unrecognized grid names). So + overriding `chunk_grid` without `shape` keeps the scalar default + `shape=()`, and consistency between the two is the caller's + responsibility. """ if "shape" in overrides and "chunk_grid" not in overrides: overrides["chunk_grid"] = NamedConfigModelV3( @@ -362,6 +370,11 @@ def create_default( keyword (the same fields accepted by `update`). Overriding `shape` without `chunks` derives `chunks` equal to `shape` (one chunk covering the array). + + The derivation is deliberately one-way, matching the v3 model: + overriding `chunks` without `shape` keeps the scalar default + `shape=()`, and consistency between the two is the caller's + responsibility. """ if "shape" in overrides and "chunks" not in overrides: overrides["chunks"] = tuple(overrides["shape"]) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index a5ffbb7080..3f7f35c50c 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1299,3 +1299,17 @@ def test_v3_create_default_zero_length_dimensions() -> None: zero chunk length is permitted exactly where the dimension is empty.""" model = ArrayMetadataModelV3.create_default(shape=(0, 3)) assert model.chunk_grid.configuration["chunk_shape"] == (0, 3) + + +def test_create_default_derivation_is_one_way() -> None: + """Overriding the chunk grid (v3) or chunks (v2) without shape leaves the + scalar default shape=() untouched: a user-supplied chunk_grid is an + extension point taken verbatim, and deriving shape from it would require + interpreting grid configurations, which the model layer never does.""" + grid = NamedConfigModelV3(name="regular", configuration={"chunk_shape": (10, 10)}) + v3 = ArrayMetadataModelV3.create_default(chunk_grid=grid) + assert v3.shape == () + assert v3.chunk_grid == grid + v2 = ArrayMetadataModelV2.create_default(chunks=(10, 10)) + assert v2.shape == () + assert v2.chunks == (10, 10) From d4a0f8eecf7a88a25eb6267cca38f8b938c837dd Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 19:02:29 +0200 Subject: [PATCH 23/70] refactor(zarr-metadata): eliminate every type-ignore comment Audited all 24 (15 src, 9 tests); each was either obsolete, replaceable by a sound cast, or avoidable by better-typed code: - Two fill_value arg-type ignores were factually obsolete: their justifying comment said 'fill_value: object in upstream TypedDict', but 0.3.0 narrowed it to JSONValue. - Eight pre-existing call-arg/reportInvalidTypeForm ignores on the PEP 728 TypedDicts and the recursive JSONValue alias were mypy-dialect suppressions that the checker of record (pyright strict with enableExperimentalFeatures) never needed; mypy has never checked this package. - The two extra_fields comprehensions are a genuine checker limitation (a key filter cannot narrow a PEP 728 TypedDict's item-value union), now expressed as casts whose comments state the soundness claim instead of suppressing the diagnostic. - pydantic.py's generic coercer factory takes the parse callable explicitly instead of calling from_json through type[_M]. - NamedConfigModelV3.from_json casts the validated configuration (sound since configuration values are now deep-validated as JSON). - Tests: _build_v2/_build_v3 gained real Unpack[...Partial] signatures; raw-document pydantic inputs go through model_validate (the idiomatic entry point for untyped data) instead of ignoring constructor signatures; the frozen-dataclass test uses setattr for its intentional runtime error. src and tests/model now carry zero type-ignore comments. Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/_common.py | 2 +- .../src/zarr_metadata/model/_array.py | 25 ++++++++++++------- .../src/zarr_metadata/model/_group.py | 16 ++++++++---- .../src/zarr_metadata/pydantic.py | 20 +++++++-------- .../src/zarr_metadata/v2/codec.py | 2 +- .../src/zarr_metadata/v3/array.py | 6 ++--- .../src/zarr_metadata/v3/codec/crc32c.py | 2 +- .../src/zarr_metadata/v3/group.py | 4 +-- .../zarr-metadata/tests/model/test_array.py | 13 ++++++---- .../tests/model/test_pydantic.py | 8 +++--- .../tests/model/test_pydantic_module.py | 4 +-- 11 files changed, 59 insertions(+), 43 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index 598a12e80c..b335cd0bd6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -13,7 +13,7 @@ JSONValue = TypeAliasType( "JSONValue", - "int | float | bool | None | str | list[JSONValue] | tuple[JSONValue, ...] | Mapping[str, JSONValue]", # type: ignore[reportInvalidTypeForm] + "int | float | bool | None | str | list[JSONValue] | tuple[JSONValue, ...] | Mapping[str, JSONValue]", ) """A recursive type alias for JSON-encodable values. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index cfd43571ec..991dd77d3f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -63,8 +63,13 @@ def from_json(cls, data: object) -> NamedConfigModelV3: field = parse_metadata_field_v3(data) if isinstance(field, str): return cls(name=field, configuration={}) - configuration = arrays_to_tuples(dict(field.get("configuration", {}))) - return cls(name=field["name"], configuration=configuration) # type: ignore[arg-type] + # Sound cast: parse_metadata_field_v3 checked the configuration is a + # string-keyed mapping of JSON values; arrays_to_tuples only converts + # lists to tuples within that shape. + configuration = cast( + "dict[str, JSONValue]", arrays_to_tuples(dict(field.get("configuration", {}))) + ) + return cls(name=field["name"], configuration=configuration) MetadataFieldModelV3: TypeAlias = NamedConfigModelV3 @@ -260,14 +265,16 @@ def to_json(self) -> ArrayMetadataV3: @classmethod def from_json(cls, data: object) -> ArrayMetadataModelV3: parsed = parse_array_metadata_v3(arrays_to_tuples(data)) - extra_fields: dict[str, ExtensionFieldV3] = { - k: v # type: ignore[misc] - for k, v in parsed.items() - if k not in ARRAY_METADATA_STANDARD_KEYS_V3 - } + # Sound cast: the TypedDict types all non-standard keys as its + # `extra_items` (`ExtensionFieldV3`); the comprehension's inferred value + # type is the union over ALL keys because the key filter cannot narrow it. + extra_fields = cast( + "dict[str, ExtensionFieldV3]", + {k: v for k, v in parsed.items() if k not in ARRAY_METADATA_STANDARD_KEYS_V3}, + ) return cls( shape=parsed["shape"], - fill_value=parsed["fill_value"], # type: ignore[arg-type] # fill_value: object in upstream TypedDict + fill_value=parsed["fill_value"], data_type=NamedConfigModelV3.from_json(parsed["data_type"]), chunk_grid=NamedConfigModelV3.from_json(parsed["chunk_grid"]), codecs=tuple(NamedConfigModelV3.from_json(c) for c in parsed["codecs"]), @@ -418,7 +425,7 @@ def from_json(cls, data: object) -> ArrayMetadataModelV2: shape=parsed["shape"], dtype=parsed["dtype"], chunks=parsed["chunks"], - fill_value=parsed["fill_value"], # type: ignore[arg-type] # fill_value: object in upstream TypedDict + fill_value=parsed["fill_value"], order=parsed["order"], compressor=parsed["compressor"], filters=parsed["filters"], diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index b8cca3849d..0980096a8b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -149,11 +149,17 @@ def from_json(cls, data: object) -> GroupMetadataModelV3: if consolidated_raw is None else ConsolidatedMetadataModelV3.from_json(consolidated_raw) ) - extra_fields: dict[str, ExtensionFieldV3] = { - k: v # type: ignore[misc] - for k, v in parsed.items() - if k not in GROUP_METADATA_STANDARD_KEYS_V3 and k != CONSOLIDATED_METADATA_KEY_V3 - } + # Sound cast: the TypedDict types all non-standard keys as its + # `extra_items` (`ExtensionFieldV3`); the comprehension's inferred value + # type is the union over ALL keys because the key filter cannot narrow it. + extra_fields = cast( + "dict[str, ExtensionFieldV3]", + { + k: v + for k, v in parsed.items() + if k not in GROUP_METADATA_STANDARD_KEYS_V3 and k != CONSOLIDATED_METADATA_KEY_V3 + }, + ) return cls( attributes=dict(parsed.get("attributes", {})), consolidated_metadata=consolidated, diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index 5ec837ee23..3eada76316 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -48,13 +48,13 @@ class ArrayManifest(BaseModel): _M = TypeVar("_M") -def _coerce_to(cls: type[_M]) -> Callable[[object], _M]: - """A validator that passes instances through and parses anything else.""" +def _coerce_to(cls: type[_M], parse: Callable[[object], _M]) -> Callable[[object], _M]: + """A validator that passes instances of `cls` through and parses anything else.""" def coerce(value: object) -> _M: if isinstance(value, cls): return value - return cls.from_json(value) # type: ignore[attr-defined, no-any-return] + return parse(value) return coerce @@ -66,7 +66,7 @@ def coerce(value: object) -> _M: ArrayMetadataV3 = Annotated[ InstanceOf[ArrayMetadataModelV3], - BeforeValidator(_coerce_to(ArrayMetadataModelV3)), + BeforeValidator(_coerce_to(ArrayMetadataModelV3, ArrayMetadataModelV3.from_json)), PlainSerializer(ArrayMetadataModelV3.to_json, return_type=dict), WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ArrayMetadataV3"}), ] @@ -74,7 +74,7 @@ def coerce(value: object) -> _M: ArrayMetadataV2 = Annotated[ InstanceOf[ArrayMetadataModelV2], - BeforeValidator(_coerce_to(ArrayMetadataModelV2)), + BeforeValidator(_coerce_to(ArrayMetadataModelV2, ArrayMetadataModelV2.from_json)), PlainSerializer(ArrayMetadataModelV2.to_json, return_type=dict), WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ArrayMetadataV2"}), ] @@ -82,7 +82,7 @@ def coerce(value: object) -> _M: GroupMetadataV3 = Annotated[ InstanceOf[GroupMetadataModelV3], - BeforeValidator(_coerce_to(GroupMetadataModelV3)), + BeforeValidator(_coerce_to(GroupMetadataModelV3, GroupMetadataModelV3.from_json)), PlainSerializer(GroupMetadataModelV3.to_json, return_type=dict), WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "GroupMetadataV3"}), ] @@ -90,7 +90,7 @@ def coerce(value: object) -> _M: GroupMetadataV2 = Annotated[ InstanceOf[GroupMetadataModelV2], - BeforeValidator(_coerce_to(GroupMetadataModelV2)), + BeforeValidator(_coerce_to(GroupMetadataModelV2, GroupMetadataModelV2.from_json)), PlainSerializer(GroupMetadataModelV2.to_json, return_type=dict), WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "GroupMetadataV2"}), ] @@ -98,7 +98,7 @@ def coerce(value: object) -> _M: ConsolidatedMetadataV3 = Annotated[ InstanceOf[ConsolidatedMetadataModelV3], - BeforeValidator(_coerce_to(ConsolidatedMetadataModelV3)), + BeforeValidator(_coerce_to(ConsolidatedMetadataModelV3, ConsolidatedMetadataModelV3.from_json)), PlainSerializer(ConsolidatedMetadataModelV3.to_json, return_type=dict), WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ConsolidatedMetadataV3"}), ] @@ -106,7 +106,7 @@ def coerce(value: object) -> _M: ConsolidatedMetadataV2 = Annotated[ InstanceOf[ConsolidatedMetadataModelV2], - BeforeValidator(_coerce_to(ConsolidatedMetadataModelV2)), + BeforeValidator(_coerce_to(ConsolidatedMetadataModelV2, ConsolidatedMetadataModelV2.from_json)), PlainSerializer(ConsolidatedMetadataModelV2.to_json, return_type=dict), WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ConsolidatedMetadataV2"}), ] @@ -114,7 +114,7 @@ def coerce(value: object) -> _M: MetadataFieldV3 = Annotated[ InstanceOf[NamedConfigModelV3], - BeforeValidator(_coerce_to(NamedConfigModelV3)), + BeforeValidator(_coerce_to(NamedConfigModelV3, NamedConfigModelV3.from_json)), PlainSerializer(NamedConfigModelV3.to_json, return_type=dict), WithJsonSchema(_FIELD_SCHEMA | {"title": "MetadataFieldV3"}), ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/codec.py b/packages/zarr-metadata/src/zarr_metadata/v2/codec.py index 6d194b7e29..8f7f2b3f88 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/codec.py @@ -10,7 +10,7 @@ from zarr_metadata._common import JSONValue -class CodecMetadataV2(TypedDict, extra_items=JSONValue): # type: ignore[call-arg] +class CodecMetadataV2(TypedDict, extra_items=JSONValue): """ A numcodecs configuration dict, used as a v2 compressor or filter. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/array.py b/packages/zarr-metadata/src/zarr_metadata/v3/array.py index a8b0fa3358..a3fec24a6c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/array.py @@ -9,7 +9,7 @@ from zarr_metadata.v3._common import MetadataV3 -class ExtensionFieldV3(TypedDict, extra_items=JSONValue): # type: ignore[call-arg] +class ExtensionFieldV3(TypedDict, extra_items=JSONValue): """ Required shape of any extension field on a v3 metadata document. @@ -41,7 +41,7 @@ class ExtensionFieldV3(TypedDict, extra_items=JSONValue): # type: ignore[call-a must_understand: bool -class ArrayMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class ArrayMetadataV3(TypedDict, extra_items=ExtensionFieldV3): """ Zarr v3 array metadata document (the `zarr.json` content for an array). @@ -63,7 +63,7 @@ class ArrayMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[ dimension_names: NotRequired[tuple[str | None, ...]] -class ArrayMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class ArrayMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): """ Partial form of `ArrayMetadataV3`: every field is `NotRequired`. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index ea35ae5f1d..6b9b46c43d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -18,7 +18,7 @@ """Literal type of the `name` field of the `crc32c` codec.""" -class Empty(TypedDict, closed=True): # type: ignore[call-arg] +class Empty(TypedDict, closed=True): """An empty mapping""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/group.py b/packages/zarr-metadata/src/zarr_metadata/v3/group.py index 27186b6059..be990b1ae7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/group.py @@ -12,7 +12,7 @@ from zarr_metadata.v3.array import ExtensionFieldV3 -class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): """ Zarr v3 group metadata document (the `zarr.json` content for a group). @@ -26,7 +26,7 @@ class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[ attributes: NotRequired[Mapping[str, JSONValue]] -class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): # type: ignore[call-arg] +class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): """ Partial form of `GroupMetadataV3`: every field is `NotRequired`. diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 3f7f35c50c..f1ce967b99 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING import pytest +from typing_extensions import Unpack from tests.model._cases import Expect, ExpectFail from zarr_metadata.model import ( @@ -814,12 +815,12 @@ def test_parse_metadata_field_v3( # invalid cases (a subset check, so accumulation of OTHER problems is allowed). -def _build_v3(**overrides: object) -> dict[str, object]: - return dict(ArrayMetadataModelV3.create_default(**overrides).to_json()) # type: ignore[arg-type] +def _build_v3(**overrides: Unpack[ArrayMetadataModelV3Partial]) -> dict[str, object]: + return dict(ArrayMetadataModelV3.create_default(**overrides).to_json()) -def _build_v2(**overrides: object) -> dict[str, object]: - return dict(ArrayMetadataModelV2.create_default(**overrides).to_json()) # type: ignore[arg-type] +def _build_v2(**overrides: Unpack[ArrayMetadataModelV2Partial]) -> dict[str, object]: + return dict(ArrayMetadataModelV2.create_default(**overrides).to_json()) def _mutate(build: Callable[[], dict], mutate: Callable[[dict], object]) -> Callable[[], dict]: @@ -1008,7 +1009,9 @@ def test_validation_problem_is_frozen() -> None: """ValidationProblem is immutable (frozen dataclass).""" p = ValidationProblem(loc=("shape",), message="x", kind="invalid_type") with pytest.raises(dataclasses.FrozenInstanceError): - p.message = "y" # type: ignore[misc] + # setattr: assigning to a frozen field is an intentional runtime error, + # spelled dynamically so it is not also a static type error. + setattr(p, "message", "y") # noqa: B010 def test_metadata_validation_error_holds_problems() -> None: diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index fa4fb5f590..c1cd3010d4 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -90,7 +90,7 @@ class ArrayManifest(BaseModel): def test_raw_document_is_validated_into_a_model() -> None: """A raw metadata document on a pydantic field is parsed by from_json, with the library's normalization applied (tuples, canonical field form).""" - manifest = ArrayManifest(path="a/b", metadata=VALID_DOC) # type: ignore[arg-type] + manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) assert isinstance(manifest.metadata, ArrayMetadataModelV3) assert manifest.metadata.shape == (10,) assert manifest.metadata.data_type.name == "uint8" @@ -109,14 +109,14 @@ def test_invalid_document_surfaces_problems_in_validation_error() -> None: doc = dict(VALID_DOC) del doc["chunk_key_encoding"] with pytest.raises(ValidationError) as exc_info: - ArrayManifest(path="a/b", metadata=doc) # type: ignore[arg-type] + ArrayManifest.model_validate({"path": "a/b", "metadata": doc}) assert "chunk_key_encoding: missing required key" in str(exc_info.value) def test_dump_emits_canonical_document() -> None: """model_dump serializes the field via to_json — the canonical document, not pydantic's field-by-field view of the dataclass.""" - manifest = ArrayManifest(path="a/b", metadata=VALID_DOC) # type: ignore[arg-type] + manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) dumped = manifest.model_dump() assert dumped["metadata"] == manifest.metadata.to_json() # the bare-string data_type was normalized to the canonical object form @@ -126,7 +126,7 @@ def test_dump_emits_canonical_document() -> None: def test_json_roundtrip_through_pydantic() -> None: """model_dump_json output re-validates to an equal manifest (JSON emits tuples as arrays; from_json converts them back).""" - manifest = ArrayManifest(path="a/b", metadata=VALID_DOC) # type: ignore[arg-type] + manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) revived = ArrayManifest.model_validate_json(manifest.model_dump_json()) assert revived == manifest diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 6202020dd6..81b9b962af 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -90,7 +90,7 @@ class Manifest(BaseModel): doc = dict(V3_ARRAY_DOC) del doc["chunk_key_encoding"] with pytest.raises(ValidationError, match="chunk_key_encoding: missing required key"): - Manifest(metadata=doc) # type: ignore[arg-type] + Manifest.model_validate({"metadata": doc}) def test_json_schema_generation() -> None: @@ -111,7 +111,7 @@ def test_json_roundtrip() -> None: class Manifest(BaseModel): metadata: zmp.ArrayMetadataV3 - manifest = Manifest(metadata=V3_ARRAY_DOC) # type: ignore[arg-type] + manifest = Manifest.model_validate({"metadata": V3_ARRAY_DOC}) assert Manifest.model_validate_json(manifest.model_dump_json()) == manifest From 0a7f674ac29f5cd51dee19ae73d37a44f1b7bb4c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 19:14:23 +0200 Subject: [PATCH 24/70] fix(zarr-metadata): absent v2 dimension_separator means '.', not '/' roborev job 426 (branch review) found that ArrayMetadataModelV2 normalized an ABSENT dimension_separator key to '/', inherited verbatim from the zng prototype. The v2 convention's default is '.': a consumer deriving chunk keys from the model against a real-world v2 array written with the default separator would have looked for '0/0' instead of '0.0'. No test caught it because every fixture started from create_default(), which always carries an explicit separator. Absence is normalized to an explicit '.' -- a semantics-preserving spelling normalization consistent with the model's existing canonical forms (bare-string metadata fields, missing configuration). The field is deliberately NOT modeled as Optional: the document grammar has no null spelling for this key, and a None in the model invites writing 'dimension_separator': null into documents. Pinned by three tests, including explicit-null rejection. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 5 +++ .../src/zarr_metadata/model/_array.py | 12 ++++-- .../zarr-metadata/tests/model/test_array.py | 37 +++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 088d6e11a6..3274fd2e3b 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -41,3 +41,8 @@ non-pydantic code. a chunk grid derives one regular chunk covering the array (v3 `chunk_shape == shape`; v2 `chunks == shape`) instead of silently keeping the scalar default's 0-d grid. + +A v2 `.zarray` that omits `dimension_separator` is interpreted with the v2 +convention's default `"."` (the model previously normalized absence to `"/"`, +which would misaddress the chunks of real-world default-separator arrays). +The value is never null: absent, `"."`, or `"/"` are the only spellings. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 991dd77d3f..6d1d2c3d6b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -340,7 +340,9 @@ class ArrayMetadataModelV2: A canonical, lossless representation of the `.zarray` content plus the sibling `.zattrs` attributes. `dtype`, `compressor`, and `filters` are held in their raw JSON forms and are never interpreted; `fill_value` is - held verbatim in its JSON form. + held verbatim in its JSON form. One spelling normalization: a `.zarray` + that omits `dimension_separator` means `"."` by the v2 convention, and + the model holds and re-emits that value explicitly. """ zarr_format: Literal[2] = field(default=2, init=False) @@ -351,7 +353,11 @@ class ArrayMetadataModelV2: order: ArrayOrderV2 compressor: CodecMetadataV2 | None filters: tuple[CodecMetadataV2, ...] | None - dimension_separator: ArrayDimensionSeparatorV2 = field(default="/") + # "." is the v2 convention's default for an ABSENT dimension_separator key; + # from_json normalizes absence to it (a semantics-preserving spelling + # normalization, like the v3 bare-string metadata-field form). The value + # is never None: the document grammar has no null spelling for this field. + dimension_separator: ArrayDimensionSeparatorV2 = field(default=".") attributes: dict[str, JSONValue] def update(self, **kwargs: Unpack[ArrayMetadataModelV2Partial]) -> ArrayMetadataModelV2: @@ -429,7 +435,7 @@ def from_json(cls, data: object) -> ArrayMetadataModelV2: order=parsed["order"], compressor=parsed["compressor"], filters=parsed["filters"], - dimension_separator=parsed.get("dimension_separator", "/"), + dimension_separator=parsed.get("dimension_separator", "."), attributes=dict(parsed.get("attributes", {})), ) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index f1ce967b99..4805fc2674 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1316,3 +1316,40 @@ def test_create_default_derivation_is_one_way() -> None: v2 = ArrayMetadataModelV2.create_default(chunks=(10, 10)) assert v2.shape == () assert v2.chunks == (10, 10) + + +# --- v2 dimension_separator default (roborev job 426) ------------------------- + + +def test_v2_absent_dimension_separator_means_dot() -> None: + """A .zarray that omits dimension_separator uses the v2 convention default + '.', not '/': chunk keys of real-world default-separator v2 arrays look + like '0.0'. The model normalizes the absent key to an explicit '.' — a + semantics-preserving spelling normalization.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) + del doc["dimension_separator"] + model = ArrayMetadataModelV2.from_json(doc) + assert model.dimension_separator == "." + assert model.to_json()["dimension_separator"] == "." + + +def test_v2_from_key_value_without_separator_means_dot() -> None: + """The .zarray store-file path applies the same '.' default for an absent + dimension_separator key.""" + doc = { + k: v + for k, v in ArrayMetadataModelV2.create_default().to_json().items() + if k not in ("dimension_separator", "attributes") + } + import json as _json + + model = ArrayMetadataModelV2.from_key_value({".zarray": _json.dumps(doc).encode()}) + assert model.dimension_separator == "." + + +def test_v2_null_dimension_separator_rejected() -> None: + """dimension_separator may be absent, '.', or '/' — never null: the + document grammar has no null spelling for this field.""" + doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dimension_separator": None} + problems = validate_array_metadata_v2(doc) + assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] From 69ae90192af1bafc07a7905f5f8b7b384a86d883 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 20:58:07 +0200 Subject: [PATCH 25/70] feat(zarr-metadata): UNSET sentinel for absent optional document keys Establishes the models' None/absence invariant: None in a model always corresponds to a JSON null in the document (a v2 compressor/filters value, an unnamed dimension inside dimension_names), and UNSET always means the key is absent. The two are never interchangeable. Applied to the two fields that used None as an absence marker: dimension_names (ArrayMetadataModelV3) and consolidated_metadata (GroupMetadataModelV3). For dimension_names this also preserves a semantic distinction d-v-b identified: an absent field ("there are no dimension names") and an explicit all-null array ("every dimension has a name, which is null") are different documents; both spellings now round-trip faithfully and compare unequal. Normalizing absence to the all-null form was considered and rejected: the spellings' interpretations coincide but interpretation-equivalence is the resolve layer's business, and collapsing document-level distinctions on that basis is the layer violation this package exists to avoid. Verified that current zarr-python never writes "consolidated_metadata": null (GroupMetadata.to_dict pops the key), so None there was purely an absence marker, not a document spelling. UnsetType is a single-member enum (identity-checkable, repr "UNSET", deliberately truthy so `if not x` cannot silently treat it as absent); UNSET and UnsetType are exported from zarr_metadata.model and the package front door. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 8 +++++ .../src/zarr_metadata/__init__.py | 4 +++ .../src/zarr_metadata/model/__init__.py | 3 ++ .../src/zarr_metadata/model/_array.py | 11 +++--- .../src/zarr_metadata/model/_group.py | 15 ++++---- .../src/zarr_metadata/model/_sentinel.py | 30 ++++++++++++++++ .../zarr-metadata/tests/model/test_array.py | 36 +++++++++++++++---- .../zarr-metadata/tests/model/test_group.py | 5 +-- .../tests/model/test_pydantic.py | 4 ++- .../zarr-metadata/tests/test_public_api.py | 2 ++ 10 files changed, 96 insertions(+), 22 deletions(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 3274fd2e3b..29c893b3c1 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -46,3 +46,11 @@ A v2 `.zarray` that omits `dimension_separator` is interpreted with the v2 convention's default `"."` (the model previously normalized absence to `"/"`, which would misaddress the chunks of real-world default-separator arrays). The value is never null: absent, `"."`, or `"/"` are the only spellings. + +Optional document keys use the `UNSET` sentinel, never `None`: in a model, +`None` always corresponds to a JSON `null` in the document (a v2 +`compressor`, an unnamed dimension inside `dimension_names`), and `UNSET` +always means the key is absent. This keeps semantically distinct spellings +distinct — an absent `dimension_names` ("there are no dimension names") and +an explicit `[null, null]` ("every dimension has a name, which is null") are +different documents and round-trip as such. diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index 354f2a97f5..4e0cfd4e36 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -2,6 +2,7 @@ from zarr_metadata._common import JSONValue, NamedConfigV3 from zarr_metadata.model import ( + UNSET, ArrayMetadataModelV2, ArrayMetadataModelV2Partial, ArrayMetadataModelV3, @@ -16,6 +17,7 @@ MetadataValidationError, NamedConfigModelV3, ProblemKind, + UnsetType, ValidationProblem, ) from zarr_metadata.v2.array import ( @@ -248,6 +250,7 @@ "UINT16_DATA_TYPE_NAME", "UINT32_DATA_TYPE_NAME", "UINT64_DATA_TYPE_NAME", + "UNSET", "V2_CHUNK_KEY_ENCODING_NAME", "V2_CHUNK_KEY_ENCODING_SEPARATOR", "ZSTD_CODEC_NAME", @@ -353,6 +356,7 @@ "Uint32FillValue", "Uint64DataTypeName", "Uint64FillValue", + "UnsetType", "V2ChunkKeyEncodingMetadata", "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingSeparator", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 42fbc233d8..76367eae32 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -34,6 +34,7 @@ GroupMetadataModelV3, GroupMetadataModelV3Partial, ) +from zarr_metadata.model._sentinel import UNSET, UnsetType from zarr_metadata.model._validation import ( ARRAY_METADATA_OPTIONAL_KEYS_V3, ARRAY_METADATA_REQUIRED_KEYS_V2, @@ -82,6 +83,7 @@ "GROUP_METADATA_STANDARD_KEYS_V3", "GROUP_METADATA_STORE_KEY_V2", "GROUP_METADATA_STORE_KEY_V3", + "UNSET", "ArrayMetadataModelV2", "ArrayMetadataModelV2Partial", "ArrayMetadataModelV3", @@ -96,6 +98,7 @@ "MetadataValidationError", "NamedConfigModelV3", "ProblemKind", + "UnsetType", "ValidationProblem", "is_array_metadata_v2", "is_array_metadata_v3", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 6d1d2c3d6b..604c9a7d1e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -10,6 +10,7 @@ from typing_extensions import TypedDict, Unpack +from zarr_metadata.model._sentinel import UNSET, UnsetType from zarr_metadata.model._validation import ( ARRAY_METADATA_STANDARD_KEYS_V3, MetadataValidationError, @@ -133,7 +134,7 @@ class ArrayMetadataModelV3Partial(TypedDict, total=False): chunk_grid: MetadataFieldModelV3 codecs: tuple[MetadataFieldModelV3, ...] chunk_key_encoding: MetadataFieldModelV3 - dimension_names: tuple[str | None, ...] | None + dimension_names: tuple[str | None, ...] | UnsetType attributes: dict[str, JSONValue] storage_transformers: tuple[MetadataFieldModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @@ -159,7 +160,7 @@ class ArrayMetadataModelV3: chunk_grid: MetadataFieldModelV3 codecs: tuple[MetadataFieldModelV3, ...] chunk_key_encoding: MetadataFieldModelV3 - dimension_names: tuple[str | None, ...] | None + dimension_names: tuple[str | None, ...] | UnsetType attributes: dict[str, JSONValue] storage_transformers: tuple[MetadataFieldModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @@ -196,7 +197,7 @@ def create_default( chunk_grid=NamedConfigModelV3(name="regular", configuration={"chunk_shape": ()}), codecs=(NamedConfigModelV3(name="bytes", configuration={}),), chunk_key_encoding=NamedConfigModelV3(name="default", configuration={}), - dimension_names=None, + dimension_names=UNSET, attributes={}, storage_transformers=(), extra_fields={}, @@ -246,7 +247,7 @@ def to_json(self) -> ArrayMetadataV3: "codecs": tuple(codec.to_json() for codec in self.codecs), "chunk_key_encoding": self.chunk_key_encoding.to_json(), } - if self.dimension_names is not None: + if self.dimension_names is not UNSET: out["dimension_names"] = self.dimension_names if len(self.attributes) > 0: out["attributes"] = self.attributes @@ -279,7 +280,7 @@ def from_json(cls, data: object) -> ArrayMetadataModelV3: chunk_grid=NamedConfigModelV3.from_json(parsed["chunk_grid"]), codecs=tuple(NamedConfigModelV3.from_json(c) for c in parsed["codecs"]), chunk_key_encoding=NamedConfigModelV3.from_json(parsed["chunk_key_encoding"]), - dimension_names=parsed.get("dimension_names"), + dimension_names=parsed.get("dimension_names", UNSET), attributes=dict(parsed.get("attributes", {})), storage_transformers=tuple( NamedConfigModelV3.from_json(t) for t in parsed.get("storage_transformers", ()) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 0980096a8b..29f32bd330 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -15,6 +15,7 @@ ArrayMetadataModelV3, must_understand_subset, ) +from zarr_metadata.model._sentinel import UNSET, UnsetType from zarr_metadata.model._validation import ( GROUP_METADATA_STANDARD_KEYS_V3, MetadataValidationError, @@ -63,7 +64,7 @@ class GroupMetadataModelV3Partial(TypedDict, total=False): """ attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | None + consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType extra_fields: dict[str, ExtensionFieldV3] @@ -80,7 +81,7 @@ class GroupMetadataModelV3: zarr_format: Literal[3] = field(default=3, init=False) node_type: Literal["group"] = field(default="group", init=False) attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | None + consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType extra_fields: dict[str, ExtensionFieldV3] def __post_init__(self) -> None: @@ -107,7 +108,7 @@ def create_default( analog of `list()` returning `[]`. Any field can be overridden by keyword (the same fields accepted by `update`). """ - default = cls(attributes={}, consolidated_metadata=None, extra_fields={}) + default = cls(attributes={}, consolidated_metadata=UNSET, extra_fields={}) return default.update(**overrides) def update(self, **kwargs: Unpack[GroupMetadataModelV3Partial]) -> GroupMetadataModelV3: @@ -128,7 +129,7 @@ def to_json(self) -> GroupMetadataV3: } if len(self.attributes) > 0: out["attributes"] = self.attributes - if self.consolidated_metadata is not None: + if self.consolidated_metadata is not UNSET: # The consolidated-metadata shape ({kind, must_understand, metadata}, # no `name`) predates the strict v3.1 extension-field rules, so it is # not assignable to `ExtensionFieldV3`; see the discussion on @@ -143,10 +144,10 @@ def to_json(self) -> GroupMetadataV3: @classmethod def from_json(cls, data: object) -> GroupMetadataModelV3: parsed = parse_group_metadata_v3(arrays_to_tuples(data)) - consolidated_raw = parsed.get(CONSOLIDATED_METADATA_KEY_V3) + consolidated_raw: object = parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET) consolidated = ( - None - if consolidated_raw is None + UNSET + if consolidated_raw is UNSET else ConsolidatedMetadataModelV3.from_json(consolidated_raw) ) # Sound cast: the TypedDict types all non-standard keys as its diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py new file mode 100644 index 0000000000..b0cab883ca --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -0,0 +1,30 @@ +"""The absence sentinel for optional metadata-document keys. + +The models observe one invariant: `None` in a model always corresponds to a +JSON `null` in the document (a v2 `compressor`/`filters` value, an unnamed +dimension inside `dimension_names`), and `UNSET` always means the document +key is absent. The two are never interchangeable: for keys where the spec +gives `null` no meaning (`dimension_names` itself, `consolidated_metadata`), +a model `None` would invite serializing an invalid `key: null` spelling, +while `UNSET` cannot leak into a document at all. + +Check with identity: `if model.dimension_names is UNSET: ...`. +""" + +from __future__ import annotations + +from enum import Enum +from typing import Final, Literal + + +class UnsetType(Enum): + """The type of `UNSET`; use in annotations as `T | UnsetType`.""" + + UNSET = "UNSET" + + def __repr__(self) -> str: + return "UNSET" + + +UNSET: Final[Literal[UnsetType.UNSET]] = UnsetType.UNSET +"""Marks a metadata-document key as absent. Test with `is UNSET`.""" diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 4805fc2674..d9af8aeff6 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -13,6 +13,7 @@ ARRAY_METADATA_OPTIONAL_KEYS_V3, ARRAY_METADATA_REQUIRED_KEYS_V3, ARRAY_METADATA_STANDARD_KEYS_V3, + UNSET, ArrayMetadataModelV2, ArrayMetadataModelV2Partial, ArrayMetadataModelV3, @@ -196,8 +197,8 @@ def test_v3_dimension_names_included_when_present() -> None: def test_v3_dimension_names_omitted_when_none() -> None: - """V3 to_json omits dimension_names when they are None.""" - out = ArrayMetadataModelV3.create_default(dimension_names=None).to_json() + """V3 to_json omits dimension_names when they are UNSET.""" + out = ArrayMetadataModelV3.create_default(dimension_names=UNSET).to_json() assert "dimension_names" not in out @@ -213,7 +214,7 @@ def test_v3_attributes_included_when_dimension_names_is_none() -> None: """ out: dict[str, object] = dict( ArrayMetadataModelV3.create_default( - dimension_names=None, attributes={"foo": "bar"} + dimension_names=UNSET, attributes={"foo": "bar"} ).to_json() ) assert out["attributes"] == {"foo": "bar"} @@ -482,13 +483,13 @@ def test_v3_from_json_reconstructs_required_fields() -> None: def test_v3_from_json_defaults_for_omitted_optionals() -> None: """V3 from_json supplies defaults for omitted optional fields.""" doc = ArrayMetadataModelV3.create_default( - attributes={}, storage_transformers=(), dimension_names=None + attributes={}, storage_transformers=(), dimension_names=UNSET ).to_json() # to_json omits these entirely; from_json must restore defaults model = ArrayMetadataModelV3.from_json(doc) assert model.attributes == {} assert model.storage_transformers == () - assert model.dimension_names is None + assert model.dimension_names is UNSET def test_v3_from_json_routes_unknown_keys_to_extra_fields() -> None: @@ -570,7 +571,7 @@ def test_from_key_value_missing_key_raises( ArrayMetadataModelV3, ArrayMetadataModelV3.create_default( attributes={}, - dimension_names=None, + dimension_names=UNSET, storage_transformers=(), extra_fields={}, ), @@ -1259,7 +1260,8 @@ def test_dimension_names_null_field_rejected() -> None: assert [(p.loc, p.kind) for p in problems] == [(("dimension_names",), "invalid_type")] # and the model's own None spelling correctly maps to key absence assert ( - "dimension_names" not in ArrayMetadataModelV3.create_default(dimension_names=None).to_json() + "dimension_names" + not in ArrayMetadataModelV3.create_default(dimension_names=UNSET).to_json() ) @@ -1353,3 +1355,23 @@ def test_v2_null_dimension_separator_rejected() -> None: doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dimension_separator": None} problems = validate_array_metadata_v2(doc) assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] + + +def test_dimension_names_absent_and_all_null_are_distinct() -> None: + """An absent dimension_names field and an explicit all-null one are + semantically different documents: the explicit form says every dimension + has a name, which is null; absence says there are no dimension names. + The model preserves the distinction (UNSET vs a tuple of Nones), and both + spellings round-trip faithfully.""" + absent_doc = dict(ArrayMetadataModelV3.create_default(shape=(2, 3)).to_json()) + explicit_doc = absent_doc | {"dimension_names": (None, None)} + + absent = ArrayMetadataModelV3.from_json(absent_doc) + explicit = ArrayMetadataModelV3.from_json(explicit_doc) + + assert absent.dimension_names is UNSET + assert explicit.dimension_names == (None, None) + assert absent != explicit + assert "dimension_names" not in absent.to_json() + assert absent.to_json() == absent_doc + assert explicit.to_json() == explicit_doc diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index d2399220be..fcf57bddde 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -5,6 +5,7 @@ import pytest +from zarr_metadata.model import UNSET from zarr_metadata.model._array import ArrayMetadataModelV3 from zarr_metadata.model._group import ( ConsolidatedMetadataModelV2, @@ -62,7 +63,7 @@ def test_group_v3_extra_fields_overlap_rejected() -> None: with pytest.raises(ValueError, match="Extra fields"): GroupMetadataModelV3( attributes={}, - consolidated_metadata=None, + consolidated_metadata=UNSET, extra_fields={"node_type": {"name": "x", "must_understand": False}}, ) @@ -72,7 +73,7 @@ def test_group_v3_consolidated_extra_field_rejected() -> None: with pytest.raises(ValueError, match="Extra fields"): GroupMetadataModelV3( attributes={}, - consolidated_metadata=None, + consolidated_metadata=UNSET, extra_fields={"consolidated_metadata": {"name": "x", "must_understand": False}}, ) diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index c1cd3010d4..65db4ae419 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -149,6 +149,7 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: cross-field checks. This test documents why the delegation pattern above is the recommended integration.""" from zarr_metadata._common import JSONValue + from zarr_metadata.model import UnsetType from zarr_metadata.v3._common import MetadataV3 from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 @@ -162,6 +163,7 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: "ArrayMetadataV3": ArrayMetadataV3, "NamedConfigModelV3": NamedConfigModelV3, "MetadataFieldModelV3": NamedConfigModelV3, + "UnsetType": UnsetType, }, ) model_shaped = { @@ -171,7 +173,7 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}}, "codecs": [{"name": "bytes", "configuration": {}}], "chunk_key_encoding": {"name": "default", "configuration": {}}, - "dimension_names": None, + "dimension_names": UnsetType.UNSET, "attributes": {}, "storage_transformers": [], "extra_fields": {}, diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index fb2f574a4f..16a540b3f9 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -55,6 +55,8 @@ def _group_rank(s: str) -> int: "ValidationProblem", "MetadataValidationError", "ProblemKind", + "UNSET", + "UnsetType", # v2 data-type encoding union "DataTypeMetadataV2", # Category B — codec canonical unions From 672d2bf016fdc39adbf8f23a3fadc3e40daa68ce Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:05:38 +0200 Subject: [PATCH 26/70] fix(zarr-metadata): accept and preserve the wild consolidated_metadata null Historical zarr-python versions wrote "consolidated_metadata": null into group documents for groups without consolidated metadata, so real stores contain the spelling; the validator was rejecting those documents ("expected a mapping"). Per the None/UNSET invariant, the field is now honestly three-state: UNSET (key absent), None (the document's literal null, preserved on round-trip), or a ConsolidatedMetadataModelV3. Interpreting null as absence is the consumer's call, not a document rewrite by this layer. Also records an implementation constraint on the sentinel itself: typing_extensions.Sentinel (PEP 661) is the intended spelling, but pyright 1.1.411 degrades a Sentinel to Unknown in dataclass FIELD annotations (function signatures work), verified by probe both with and without enableExperimentalFeatures. Using it would reintroduce suppressions at every use site under the strict gate, so UNSET stays a single-member enum, with the Sentinel switch documented in _sentinel.py for when pyright catches up. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 5 ++- .../src/zarr_metadata/model/_group.py | 32 +++++++++++++------ .../src/zarr_metadata/model/_sentinel.py | 16 +++++++--- .../src/zarr_metadata/model/_validation.py | 4 ++- .../zarr-metadata/tests/model/test_group.py | 18 +++++++++++ .../tests/model/test_pydantic.py | 4 +-- 6 files changed, 60 insertions(+), 19 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 29c893b3c1..4cb35ad9cf 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -53,4 +53,7 @@ Optional document keys use the `UNSET` sentinel, never `None`: in a model, always means the key is absent. This keeps semantically distinct spellings distinct — an absent `dimension_names` ("there are no dimension names") and an explicit `[null, null]` ("every dimension has a name, which is null") are -different documents and round-trip as such. +different documents and round-trip as such. A group's +`consolidated_metadata` is accordingly three-state: absent (`UNSET`), +present, or the literal `null` written by historical zarr-python versions, +which is accepted and preserved rather than rejected or silently rewritten. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 29f32bd330..826492e26e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -64,7 +64,7 @@ class GroupMetadataModelV3Partial(TypedDict, total=False): """ attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType + consolidated_metadata: ConsolidatedMetadataModelV3 | None | UnsetType extra_fields: dict[str, ExtensionFieldV3] @@ -81,7 +81,7 @@ class GroupMetadataModelV3: zarr_format: Literal[3] = field(default=3, init=False) node_type: Literal["group"] = field(default="group", init=False) attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType + consolidated_metadata: ConsolidatedMetadataModelV3 | None | UnsetType extra_fields: dict[str, ExtensionFieldV3] def __post_init__(self) -> None: @@ -133,9 +133,14 @@ def to_json(self) -> GroupMetadataV3: # The consolidated-metadata shape ({kind, must_understand, metadata}, # no `name`) predates the strict v3.1 extension-field rules, so it is # not assignable to `ExtensionFieldV3`; see the discussion on - # `zarr_metadata.v3.consolidated`. + # `zarr_metadata.v3.consolidated`. A model None is the document's + # literal null spelling (written by historical zarr-python versions) + # and round-trips as such. out[CONSOLIDATED_METADATA_KEY_V3] = cast( - "ExtensionFieldV3", self.consolidated_metadata.to_json() + "ExtensionFieldV3", + None + if self.consolidated_metadata is None + else self.consolidated_metadata.to_json(), ) for key, value in self.extra_fields.items(): out[key] = value @@ -144,12 +149,19 @@ def to_json(self) -> GroupMetadataV3: @classmethod def from_json(cls, data: object) -> GroupMetadataModelV3: parsed = parse_group_metadata_v3(arrays_to_tuples(data)) - consolidated_raw: object = parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET) - consolidated = ( - UNSET - if consolidated_raw is UNSET - else ConsolidatedMetadataModelV3.from_json(consolidated_raw) - ) + # Cast to object: the TypedDict's extra_items type does not admit null, + # but wild documents (historical zarr-python) contain it. + consolidated_raw = cast("object", parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET)) + consolidated: ConsolidatedMetadataModelV3 | None | UnsetType + if consolidated_raw is UNSET: + consolidated = UNSET + elif consolidated_raw is None: + # Historical zarr-python versions wrote consolidated_metadata: null + # for groups without consolidated metadata; the spelling is + # preserved (interpreting it as absence is the consumer's call). + consolidated = None + else: + consolidated = ConsolidatedMetadataModelV3.from_json(consolidated_raw) # Sound cast: the TypedDict types all non-standard keys as its # `extra_items` (`ExtensionFieldV3`); the comprehension's inferred value # type is the union over ALL keys because the key filter cannot narrow it. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index b0cab883ca..29eaa104ab 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -2,13 +2,19 @@ The models observe one invariant: `None` in a model always corresponds to a JSON `null` in the document (a v2 `compressor`/`filters` value, an unnamed -dimension inside `dimension_names`), and `UNSET` always means the document -key is absent. The two are never interchangeable: for keys where the spec -gives `null` no meaning (`dimension_names` itself, `consolidated_metadata`), -a model `None` would invite serializing an invalid `key: null` spelling, -while `UNSET` cannot leak into a document at all. +dimension inside `dimension_names`, a group's historical +`consolidated_metadata: null`), and `UNSET` always means the document key is +absent. The two are never interchangeable, so a model value can never leak +into a document as a spelling the writer did not intend. Check with identity: `if model.dimension_names is UNSET: ...`. + +Implementation note: `typing_extensions.Sentinel` (PEP 661) is the intended +spelling, but pyright (1.1.411) degrades a Sentinel to `Unknown` in dataclass +FIELD annotations (function signatures work), which would force suppressions +under this package's strict gate at every use site. The single-member enum +gives the same identity semantics with exact `Literal` narrowing; switch to +`Sentinel` once pyright supports it in dataclass fields. """ from __future__ import annotations diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index fc781c7899..4558b57ef4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -518,7 +518,9 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: problems.extend(_check_literal(doc, "node_type", "group")) if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) - if "consolidated_metadata" in doc: + if "consolidated_metadata" in doc and doc["consolidated_metadata"] is not None: + # consolidated_metadata: null is accepted: historical zarr-python + # versions wrote it for groups without consolidated metadata. problems.extend( _prefix( "consolidated_metadata", diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index fcf57bddde..4286976962 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -351,3 +351,21 @@ def test_group_must_understand_fields_partition() -> None: } ) assert set(model.must_understand_fields) == {"implicit"} + + +def test_group_v3_null_consolidated_metadata_preserved() -> None: + """Historical zarr-python versions wrote consolidated_metadata: null for + groups without consolidated metadata, so null is a real wild spelling: + it is accepted and preserved on round-trip (model None = document null), + distinct from key absence (UNSET). Interpreting null as "no consolidated + metadata" is the consumer's call, not a document rewrite.""" + null_doc = {"zarr_format": 3, "node_type": "group", "consolidated_metadata": None} + absent_doc = {"zarr_format": 3, "node_type": "group"} + assert validate_group_metadata_v3(null_doc) == [] + null_model = GroupMetadataModelV3.from_json(null_doc) + absent_model = GroupMetadataModelV3.from_json(absent_doc) + assert null_model.consolidated_metadata is None + assert absent_model.consolidated_metadata is UNSET + assert null_model != absent_model + assert null_model.to_json() == null_doc + assert absent_model.to_json() == absent_doc diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index 65db4ae419..c10d1cb471 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -149,7 +149,7 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: cross-field checks. This test documents why the delegation pattern above is the recommended integration.""" from zarr_metadata._common import JSONValue - from zarr_metadata.model import UnsetType + from zarr_metadata.model import UNSET, UnsetType from zarr_metadata.v3._common import MetadataV3 from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 @@ -173,7 +173,7 @@ def test_native_dataclass_introspection_is_possible_but_diverges() -> None: "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}}, "codecs": [{"name": "bytes", "configuration": {}}], "chunk_key_encoding": {"name": "default", "configuration": {}}, - "dimension_names": UnsetType.UNSET, + "dimension_names": UNSET, "attributes": {}, "storage_transformers": [], "extra_fields": {}, From fa7b7cf3660afa02e16b633a6505d10c1b4dac91 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:10:31 +0200 Subject: [PATCH 27/70] fix(zarr-metadata): repair consolidated_metadata null to absence, not preserve it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit d-v-b: the bugged spelling should not be preserved or honored. The three-state field reverts to two states (model | UNSET): a document carrying "consolidated_metadata": null — written by a historical zarr-python bug — remains readable (the validator accepts it so real stores open), but the spelling gets no model representation: it is read as absence and never written back. This is the one deliberate exception to faithful round-tripping, pinned as such: from_json(null_doc) equals from_json(absent_doc), and to_json omits the key. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 8 +++--- .../src/zarr_metadata/model/_group.py | 25 +++++++------------ .../src/zarr_metadata/model/_sentinel.py | 5 ++-- .../src/zarr_metadata/model/_validation.py | 5 ++-- .../zarr-metadata/tests/model/test_group.py | 23 +++++++---------- 5 files changed, 27 insertions(+), 39 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 4cb35ad9cf..c4efbe6f01 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -53,7 +53,7 @@ Optional document keys use the `UNSET` sentinel, never `None`: in a model, always means the key is absent. This keeps semantically distinct spellings distinct — an absent `dimension_names` ("there are no dimension names") and an explicit `[null, null]` ("every dimension has a name, which is null") are -different documents and round-trip as such. A group's -`consolidated_metadata` is accordingly three-state: absent (`UNSET`), -present, or the literal `null` written by historical zarr-python versions, -which is accepted and preserved rather than rejected or silently rewritten. +different documents and round-trip as such. The `consolidated_metadata: null` +written by a historical zarr-python bug is the one deliberate exception to +faithful round-tripping: those stores remain readable, but the bug spelling +is repaired to absence on read and never written back. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 826492e26e..5c2c2287fc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -64,7 +64,7 @@ class GroupMetadataModelV3Partial(TypedDict, total=False): """ attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | None | UnsetType + consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType extra_fields: dict[str, ExtensionFieldV3] @@ -81,7 +81,7 @@ class GroupMetadataModelV3: zarr_format: Literal[3] = field(default=3, init=False) node_type: Literal["group"] = field(default="group", init=False) attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | None | UnsetType + consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType extra_fields: dict[str, ExtensionFieldV3] def __post_init__(self) -> None: @@ -133,14 +133,9 @@ def to_json(self) -> GroupMetadataV3: # The consolidated-metadata shape ({kind, must_understand, metadata}, # no `name`) predates the strict v3.1 extension-field rules, so it is # not assignable to `ExtensionFieldV3`; see the discussion on - # `zarr_metadata.v3.consolidated`. A model None is the document's - # literal null spelling (written by historical zarr-python versions) - # and round-trips as such. + # `zarr_metadata.v3.consolidated`. out[CONSOLIDATED_METADATA_KEY_V3] = cast( - "ExtensionFieldV3", - None - if self.consolidated_metadata is None - else self.consolidated_metadata.to_json(), + "ExtensionFieldV3", self.consolidated_metadata.to_json() ) for key, value in self.extra_fields.items(): out[key] = value @@ -152,14 +147,12 @@ def from_json(cls, data: object) -> GroupMetadataModelV3: # Cast to object: the TypedDict's extra_items type does not admit null, # but wild documents (historical zarr-python) contain it. consolidated_raw = cast("object", parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET)) - consolidated: ConsolidatedMetadataModelV3 | None | UnsetType - if consolidated_raw is UNSET: + consolidated: ConsolidatedMetadataModelV3 | UnsetType + if consolidated_raw is UNSET or consolidated_raw is None: + # consolidated_metadata: null was written by a historical + # zarr-python bug; it gets no model representation. It is read as + # absence and never written back — repaired, not preserved. consolidated = UNSET - elif consolidated_raw is None: - # Historical zarr-python versions wrote consolidated_metadata: null - # for groups without consolidated metadata; the spelling is - # preserved (interpreting it as absence is the consumer's call). - consolidated = None else: consolidated = ConsolidatedMetadataModelV3.from_json(consolidated_raw) # Sound cast: the TypedDict types all non-standard keys as its diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index 29eaa104ab..74f5feec64 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -2,9 +2,8 @@ The models observe one invariant: `None` in a model always corresponds to a JSON `null` in the document (a v2 `compressor`/`filters` value, an unnamed -dimension inside `dimension_names`, a group's historical -`consolidated_metadata: null`), and `UNSET` always means the document key is -absent. The two are never interchangeable, so a model value can never leak +dimension inside `dimension_names`), and `UNSET` always means the document +key is absent. The two are never interchangeable, so a model value can never leak into a document as a spelling the writer did not intend. Check with identity: `if model.dimension_names is UNSET: ...`. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 4558b57ef4..fae09187a4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -519,8 +519,9 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) if "consolidated_metadata" in doc and doc["consolidated_metadata"] is not None: - # consolidated_metadata: null is accepted: historical zarr-python - # versions wrote it for groups without consolidated metadata. + # consolidated_metadata: null (a historical zarr-python bug) is + # structurally accepted so those stores remain readable, but the model + # repairs it to absence on read and never writes it back. problems.extend( _prefix( "consolidated_metadata", diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 4286976962..0062bf7620 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -353,19 +353,14 @@ def test_group_must_understand_fields_partition() -> None: assert set(model.must_understand_fields) == {"implicit"} -def test_group_v3_null_consolidated_metadata_preserved() -> None: - """Historical zarr-python versions wrote consolidated_metadata: null for - groups without consolidated metadata, so null is a real wild spelling: - it is accepted and preserved on round-trip (model None = document null), - distinct from key absence (UNSET). Interpreting null as "no consolidated - metadata" is the consumer's call, not a document rewrite.""" +def test_group_v3_null_consolidated_metadata_repaired_to_absence() -> None: + """consolidated_metadata: null was written by a historical zarr-python bug. + Those stores must remain readable, but the bug spelling is not honored: + it is read as absence (UNSET) and never written back — the round-trip + deliberately repairs the document rather than preserving the bug.""" null_doc = {"zarr_format": 3, "node_type": "group", "consolidated_metadata": None} - absent_doc = {"zarr_format": 3, "node_type": "group"} assert validate_group_metadata_v3(null_doc) == [] - null_model = GroupMetadataModelV3.from_json(null_doc) - absent_model = GroupMetadataModelV3.from_json(absent_doc) - assert null_model.consolidated_metadata is None - assert absent_model.consolidated_metadata is UNSET - assert null_model != absent_model - assert null_model.to_json() == null_doc - assert absent_model.to_json() == absent_doc + model = GroupMetadataModelV3.from_json(null_doc) + assert model.consolidated_metadata is UNSET + assert "consolidated_metadata" not in model.to_json() + assert model == GroupMetadataModelV3.from_json({"zarr_format": 3, "node_type": "group"}) From da3094deadf0d3dd88189b00d70eaace19a80e2c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:14:45 +0200 Subject: [PATCH 28/70] docs(zarr-metadata): pin the Sentinel blocker to pyright regression #11115 Investigated: the Unknown-degradation of typing_extensions.Sentinel is a confirmed upstream pyright regression, not by-design. Introduced in 1.1.405 (verified: 1.1.404 is clean on the same probe, 1.1.411 fails), affects reads of any class-body attribute annotation (dataclass or plain class), does not affect function signatures or module variables, and Final on the sentinel does not help. Tracked as microsoft/pyright#11115 (open, bug+regression); #11467 closed as its duplicate. The enum sentinel stays until the fix lands. Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/_sentinel.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index 74f5feec64..e83d49852a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -9,11 +9,13 @@ Check with identity: `if model.dimension_names is UNSET: ...`. Implementation note: `typing_extensions.Sentinel` (PEP 661) is the intended -spelling, but pyright (1.1.411) degrades a Sentinel to `Unknown` in dataclass -FIELD annotations (function signatures work), which would force suppressions -under this package's strict gate at every use site. The single-member enum -gives the same identity semantics with exact `Literal` narrowing; switch to -`Sentinel` once pyright supports it in dataclass fields. +spelling, but a confirmed pyright regression (1.1.405 through at least +1.1.411; worked in <= 1.1.404) degrades a Sentinel to `Unknown` when read +from any class-body attribute annotation — dataclass or not; function +signatures and module variables are unaffected, and `Final` on the sentinel +does not help. Tracked as https://github.com/microsoft/pyright/issues/11115. +The single-member enum gives the same identity semantics with exact +`Literal` narrowing; switch to `Sentinel` once that regression is fixed. """ from __future__ import annotations From a9edd1faf9f9f0e801ea1faacdc659335904608b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:17:38 +0200 Subject: [PATCH 29/70] docs(zarr-metadata): sentinel switch is blocked by mypy too, not just pyright MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pinning a working pyright (<= 1.1.404) in CI was considered and does not suffice: the pin controls one of four checker surfaces. Contributor IDEs (Pylance bundles current pyright) and downstream consumers' pyright read the py.typed inline annotations with their own versions, and decisively, mypy 2.1.0 has no PEP 661 support at all — a sentinel in type position is a hard [valid-type] error, which would degrade these fields to Any for mypy consumers, including zarr-python itself. The enum is currently the only spelling with exact types on every surface; switch when pyright#11115 is fixed AND mypy implements PEP 661. Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/_sentinel.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index e83d49852a..2b77b44b95 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -9,13 +9,17 @@ Check with identity: `if model.dimension_names is UNSET: ...`. Implementation note: `typing_extensions.Sentinel` (PEP 661) is the intended -spelling, but a confirmed pyright regression (1.1.405 through at least -1.1.411; worked in <= 1.1.404) degrades a Sentinel to `Unknown` when read -from any class-body attribute annotation — dataclass or not; function -signatures and module variables are unaffected, and `Final` on the sentinel -does not help. Tracked as https://github.com/microsoft/pyright/issues/11115. -The single-member enum gives the same identity semantics with exact -`Literal` narrowing; switch to `Sentinel` once that regression is fixed. +spelling, but two independent checker gaps block it. Pyright: a confirmed +regression (1.1.405 through at least 1.1.411; worked in <= 1.1.404, +https://github.com/microsoft/pyright/issues/11115) degrades a Sentinel to +`Unknown` when read from any class-body attribute annotation. Mypy (2.1.0): +no PEP 661 support at all — a sentinel in type position is a hard +`[valid-type]` error, so downstream mypy users (zarr-python itself) would +see these fields as `Any`. Pinning a working pyright in this package's CI +would fix neither contributors' IDEs nor downstream checkers reading the +py.typed annotations. The single-member enum gives the same identity +semantics with exact `Literal` narrowing on every checker; switch to +`Sentinel` once the pyright regression is fixed AND mypy implements PEP 661. """ from __future__ import annotations From 934095e8bf77fc0f7b4c87d4de0a66405ec19432 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:20:01 +0200 Subject: [PATCH 30/70] docs(zarr-metadata): PEP 661 is Final (Python 3.15), not a draft Corrects the sentinel implementation note: PEP 661 was accepted 2026-04-23 and ships as stdlib sentinel in Python 3.15. The two checker gaps blocking the Sentinel spelling (pyright regression #11115, mypy not yet implementing the PEP) are therefore temporary gaps against a Final standard, and the enum is a stopgap with a defined end state. Assisted-by: ClaudeCode:claude-fable-5 --- .../src/zarr_metadata/model/_sentinel.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index 2b77b44b95..5fa80004d2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -8,18 +8,20 @@ Check with identity: `if model.dimension_names is UNSET: ...`. -Implementation note: `typing_extensions.Sentinel` (PEP 661) is the intended -spelling, but two independent checker gaps block it. Pyright: a confirmed -regression (1.1.405 through at least 1.1.411; worked in <= 1.1.404, +Implementation note: `typing_extensions.Sentinel` (PEP 661, Final as of +2026-04-23, stdlib in Python 3.15) is the intended spelling, but two +independent checker gaps block it for now. Pyright: a confirmed regression +(1.1.405 through at least 1.1.411; worked in <= 1.1.404, https://github.com/microsoft/pyright/issues/11115) degrades a Sentinel to `Unknown` when read from any class-body attribute annotation. Mypy (2.1.0): -no PEP 661 support at all — a sentinel in type position is a hard +has not yet implemented PEP 661 — a sentinel in type position is a hard `[valid-type]` error, so downstream mypy users (zarr-python itself) would see these fields as `Any`. Pinning a working pyright in this package's CI would fix neither contributors' IDEs nor downstream checkers reading the py.typed annotations. The single-member enum gives the same identity semantics with exact `Literal` narrowing on every checker; switch to -`Sentinel` once the pyright regression is fixed AND mypy implements PEP 661. +`Sentinel` once the pyright regression is fixed and mypy support lands — +both expected, now that the PEP is Final. """ from __future__ import annotations From 800546aed3db4129d2bee2843859f5ee91127cf5 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:25:46 +0200 Subject: [PATCH 31/70] docs(zarr-metadata): ty fully supports typed sentinels; pyright/mypy are the laggards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ty 0.0.56 types the Sentinel spelling perfectly in dataclass fields: exact T | UNSET unions, both-direction is/is-not narrowing, and wrong-typed constructor arguments rejected (verified with reveal_type, so it is real inference, not silent Any). The checker matrix for sentinel-in-type-position is therefore ty full / pyright regressed (#11115) / mypy not implemented — recorded so the switch decision has current calibration. Assisted-by: ClaudeCode:claude-fable-5 --- .../zarr-metadata/src/zarr_metadata/model/_sentinel.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index 5fa80004d2..749e353c8a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -18,10 +18,12 @@ `[valid-type]` error, so downstream mypy users (zarr-python itself) would see these fields as `Any`. Pinning a working pyright in this package's CI would fix neither contributors' IDEs nor downstream checkers reading the -py.typed annotations. The single-member enum gives the same identity -semantics with exact `Literal` narrowing on every checker; switch to -`Sentinel` once the pyright regression is fixed and mypy support lands — -both expected, now that the PEP is Final. +py.typed annotations. For calibration: ty (0.0.56) +already types the Sentinel spelling perfectly — exact unions and +`is`/`is not` narrowing in dataclass fields — so the standard is landing; +pyright and mypy are the laggards. The single-member enum gives the same +identity semantics with exact `Literal` narrowing on every checker; switch +to `Sentinel` once the pyright regression is fixed and mypy support lands. """ from __future__ import annotations From f1fba4123cf50c404af13daaeafa86af801fbf06 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:34:32 +0200 Subject: [PATCH 32/70] feat(zarr-metadata): adopt the PEP 661 sentinel for UNSET MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit d-v-b's call: PEP 661 is Final, ty already types the sentinel spelling exactly, mypy support is in review (python/mypy#21647) and treated as imminent, and pyright has a known-good version — so use the standard sentinel today rather than carrying the enum stopgap. - UNSET is now typing_extensions.Sentinel("UNSET"), used directly in type expressions (tuple[str | None, ...] | UNSET); the UnsetType companion enum is gone from the API. - typing_extensions floor bumped to 4.14 (where Sentinel arrived). - CI pins pyright==1.1.404, the last version before the class-attribute sentinel regression (microsoft/pyright#11115); pyproject documents the same pin for local runs. 0 errors on the pin; ty checks the sentinel fields clean (its 2 remaining diagnostics are its incomplete PEP 728 extra_items write support, unrelated). - Known short-term cost, accepted deliberately: mypy-checked consumers need cast/type-ignore at narrowing sites until mypy#21647 merges, and contributors' Pylance may show phantom Unknowns until the pyright fix ships. Recorded in _sentinel.py and the changelog. - The pydantic native-introspection test reverts to documenting that introspection is unsupported (pydantic 2.13 cannot schema a Sentinel); the delegation patterns are unaffected. Assisted-by: ClaudeCode:claude-fable-5 --- .github/workflows/zarr-metadata.yml | 5 +- packages/zarr-metadata/changes/210.feature.md | 12 ++- packages/zarr-metadata/pyproject.toml | 5 +- .../src/zarr_metadata/__init__.py | 2 - .../src/zarr_metadata/model/__init__.py | 3 +- .../src/zarr_metadata/model/_array.py | 6 +- .../src/zarr_metadata/model/_group.py | 8 +- .../src/zarr_metadata/model/_sentinel.py | 48 ++++------- .../tests/model/test_pydantic.py | 79 ++++++++----------- .../zarr-metadata/tests/test_public_api.py | 1 - 10 files changed, 71 insertions(+), 98 deletions(-) diff --git a/.github/workflows/zarr-metadata.yml b/.github/workflows/zarr-metadata.yml index 95e8251227..e660483220 100644 --- a/.github/workflows/zarr-metadata.yml +++ b/.github/workflows/zarr-metadata.yml @@ -82,7 +82,10 @@ jobs: - name: Sync test dependency group run: uv sync --group test --python 3.11 - name: Run pyright - run: uv run --group test --with pyright pyright src + # Pinned to the last version that types PEP 661 sentinels in class + # attributes correctly; 1.1.405+ regressed (microsoft/pyright#11115). + # Unpin when the fix lands. + run: uv run --group test --with 'pyright==1.1.404' pyright src zarr-metadata-complete: name: zarr-metadata complete diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index c4efbe6f01..12f57c1525 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -47,10 +47,14 @@ convention's default `"."` (the model previously normalized absence to `"/"`, which would misaddress the chunks of real-world default-separator arrays). The value is never null: absent, `"."`, or `"/"` are the only spellings. -Optional document keys use the `UNSET` sentinel, never `None`: in a model, -`None` always corresponds to a JSON `null` in the document (a v2 -`compressor`, an unnamed dimension inside `dimension_names`), and `UNSET` -always means the key is absent. This keeps semantically distinct spellings +Optional document keys use `UNSET` — a PEP 661 sentinel +(`typing_extensions.Sentinel`), usable directly in type expressions — never +`None`: in a model, `None` always corresponds to a JSON `null` in the +document (a v2 `compressor`, an unnamed dimension inside `dimension_names`), +and `UNSET` always means the key is absent. Checker note: ty types the +sentinel exactly; pyright needs `<= 1.1.404` until microsoft/pyright#11115 +is fixed (this package's CI pins it); mypy users need a `cast` or +`type: ignore` at narrowing sites until python/mypy#21647 merges. This keeps semantically distinct spellings distinct — an absent `dimension_names` ("there are no dimension names") and an explicit `[null, null]` ("every dimension has a name, which is null") are different documents and round-trip as such. The `consolidated_metadata: null` diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 05667d59e3..87f35db1db 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -32,7 +32,7 @@ classifiers = [ ] keywords = ["zarr"] dependencies = [ - "typing_extensions>=4.13", + "typing_extensions>=4.14", ] [project.urls] @@ -82,6 +82,9 @@ checks = [ "PR06", ] +# CI pins pyright==1.1.404: later versions regress PEP 661 sentinel typing in +# class attributes (microsoft/pyright#11115), which zarr_metadata.model._sentinel +# relies on. Use the same pin locally; unpin when the fix lands. [tool.pyright] include = ["src"] enableExperimentalFeatures = true diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index 4e0cfd4e36..4c1328bfe1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -17,7 +17,6 @@ MetadataValidationError, NamedConfigModelV3, ProblemKind, - UnsetType, ValidationProblem, ) from zarr_metadata.v2.array import ( @@ -356,7 +355,6 @@ "Uint32FillValue", "Uint64DataTypeName", "Uint64FillValue", - "UnsetType", "V2ChunkKeyEncodingMetadata", "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingSeparator", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 76367eae32..1257109465 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -34,7 +34,7 @@ GroupMetadataModelV3, GroupMetadataModelV3Partial, ) -from zarr_metadata.model._sentinel import UNSET, UnsetType +from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ( ARRAY_METADATA_OPTIONAL_KEYS_V3, ARRAY_METADATA_REQUIRED_KEYS_V2, @@ -98,7 +98,6 @@ "MetadataValidationError", "NamedConfigModelV3", "ProblemKind", - "UnsetType", "ValidationProblem", "is_array_metadata_v2", "is_array_metadata_v3", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 604c9a7d1e..bfa57ddb34 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -10,7 +10,7 @@ from typing_extensions import TypedDict, Unpack -from zarr_metadata.model._sentinel import UNSET, UnsetType +from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ( ARRAY_METADATA_STANDARD_KEYS_V3, MetadataValidationError, @@ -134,7 +134,7 @@ class ArrayMetadataModelV3Partial(TypedDict, total=False): chunk_grid: MetadataFieldModelV3 codecs: tuple[MetadataFieldModelV3, ...] chunk_key_encoding: MetadataFieldModelV3 - dimension_names: tuple[str | None, ...] | UnsetType + dimension_names: tuple[str | None, ...] | UNSET attributes: dict[str, JSONValue] storage_transformers: tuple[MetadataFieldModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] @@ -160,7 +160,7 @@ class ArrayMetadataModelV3: chunk_grid: MetadataFieldModelV3 codecs: tuple[MetadataFieldModelV3, ...] chunk_key_encoding: MetadataFieldModelV3 - dimension_names: tuple[str | None, ...] | UnsetType + dimension_names: tuple[str | None, ...] | UNSET attributes: dict[str, JSONValue] storage_transformers: tuple[MetadataFieldModelV3, ...] extra_fields: dict[str, ExtensionFieldV3] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 5c2c2287fc..e3a006699b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -15,7 +15,7 @@ ArrayMetadataModelV3, must_understand_subset, ) -from zarr_metadata.model._sentinel import UNSET, UnsetType +from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ( GROUP_METADATA_STANDARD_KEYS_V3, MetadataValidationError, @@ -64,7 +64,7 @@ class GroupMetadataModelV3Partial(TypedDict, total=False): """ attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType + consolidated_metadata: ConsolidatedMetadataModelV3 | UNSET extra_fields: dict[str, ExtensionFieldV3] @@ -81,7 +81,7 @@ class GroupMetadataModelV3: zarr_format: Literal[3] = field(default=3, init=False) node_type: Literal["group"] = field(default="group", init=False) attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType + consolidated_metadata: ConsolidatedMetadataModelV3 | UNSET extra_fields: dict[str, ExtensionFieldV3] def __post_init__(self) -> None: @@ -147,7 +147,7 @@ def from_json(cls, data: object) -> GroupMetadataModelV3: # Cast to object: the TypedDict's extra_items type does not admit null, # but wild documents (historical zarr-python) contain it. consolidated_raw = cast("object", parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET)) - consolidated: ConsolidatedMetadataModelV3 | UnsetType + consolidated: ConsolidatedMetadataModelV3 | UNSET if consolidated_raw is UNSET or consolidated_raw is None: # consolidated_metadata: null was written by a historical # zarr-python bug; it gets no model representation. It is read as diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index 749e353c8a..eeae0da97a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -3,43 +3,27 @@ The models observe one invariant: `None` in a model always corresponds to a JSON `null` in the document (a v2 `compressor`/`filters` value, an unnamed dimension inside `dimension_names`), and `UNSET` always means the document -key is absent. The two are never interchangeable, so a model value can never leak -into a document as a spelling the writer did not intend. +key is absent. The two are never interchangeable, so a model value can never +leak into a document as a spelling the writer did not intend. Check with identity: `if model.dimension_names is UNSET: ...`. -Implementation note: `typing_extensions.Sentinel` (PEP 661, Final as of -2026-04-23, stdlib in Python 3.15) is the intended spelling, but two -independent checker gaps block it for now. Pyright: a confirmed regression -(1.1.405 through at least 1.1.411; worked in <= 1.1.404, -https://github.com/microsoft/pyright/issues/11115) degrades a Sentinel to -`Unknown` when read from any class-body attribute annotation. Mypy (2.1.0): -has not yet implemented PEP 661 — a sentinel in type position is a hard -`[valid-type]` error, so downstream mypy users (zarr-python itself) would -see these fields as `Any`. Pinning a working pyright in this package's CI -would fix neither contributors' IDEs nor downstream checkers reading the -py.typed annotations. For calibration: ty (0.0.56) -already types the Sentinel spelling perfectly — exact unions and -`is`/`is not` narrowing in dataclass fields — so the standard is landing; -pyright and mypy are the laggards. The single-member enum gives the same -identity semantics with exact `Literal` narrowing on every checker; switch -to `Sentinel` once the pyright regression is fixed and mypy support lands. +Checker support (PEP 661 is Final; stdlib `sentinel` arrives in Python +3.15): ty types this spelling exactly, including `is`/`is not` narrowing. +Pyright supports it but a regression (1.1.405+, tracked as +https://github.com/microsoft/pyright/issues/11115) degrades class-attribute +reads to `Unknown`, so this package pins pyright to the last good version +until the fix lands. Mypy support is in review +(https://github.com/python/mypy/pull/21647); until it merges, mypy-checked +consumers of these fields need a `cast` or `type: ignore` at narrowing +sites. This is a deliberate short-term cost: the sentinel is the standard, +and the checkers are converging on it. """ from __future__ import annotations -from enum import Enum -from typing import Final, Literal +from typing_extensions import Sentinel - -class UnsetType(Enum): - """The type of `UNSET`; use in annotations as `T | UnsetType`.""" - - UNSET = "UNSET" - - def __repr__(self) -> str: - return "UNSET" - - -UNSET: Final[Literal[UnsetType.UNSET]] = UnsetType.UNSET -"""Marks a metadata-document key as absent. Test with `is UNSET`.""" +UNSET = Sentinel("UNSET") +"""Marks a metadata-document key as absent (PEP 661 sentinel; usable directly +in type expressions, e.g. `tuple[str, ...] | UNSET`). Test with `is UNSET`.""" diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index c10d1cb471..7e80d52ee8 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -34,6 +34,8 @@ ConfigDict, InstanceOf, PlainSerializer, + PydanticSchemaGenerationError, + PydanticUserError, TypeAdapter, ValidationError, model_validator, @@ -142,59 +144,40 @@ def test_type_adapter_standalone() -> None: # --- the road not taken: native dataclass introspection ---------------------- -def test_native_dataclass_introspection_is_possible_but_diverges() -> None: - """Pydantic CAN introspect the model dataclass after a namespace rebuild, - but that path validates the model shape, not the document: it rejects the - document form, coerces booleans into dimensions, and skips the library's - cross-field checks. This test documents why the delegation pattern above - is the recommended integration.""" +def test_native_dataclass_introspection_is_not_supported() -> None: + """Pydantic cannot field-introspect the model dataclasses: the UNSET + sentinel (PEP 661, typing_extensions.Sentinel) in the optional-field + annotations has no pydantic schema (as of pydantic 2.13), so even the + rebuild-with-namespace recipe fails. Introspection was already the wrong + tool before the sentinel existed — it validated the model shape rather + than the document, and its lax coercion re-opened validator holes (e.g. + shape=[True, -5] coerced to (1, -5)) — so the delegation patterns above + are the only supported integrations. If this test ever fails because + pydantic learned to handle sentinels, revisit whether the introspection + path needs its divergences documented again.""" from zarr_metadata._common import JSONValue - from zarr_metadata.model import UNSET, UnsetType + from zarr_metadata.model import UNSET from zarr_metadata.v3._common import MetadataV3 from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 - adapter = TypeAdapter(ArrayMetadataModelV3) - adapter.rebuild( - force=True, - _types_namespace={ - "JSONValue": JSONValue, - "ExtensionFieldV3": ExtensionFieldV3, - "MetadataV3": MetadataV3, - "ArrayMetadataV3": ArrayMetadataV3, - "NamedConfigModelV3": NamedConfigModelV3, - "MetadataFieldModelV3": NamedConfigModelV3, - "UnsetType": UnsetType, - }, - ) - model_shaped = { - "shape": [10], - "fill_value": 0, - "data_type": {"name": "uint8", "configuration": {}}, - "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}}, - "codecs": [{"name": "bytes", "configuration": {}}], - "chunk_key_encoding": {"name": "default", "configuration": {}}, - "dimension_names": UNSET, - "attributes": {}, - "storage_transformers": [], - "extra_fields": {}, - } - # model-shaped data validates, nested named configs and all - model = adapter.validate_python(model_shaped) - assert isinstance(model.data_type, NamedConfigModelV3) - - # divergence 1: the DOCUMENT form is rejected — no from_json normalization - with pytest.raises(ValidationError): - adapter.validate_python(model_shaped | {"data_type": "uint8"}) - - # divergence 2: lax coercion re-opens holes the library validators close - coerced = adapter.validate_python(model_shaped | {"shape": [True, -5]}) - assert coerced.shape == (1, -5) # from_json would reject both entries - - # __post_init__ invariants DO still run under pydantic construction - with pytest.raises(ValidationError, match="Extra fields"): - adapter.validate_python( - model_shaped | {"extra_fields": {"shape": {"must_understand": False}}} + def build_and_use() -> None: + adapter = TypeAdapter(ArrayMetadataModelV3) + adapter.rebuild( + force=True, + _types_namespace={ + "JSONValue": JSONValue, + "ExtensionFieldV3": ExtensionFieldV3, + "MetadataV3": MetadataV3, + "ArrayMetadataV3": ArrayMetadataV3, + "NamedConfigModelV3": NamedConfigModelV3, + "MetadataFieldModelV3": NamedConfigModelV3, + "UNSET": UNSET, + }, ) + adapter.validate_python({}) + + with pytest.raises((AttributeError, PydanticSchemaGenerationError, PydanticUserError)): + build_and_use() # --- a first-class pydantic model, engine-backed (the pydantic-zarr pattern) -- diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 16a540b3f9..ee26e80c5a 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -56,7 +56,6 @@ def _group_rank(s: str) -> int: "MetadataValidationError", "ProblemKind", "UNSET", - "UnsetType", # v2 data-type encoding union "DataTypeMetadataV2", # Category B — codec canonical unions From 8156ac0df92ea796cba83c64c555f9290af71ee7 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 5 Jul 2026 21:50:51 +0200 Subject: [PATCH 33/70] fix(zarr-metadata): .zattrs presence is part of the store, not an artifact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves the last flagged round-trip question from the initial port: to_key_value on the v2 models always emitted a .zattrs key, so a store that never had one gained a file on round-trip. Per d-v-b's ruling, attributes on ArrayMetadataModelV2/GroupMetadataModelV2 is now `dict[str, JSONValue] | UNSET`: UNSET means no .zattrs file (and no attributes key in the merged document form) and emits nothing, while any dict — including an explicit empty {} — means the file exists and is emitted. The two spellings stay distinct through round-trips, per the None/UNSET invariant; create_default defaults to UNSET (a fresh minimal node has no .zattrs). Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/changes/210.feature.md | 6 +++ .../src/zarr_metadata/model/_array.py | 50 +++++++++++-------- .../src/zarr_metadata/model/_group.py | 45 +++++++++-------- .../zarr-metadata/tests/model/test_array.py | 39 ++++++++++----- .../zarr-metadata/tests/model/test_group.py | 17 +++++-- 5 files changed, 98 insertions(+), 59 deletions(-) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index 12f57c1525..f711d46af5 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -61,3 +61,9 @@ different documents and round-trip as such. The `consolidated_metadata: null` written by a historical zarr-python bug is the one deliberate exception to faithful round-tripping: those stores remain readable, but the bug spelling is repaired to absence on read and never written back. + +The v2 models treat the `.zattrs` file's presence as part of the store: +`attributes` is `UNSET` when no `.zattrs` file exists (and `to_key_value` +emits none), while an explicit empty `.zattrs` is `{}` and round-trips as a +file. Previously `to_key_value` always emitted `.zattrs`, silently adding a +file to stores that never had one. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index bfa57ddb34..f68c1b0f5e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -331,7 +331,7 @@ class ArrayMetadataModelV2Partial(TypedDict, total=False): compressor: CodecMetadataV2 | None filters: tuple[CodecMetadataV2, ...] | None dimension_separator: ArrayDimensionSeparatorV2 - attributes: dict[str, JSONValue] + attributes: dict[str, JSONValue] | UNSET @dataclass(frozen=True, slots=True, kw_only=True) @@ -341,9 +341,12 @@ class ArrayMetadataModelV2: A canonical, lossless representation of the `.zarray` content plus the sibling `.zattrs` attributes. `dtype`, `compressor`, and `filters` are held in their raw JSON forms and are never interpreted; `fill_value` is - held verbatim in its JSON form. One spelling normalization: a `.zarray` - that omits `dimension_separator` means `"."` by the v2 convention, and - the model holds and re-emits that value explicitly. + held verbatim in its JSON form. `attributes` is `UNSET` when no + `.zattrs` file (or merged `attributes` key) exists — distinct from an + explicit empty `.zattrs`, which is `{}` and round-trips as a file. One + spelling normalization: a `.zarray` that omits `dimension_separator` + means `"."` by the v2 convention, and the model holds and re-emits that + value explicitly. """ zarr_format: Literal[2] = field(default=2, init=False) @@ -359,7 +362,7 @@ class ArrayMetadataModelV2: # normalization, like the v3 bare-string metadata-field form). The value # is never None: the document grammar has no null spelling for this field. dimension_separator: ArrayDimensionSeparatorV2 = field(default=".") - attributes: dict[str, JSONValue] + attributes: dict[str, JSONValue] | UNSET def update(self, **kwargs: Unpack[ArrayMetadataModelV2Partial]) -> ArrayMetadataModelV2: """ @@ -400,15 +403,16 @@ def create_default( order="C", compressor=None, filters=None, - attributes={}, + attributes=UNSET, ) return default.update(**overrides) def to_json(self) -> ArrayMetadataV2: - """Return the merged in-memory document form, INCLUDING `attributes`. + """Return the merged in-memory document form. - This is not the on-disk `.zarray` content: a conforming `.zarray` must - exclude `attributes` (they live in the sibling `.zattrs` file). Use + `attributes` is included when set (even empty). This is not the + on-disk `.zarray` content: a conforming `.zarray` must exclude + `attributes` (they live in the sibling `.zattrs` file). Use `to_key_value` to produce the spec-conforming split for storage. """ out: ArrayMetadataV2 = { @@ -419,10 +423,11 @@ def to_json(self) -> ArrayMetadataV2: "chunks": self.chunks, "fill_value": self.fill_value, "dimension_separator": self.dimension_separator, - "attributes": self.attributes, "compressor": self.compressor, "filters": self.filters, } + if self.attributes is not UNSET: + out["attributes"] = self.attributes return out @classmethod @@ -437,24 +442,25 @@ def from_json(cls, data: object) -> ArrayMetadataModelV2: compressor=parsed["compressor"], filters=parsed["filters"], dimension_separator=parsed.get("dimension_separator", "."), - attributes=dict(parsed.get("attributes", {})), + attributes=(dict(parsed["attributes"]) if "attributes" in parsed else UNSET), ) @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV2: zarray = load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V2) - zattrs: dict[str, JSONValue] = ( - load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) - if ATTRIBUTES_STORE_KEY_V2 in mapping - else {} - ) - return cls.from_json({**zarray, "attributes": zattrs}) + if ATTRIBUTES_STORE_KEY_V2 in mapping: + zattrs = load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) + return cls.from_json({**zarray, "attributes": zattrs}) + return cls.from_json(dict(zarray)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: # Attributes live only in the sibling `.zattrs` file; the `.zarray` - # document must exclude them. + # document must exclude them. The `.zattrs` key is present exactly + # when attributes are set (even empty) — UNSET emits no file. zarray = {k: v for k, v in self.to_json().items() if k != "attributes"} - return { - ARRAY_METADATA_STORE_KEY_V2: json.dumps(zarray, indent=indent).encode("utf-8"), - ATTRIBUTES_STORE_KEY_V2: json.dumps(self.attributes, indent=indent).encode("utf-8"), - } + out = {ARRAY_METADATA_STORE_KEY_V2: json.dumps(zarray, indent=indent).encode("utf-8")} + if self.attributes is not UNSET: + out[ATTRIBUTES_STORE_KEY_V2] = json.dumps(self.attributes, indent=indent).encode( + "utf-8" + ) + return out diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index e3a006699b..222b0b3bf9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -260,7 +260,7 @@ class GroupMetadataModelV2Partial(TypedDict, total=False): `tests/model/test_group.py::test_group_partial_keys_match_settable_model_fields`. """ - attributes: dict[str, JSONValue] + attributes: dict[str, JSONValue] | UNSET @dataclass(frozen=True, slots=True, kw_only=True) @@ -269,11 +269,14 @@ class GroupMetadataModelV2: A canonical, lossless representation of the `.zgroup` content plus the sibling `.zattrs` attributes, folded into a single in-memory value - (mirroring the merged `GroupMetadataV2` document form). + (mirroring the merged `GroupMetadataV2` document form). `attributes` is + `UNSET` when no `.zattrs` file (or merged `attributes` key) exists — + distinct from an explicit empty `.zattrs`, which is `{}` and round-trips + as a file. """ zarr_format: Literal[2] = field(default=2, init=False) - attributes: dict[str, JSONValue] + attributes: dict[str, JSONValue] | UNSET @classmethod def create_default( @@ -286,7 +289,7 @@ def create_default( analog of `list()` returning `[]`. Any field can be overridden by keyword (the same fields accepted by `update`). """ - default = cls(attributes={}) + default = cls(attributes=UNSET) return default.update(**overrides) def update(self, **kwargs: Unpack[GroupMetadataModelV2Partial]) -> GroupMetadataModelV2: @@ -301,40 +304,42 @@ def update(self, **kwargs: Unpack[GroupMetadataModelV2Partial]) -> GroupMetadata return dataclasses.replace(self, **kwargs) def to_json(self) -> GroupMetadataV2: - """Return the merged in-memory document form, INCLUDING `attributes`. + """Return the merged in-memory document form. - This is not the on-disk `.zgroup` content: a conforming `.zgroup` must - exclude `attributes` (they live in the sibling `.zattrs` file). Use + `attributes` is included when set (even empty). This is not the + on-disk `.zgroup` content: a conforming `.zgroup` must exclude + `attributes` (they live in the sibling `.zattrs` file). Use `to_key_value` to produce the spec-conforming split for storage. """ out: GroupMetadataV2 = {"zarr_format": self.zarr_format} - if len(self.attributes) > 0: + if self.attributes is not UNSET: out["attributes"] = self.attributes return out @classmethod def from_json(cls, data: object) -> GroupMetadataModelV2: parsed = parse_group_metadata_v2(arrays_to_tuples(data)) - return cls(attributes=dict(parsed.get("attributes", {}))) + return cls(attributes=(dict(parsed["attributes"]) if "attributes" in parsed else UNSET)) @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV2: zgroup = load_store_json(mapping, GROUP_METADATA_STORE_KEY_V2) - zattrs: dict[str, JSONValue] = ( - load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) - if ATTRIBUTES_STORE_KEY_V2 in mapping - else {} - ) - return cls.from_json({**zgroup, "attributes": zattrs}) + if ATTRIBUTES_STORE_KEY_V2 in mapping: + zattrs = load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) + return cls.from_json({**zgroup, "attributes": zattrs}) + return cls.from_json(dict(zgroup)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: # Attributes live only in the sibling `.zattrs` file; the `.zgroup` - # document must exclude them. + # document must exclude them. The `.zattrs` key is present exactly + # when attributes are set (even empty) — UNSET emits no file. zgroup = {k: v for k, v in self.to_json().items() if k != "attributes"} - return { - GROUP_METADATA_STORE_KEY_V2: json.dumps(zgroup, indent=indent).encode("utf-8"), - ATTRIBUTES_STORE_KEY_V2: json.dumps(self.attributes, indent=indent).encode("utf-8"), - } + out = {GROUP_METADATA_STORE_KEY_V2: json.dumps(zgroup, indent=indent).encode("utf-8")} + if self.attributes is not UNSET: + out[ATTRIBUTES_STORE_KEY_V2] = json.dumps(self.attributes, indent=indent).encode( + "utf-8" + ) + return out @dataclass(frozen=True, slots=True, kw_only=True) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index d9af8aeff6..ff27923635 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -326,7 +326,7 @@ def test_v2_create_default_is_valid_empty_array() -> None: assert m.fill_value == 0 assert m.compressor is None assert m.filters is None - assert m.attributes == {} + assert m.attributes is UNSET assert validate_array_metadata_v2(m.to_json()) == [] assert ArrayMetadataModelV2.from_json(m.to_json()) == m @@ -671,12 +671,16 @@ def test_v2_from_json_reconstructs_fields() -> None: assert model.attributes == {"a": 1} -def test_v2_from_json_defaults_when_attributes_absent() -> None: - """V2 from_json defaults attributes to empty when the key is absent.""" - doc = ArrayMetadataModelV2.create_default().to_json() - del doc["attributes"] - model = ArrayMetadataModelV2.from_json(doc) - assert model.attributes == {} +def test_v2_from_json_attributes_absent_is_unset() -> None: + """V2 from_json reads an absent attributes key as UNSET, distinct from an + explicit empty mapping.""" + absent = ArrayMetadataModelV2.from_json(ArrayMetadataModelV2.create_default().to_json()) + explicit = ArrayMetadataModelV2.from_json( + ArrayMetadataModelV2.create_default(attributes={}).to_json() + ) + assert absent.attributes is UNSET + assert explicit.attributes == {} + assert absent != explicit # --- ArrayMetadataModelV2.from_key_value -------------------------------- @@ -690,12 +694,21 @@ def test_v2_from_key_value_remerges_zattrs() -> None: assert model.shape == (10,) -def test_v2_from_key_value_absent_zattrs_gives_empty_attributes() -> None: - """V2 from_key_value yields empty attributes when .zattrs is absent.""" - kv: dict[str, bytes] = dict(ArrayMetadataModelV2.create_default(attributes={}).to_key_value()) - del kv[".zattrs"] - model = ArrayMetadataModelV2.from_key_value(kv) - assert model.attributes == {} +def test_v2_zattrs_presence_round_trips() -> None: + """The .zattrs file's presence is part of the store: an absent file reads + as UNSET and emits no .zattrs; an explicit empty file reads as {} and + emits .zattrs — the two stores stay distinct through a round-trip.""" + explicit_kv = dict(ArrayMetadataModelV2.create_default(attributes={}).to_key_value()) + assert ".zattrs" in explicit_kv + absent_kv = dict(explicit_kv) + del absent_kv[".zattrs"] + + absent = ArrayMetadataModelV2.from_key_value(absent_kv) + explicit = ArrayMetadataModelV2.from_key_value(explicit_kv) + assert absent.attributes is UNSET + assert explicit.attributes == {} + assert ".zattrs" not in absent.to_key_value() + assert ".zattrs" in explicit.to_key_value() def test_v2_from_json_nested_arrays_in_attributes_become_tuples() -> None: diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 0062bf7620..58b92f5d71 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -116,10 +116,19 @@ def test_group_v2_key_value_split() -> None: assert GroupMetadataModelV2.from_key_value(kv) == model -def test_group_v2_from_key_value_without_zattrs() -> None: - """A v2 group with no .zattrs file parses with empty attributes.""" - model = GroupMetadataModelV2.from_key_value({".zgroup": b'{"zarr_format": 2}'}) - assert model.attributes == {} +def test_group_v2_zattrs_presence_round_trips() -> None: + """A v2 group with no .zattrs file parses with UNSET attributes and emits + no .zattrs; an explicit empty .zattrs stays a file — the stores remain + distinct through a round-trip.""" + absent = GroupMetadataModelV2.from_key_value({".zgroup": b'{"zarr_format": 2}'}) + assert absent.attributes is UNSET + assert ".zattrs" not in absent.to_key_value() + explicit = GroupMetadataModelV2.from_key_value( + {".zgroup": b'{"zarr_format": 2}', ".zattrs": b"{}"} + ) + assert explicit.attributes == {} + assert ".zattrs" in explicit.to_key_value() + assert absent != explicit def test_group_v2_json_roundtrip() -> None: From 340a17d9f91547f788790ac54aee74a8587dc761 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 8 Jul 2026 10:35:50 +0200 Subject: [PATCH 34/70] fix(zarr-metadata): require typing_extensions>=4.16 so UNSET pickles by reference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Models hold the UNSET sentinel as field values (dimension_names, attributes), so any object graph containing a model must survive pickling and deep-copying. A sentinel's contract is identity — state-based pickling would produce impostor objects that fail every `is UNSET` check — which is why typing_extensions <= 4.15 refused to pickle sentinels at all. typing_extensions 4.16 implements Sentinel.__reduce__ as pickling by reference (a lookup of the sentinel's name on its defining module), the same mechanism enum members use, so the singleton identity survives the round trip. Bump the floor and pin the behavior with tests: identity across pickle/copy/deepcopy, models holding UNSET round-tripping, and a guard that a non-importable sentinel still fails loudly rather than pickling by state. Co-Authored-By: Claude Fable 5 --- packages/zarr-metadata/pyproject.toml | 6 +- .../src/zarr_metadata/model/_sentinel.py | 8 ++ .../tests/model/test_sentinel.py | 95 +++++++++++++++++++ 3 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 packages/zarr-metadata/tests/model/test_sentinel.py diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 87f35db1db..60f5c4aa95 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -32,7 +32,11 @@ classifiers = [ ] keywords = ["zarr"] dependencies = [ - "typing_extensions>=4.14", + # >=4.16: first release where `Sentinel` pickles by reference + # (`__reduce__` returns the sentinel's name), so `UNSET` — and any model + # holding it — can cross process boundaries and be deep-copied with its + # singleton identity intact. 4.15 and earlier refuse to pickle sentinels. + "typing_extensions>=4.16", ] [project.urls] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py index eeae0da97a..ad71e216fa 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py @@ -8,6 +8,14 @@ Check with identity: `if model.dimension_names is UNSET: ...`. +Because the contract is identity, the sentinel must never be reconstructed +from state: pickling and copying work by *reference* (typing_extensions >= +4.16 implements `Sentinel.__reduce__` as a lookup of the sentinel's name on +its defining module), so `pickle.loads(pickle.dumps(UNSET)) is UNSET` holds +across process boundaries, and models holding `UNSET` pickle and deep-copy +freely. Earlier typing_extensions releases refused to pickle sentinels +outright — hence the `>=4.16` floor in this package's dependencies. + Checker support (PEP 661 is Final; stdlib `sentinel` arrives in Python 3.15): ty types this spelling exactly, including `is`/`is not` narrowing. Pyright supports it but a regression (1.1.405+, tracked as diff --git a/packages/zarr-metadata/tests/model/test_sentinel.py b/packages/zarr-metadata/tests/model/test_sentinel.py new file mode 100644 index 0000000000..78aa60e208 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_sentinel.py @@ -0,0 +1,95 @@ +"""Tests for the pickling and copying behavior of the `UNSET` sentinel. + +The sentinel's contract is identity, so it must never be reconstructed from +state. typing_extensions >= 4.16 pickles sentinels by reference (a lookup of +the sentinel's name on its defining module), which preserves the singleton +across process boundaries; these tests pin that behavior, since models hold +`UNSET` as field values and must survive pickling and deep-copying. + +The model round-trip tests compare whole structures: dataclass equality +compares every field, and `UNSET` compares by identity, so an impostor +sentinel produced by state-based pickling would fail the equality check. +""" + +from __future__ import annotations + +import copy +import pickle + +import pytest +from typing_extensions import Sentinel + +from zarr_metadata.model import ( + UNSET, + ArrayMetadataModelV2, + ArrayMetadataModelV3, + ConsolidatedMetadataModelV3, + GroupMetadataModelV2, + GroupMetadataModelV3, +) + +# Whole-model cases covering the states we know are problematic for +# serialization: every optional-key field in the UNSET (absent) state, the +# same fields in the present state (including present-but-empty, which must +# stay distinct from absent), and UNSET nested inside consolidated metadata. +MODEL_CASES = { + "array-v3-dimension-names-unset": ArrayMetadataModelV3.create_default(shape=(4,)), + "array-v3-dimension-names-set": ArrayMetadataModelV3.create_default(shape=(2, 2)).update( + dimension_names=("x", None) + ), + "array-v2-attributes-unset": ArrayMetadataModelV2.create_default(shape=(4,)), + "array-v2-attributes-empty": ArrayMetadataModelV2.create_default(shape=(4,), attributes={}), + "group-v2-attributes-unset": GroupMetadataModelV2.create_default(), + "group-v2-attributes-set": GroupMetadataModelV2.create_default(attributes={"a": 1}), + "group-v3-consolidated-unset": GroupMetadataModelV3.create_default(), + "group-v3-consolidated-with-unset-inside": GroupMetadataModelV3.create_default( + consolidated_metadata=ConsolidatedMetadataModelV3( + metadata={ + "child": ArrayMetadataModelV3.create_default(shape=(4,)), + "subgroup": GroupMetadataModelV3.create_default(), + } + ) + ), +} + + +def test_unset_pickle_round_trip_preserves_identity() -> None: + restored = pickle.loads(pickle.dumps(UNSET)) + assert restored is UNSET + + +def test_unset_copy_preserves_identity() -> None: + assert copy.copy(UNSET) is UNSET + assert copy.deepcopy(UNSET) is UNSET + + +@pytest.mark.parametrize("model", MODEL_CASES.values(), ids=MODEL_CASES.keys()) +def test_model_pickle_round_trip( + model: ArrayMetadataModelV2 + | ArrayMetadataModelV3 + | GroupMetadataModelV2 + | GroupMetadataModelV3, +) -> None: + restored = pickle.loads(pickle.dumps(model)) + assert restored == model + + +@pytest.mark.parametrize("model", MODEL_CASES.values(), ids=MODEL_CASES.keys()) +def test_model_deepcopy( + model: ArrayMetadataModelV2 + | ArrayMetadataModelV3 + | GroupMetadataModelV2 + | GroupMetadataModelV3, +) -> None: + assert copy.deepcopy(model) == model + + +def test_non_importable_sentinel_fails_to_pickle() -> None: + """Sentinels pickle by reference, never by state. A sentinel that is not + an importable attribute of its module has no reference to pickle, so + dumping it must fail loudly — a successful dump here would mean the + implementation regressed to state-based pickling, which would produce + identity-breaking impostor objects on the receiving side.""" + local_sentinel = Sentinel("local_sentinel") + with pytest.raises((pickle.PicklingError, TypeError)): + pickle.dumps(local_sentinel) From 06668a74dbb73725f6efae9ea2aa04ccc792ec2c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 8 Jul 2026 11:48:53 +0200 Subject: [PATCH 35/70] refactor(zarr-metadata): format-version-first naming; dataclasses primary, JSON suffix for documents Applies the naming decisions from the PR discussion: ZarrV2/ZarrV3 moves to the front of every type name so a format version cannot be misread as a class revision, and the model dataclasses take the bare entity names (ZarrV3ArrayMetadata, ZarrV3GroupMetadata, ZarrV3ConsolidatedMetadata, ZarrV3NamedConfig, role alias ZarrV3MetadataField) while the TypedDict document forms carry a JSON suffix (ZarrV3ArrayMetadataJSON, ..., ZarrV3MetadataFieldJSON, ZarrV3NamedConfigJSON). The zarr_metadata.pydantic field types take the bare entity names, matching the model classes they validate into; the module now references the model module qualified to keep those names free. Raw-layer names released in 0.3.0 are renamed without aliases (pre-1.0), documented in changes/4119.removal.md. Validation problem messages name documents in plain English instead of type names. snake_case function names (validate_array_metadata_v3, ...) and SCREAMING_SNAKE constants are deliberately untouched: the revision ambiguity the rename fixes does not arise for them, and renaming them is a separate decision. Co-Authored-By: Claude Fable 5 --- packages/zarr-metadata/README.md | 4 +- packages/zarr-metadata/changes/210.feature.md | 10 +- .../zarr-metadata/changes/4119.removal.md | 23 ++ .../src/zarr_metadata/__init__.py | 116 +++--- .../src/zarr_metadata/_common.py | 2 +- .../src/zarr_metadata/model/__init__.py | 48 +-- .../src/zarr_metadata/model/_array.py | 180 +++++---- .../src/zarr_metadata/model/_group.py | 120 +++--- .../src/zarr_metadata/model/_validation.py | 64 ++-- .../src/zarr_metadata/pydantic.py | 102 +++-- .../src/zarr_metadata/v2/__init__.py | 28 +- .../src/zarr_metadata/v2/array.py | 60 +-- .../src/zarr_metadata/v2/codec.py | 4 +- .../src/zarr_metadata/v2/consolidated.py | 4 +- .../src/zarr_metadata/v2/group.py | 16 +- .../src/zarr_metadata/v3/__init__.py | 18 +- .../src/zarr_metadata/v3/_common.py | 8 +- .../src/zarr_metadata/v3/array.py | 46 +-- .../src/zarr_metadata/v3/codec/__init__.py | 2 +- .../src/zarr_metadata/v3/codec/cast_value.py | 4 +- .../v3/codec/sharding_indexed.py | 6 +- .../src/zarr_metadata/v3/consolidated.py | 12 +- .../src/zarr_metadata/v3/data_type/struct.py | 4 +- .../src/zarr_metadata/v3/group.py | 20 +- .../zarr-metadata/tests/model/test_array.py | 361 +++++++++--------- .../zarr-metadata/tests/model/test_group.py | 100 ++--- .../tests/model/test_pydantic.py | 48 +-- .../tests/model/test_pydantic_module.py | 50 +-- .../tests/model/test_sentinel.py | 42 +- .../tests/test_partial_equivalence.py | 16 +- .../zarr-metadata/tests/test_public_api.py | 60 +-- .../tests/v2/consolidated/test_fixtures.py | 4 +- .../tests/v3/array/test_fixtures.py | 6 +- .../tests/v3/array/with_extra_field.json | 2 +- .../tests/v3/consolidated/test_fixtures.py | 4 +- .../tests/v3/group/test_fixtures.py | 4 +- 36 files changed, 803 insertions(+), 795 deletions(-) create mode 100644 packages/zarr-metadata/changes/4119.removal.md diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index a842e07886..f2884782fd 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -20,12 +20,12 @@ Zarr metadata. Pair them with a runtime validator like ```python import json from pydantic import TypeAdapter -from zarr_metadata.v3.array import ArrayMetadataV3 +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON with open("zarr.json", "rb") as f: raw = json.load(f) -metadata = TypeAdapter(ArrayMetadataV3).validate_python(raw) +metadata = TypeAdapter(ZarrV3ArrayMetadataJSON).validate_python(raw) ``` ## What this is *not* diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/210.feature.md index f711d46af5..fda1a88cf7 100644 --- a/packages/zarr-metadata/changes/210.feature.md +++ b/packages/zarr-metadata/changes/210.feature.md @@ -1,12 +1,12 @@ -Added `zarr_metadata.model`: frozen-dataclass models (`ArrayMetadataModelV2`, -`ArrayMetadataModelV3`, `GroupMetadataModelV2`, `GroupMetadataModelV3`, -`ConsolidatedMetadataModelV2`, `ConsolidatedMetadataModelV3`, `NamedConfigModelV3`) +Added `zarr_metadata.model`: frozen-dataclass models (`ZarrV2ArrayMetadata`, +`ZarrV3ArrayMetadata`, `ZarrV2GroupMetadata`, `ZarrV3GroupMetadata`, +`ZarrV2ConsolidatedMetadata`, `ZarrV3ConsolidatedMetadata`, `ZarrV3NamedConfig`) that are canonical, lossless representations of Zarr metadata documents, plus structural validators (`validate_*` / `is_*` / `parse_*`). Every v3 extension point (data type, chunk grid, chunk key encoding, codecs, storage transformers) is held as a name + configuration pair; nothing is interpreted. Model fields -are annotated with the role alias `MetadataFieldModelV3` (today exactly -`NamedConfigModelV3`), so the annotations convey the logical meaning of the +are annotated with the role alias `ZarrV3MetadataField` (today exactly +`ZarrV3NamedConfig`), so the annotations convey the logical meaning of the field and stay put if the spec ever adds a new field form. Validation is strict about what the types declare: v2 `dtype` / `order` / diff --git a/packages/zarr-metadata/changes/4119.removal.md b/packages/zarr-metadata/changes/4119.removal.md new file mode 100644 index 0000000000..0c90f59131 --- /dev/null +++ b/packages/zarr-metadata/changes/4119.removal.md @@ -0,0 +1,23 @@ +The document (TypedDict) types are renamed to put the format version at the +front of the name and to mark the JSON-document form with a `JSON` suffix, +so a format version can never be misread as a class revision and the bare +entity names are reserved for the `zarr_metadata.model` dataclasses: + +- `ArrayMetadataV2` → `ZarrV2ArrayMetadataJSON` (and `...Partial` accordingly) +- `ArrayMetadataV3` → `ZarrV3ArrayMetadataJSON` (and `...Partial` accordingly) +- `GroupMetadataV2` → `ZarrV2GroupMetadataJSON` (and `...Partial` accordingly) +- `GroupMetadataV3` → `ZarrV3GroupMetadataJSON` (and `...Partial` accordingly) +- `ConsolidatedMetadataV2` → `ZarrV2ConsolidatedMetadataJSON` +- `ConsolidatedMetadataV3` → `ZarrV3ConsolidatedMetadataJSON` +- `NamedConfigV3` → `ZarrV3NamedConfigJSON` +- `MetadataV3` → `ZarrV3MetadataFieldJSON` (the union of the bare-name and + named-configuration spellings of one metadata field) +- `ExtensionFieldV3` → `ZarrV3ExtensionField` +- `CodecMetadataV2` → `ZarrV2CodecMetadata` +- `DataTypeMetadataV2` → `ZarrV2DataTypeMetadata` +- `ArrayOrderV2` → `ZarrV2ArrayOrder` +- `ArrayDimensionSeparatorV2` → `ZarrV2ArrayDimensionSeparator` + +The old names are removed, not aliased. The `zarr_metadata.pydantic` field +types take the bare entity names (`ZarrV3ArrayMetadata`, ...), matching the +model classes they validate into. diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index 4c1328bfe1..ab7bce2ff2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -1,40 +1,48 @@ from importlib.metadata import version -from zarr_metadata._common import JSONValue, NamedConfigV3 +from zarr_metadata._common import JSONValue, ZarrV3NamedConfigJSON from zarr_metadata.model import ( UNSET, - ArrayMetadataModelV2, - ArrayMetadataModelV2Partial, - ArrayMetadataModelV3, - ArrayMetadataModelV3Partial, - ConsolidatedMetadataModelV2, - ConsolidatedMetadataModelV3, - GroupMetadataModelV2, - GroupMetadataModelV2Partial, - GroupMetadataModelV3, - GroupMetadataModelV3Partial, - MetadataFieldModelV3, MetadataValidationError, - NamedConfigModelV3, ProblemKind, ValidationProblem, + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadataPartial, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV2GroupMetadataPartial, + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadataPartial, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, + ZarrV3GroupMetadataPartial, + ZarrV3MetadataField, + ZarrV3NamedConfig, ) from zarr_metadata.v2.array import ( ARRAY_DIMENSION_SEPARATOR_V2, ARRAY_ORDER_V2, - ArrayDimensionSeparatorV2, - ArrayMetadataV2, - ArrayMetadataV2Partial, - ArrayOrderV2, - DataTypeMetadataV2, ZArrayMetadata, + ZarrV2ArrayDimensionSeparator, + ZarrV2ArrayMetadataJSON, + ZarrV2ArrayMetadataJSONPartial, + ZarrV2ArrayOrder, + ZarrV2DataTypeMetadata, ) from zarr_metadata.v2.attributes import ZAttrsMetadata -from zarr_metadata.v2.codec import CodecMetadataV2 -from zarr_metadata.v2.consolidated import ConsolidatedMetadataV2 -from zarr_metadata.v2.group import GroupMetadataV2, GroupMetadataV2Partial, ZGroupMetadata -from zarr_metadata.v3._common import MetadataV3 -from zarr_metadata.v3.array import ArrayMetadataV3, ArrayMetadataV3Partial, ExtensionFieldV3 +from zarr_metadata.v2.codec import ZarrV2CodecMetadata +from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON +from zarr_metadata.v2.group import ( + ZarrV2GroupMetadataJSON, + ZarrV2GroupMetadataJSONPartial, + ZGroupMetadata, +) +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +from zarr_metadata.v3.array import ( + ZarrV3ArrayMetadataJSON, + ZarrV3ArrayMetadataJSONPartial, + ZarrV3ExtensionField, +) from zarr_metadata.v3.chunk_grid.rectilinear import ( RECTILINEAR_CHUNK_GRID_NAME, RectilinearChunkGridMetadata, @@ -104,7 +112,7 @@ TransposeCodecName, ) from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME, ZstdCodecMetadata, ZstdCodecName -from zarr_metadata.v3.consolidated import ConsolidatedMetadataV3 +from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON from zarr_metadata.v3.data_type.bool import ( BOOL_DATA_TYPE_NAME, BoolDataTypeName, @@ -203,7 +211,7 @@ Uint64DataTypeName, Uint64FillValue, ) -from zarr_metadata.v3.group import GroupMetadataV3, GroupMetadataV3Partial +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON, ZarrV3GroupMetadataJSONPartial __version__ = version("zarr-metadata") @@ -253,16 +261,6 @@ "V2_CHUNK_KEY_ENCODING_NAME", "V2_CHUNK_KEY_ENCODING_SEPARATOR", "ZSTD_CODEC_NAME", - "ArrayDimensionSeparatorV2", - "ArrayMetadataModelV2", - "ArrayMetadataModelV2Partial", - "ArrayMetadataModelV3", - "ArrayMetadataModelV3Partial", - "ArrayMetadataV2", - "ArrayMetadataV2Partial", - "ArrayMetadataV3", - "ArrayMetadataV3Partial", - "ArrayOrderV2", "BloscCName", "BloscCodecMetadata", "BloscCodecName", @@ -277,37 +275,22 @@ "CastRoundingMode", "CastValueCodecMetadata", "CastValueCodecName", - "CodecMetadataV2", "Complex64DataTypeName", "Complex64FillValue", "Complex128DataTypeName", "Complex128FillValue", - "ConsolidatedMetadataModelV2", - "ConsolidatedMetadataModelV3", - "ConsolidatedMetadataV2", - "ConsolidatedMetadataV3", "Crc32cCodecMetadata", "Crc32cCodecName", - "DataTypeMetadataV2", "DefaultChunkKeyEncodingMetadata", "DefaultChunkKeyEncodingName", "DefaultChunkKeyEncodingSeparator", "Endianness", - "ExtensionFieldV3", "Float16DataTypeName", "Float16FillValue", "Float32DataTypeName", "Float32FillValue", "Float64DataTypeName", "Float64FillValue", - "GroupMetadataModelV2", - "GroupMetadataModelV2Partial", - "GroupMetadataModelV3", - "GroupMetadataModelV3Partial", - "GroupMetadataV2", - "GroupMetadataV2Partial", - "GroupMetadataV3", - "GroupMetadataV3Partial", "GzipCodecMetadata", "GzipCodecName", "Int8DataTypeName", @@ -319,11 +302,7 @@ "Int64DataTypeName", "Int64FillValue", "JSONValue", - "MetadataFieldModelV3", - "MetadataV3", "MetadataValidationError", - "NamedConfigModelV3", - "NamedConfigV3", "NumpyDatetime64DataTypeName", "NumpyDatetime64FillValue", "NumpyTimeUnit", @@ -362,6 +341,35 @@ "ZArrayMetadata", "ZAttrsMetadata", "ZGroupMetadata", + "ZarrV2ArrayDimensionSeparator", + "ZarrV2ArrayMetadata", + "ZarrV2ArrayMetadataJSON", + "ZarrV2ArrayMetadataJSONPartial", + "ZarrV2ArrayMetadataPartial", + "ZarrV2ArrayOrder", + "ZarrV2CodecMetadata", + "ZarrV2ConsolidatedMetadata", + "ZarrV2ConsolidatedMetadataJSON", + "ZarrV2DataTypeMetadata", + "ZarrV2GroupMetadata", + "ZarrV2GroupMetadataJSON", + "ZarrV2GroupMetadataJSONPartial", + "ZarrV2GroupMetadataPartial", + "ZarrV3ArrayMetadata", + "ZarrV3ArrayMetadataJSON", + "ZarrV3ArrayMetadataJSONPartial", + "ZarrV3ArrayMetadataPartial", + "ZarrV3ConsolidatedMetadata", + "ZarrV3ConsolidatedMetadataJSON", + "ZarrV3ExtensionField", + "ZarrV3GroupMetadata", + "ZarrV3GroupMetadataJSON", + "ZarrV3GroupMetadataJSONPartial", + "ZarrV3GroupMetadataPartial", + "ZarrV3MetadataField", + "ZarrV3MetadataFieldJSON", + "ZarrV3NamedConfig", + "ZarrV3NamedConfigJSON", "ZstdCodecMetadata", "ZstdCodecName", "__version__", diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index b335cd0bd6..b66183f875 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -24,7 +24,7 @@ """ -class NamedConfigV3(TypedDict): +class ZarrV3NamedConfigJSON(TypedDict): """ Externally-tagged union member for a metadata field. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 1257109465..1889b20300 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -15,24 +15,24 @@ ARRAY_METADATA_STORE_KEY_V2, ARRAY_METADATA_STORE_KEY_V3, ATTRIBUTES_STORE_KEY_V2, - ArrayMetadataModelV2, - ArrayMetadataModelV2Partial, - ArrayMetadataModelV3, - ArrayMetadataModelV3Partial, - MetadataFieldModelV3, - NamedConfigModelV3, + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadataPartial, + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadataPartial, + ZarrV3MetadataField, + ZarrV3NamedConfig, ) from zarr_metadata.model._group import ( CONSOLIDATED_METADATA_KEY_V3, CONSOLIDATED_METADATA_STORE_KEY_V2, GROUP_METADATA_STORE_KEY_V2, GROUP_METADATA_STORE_KEY_V3, - ConsolidatedMetadataModelV2, - ConsolidatedMetadataModelV3, - GroupMetadataModelV2, - GroupMetadataModelV2Partial, - GroupMetadataModelV3, - GroupMetadataModelV3Partial, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV2GroupMetadataPartial, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, + ZarrV3GroupMetadataPartial, ) from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ( @@ -84,21 +84,21 @@ "GROUP_METADATA_STORE_KEY_V2", "GROUP_METADATA_STORE_KEY_V3", "UNSET", - "ArrayMetadataModelV2", - "ArrayMetadataModelV2Partial", - "ArrayMetadataModelV3", - "ArrayMetadataModelV3Partial", - "ConsolidatedMetadataModelV2", - "ConsolidatedMetadataModelV3", - "GroupMetadataModelV2", - "GroupMetadataModelV2Partial", - "GroupMetadataModelV3", - "GroupMetadataModelV3Partial", - "MetadataFieldModelV3", "MetadataValidationError", - "NamedConfigModelV3", "ProblemKind", "ValidationProblem", + "ZarrV2ArrayMetadata", + "ZarrV2ArrayMetadataPartial", + "ZarrV2ConsolidatedMetadata", + "ZarrV2GroupMetadata", + "ZarrV2GroupMetadataPartial", + "ZarrV3ArrayMetadata", + "ZarrV3ArrayMetadataPartial", + "ZarrV3ConsolidatedMetadata", + "ZarrV3GroupMetadata", + "ZarrV3GroupMetadataPartial", + "ZarrV3MetadataField", + "ZarrV3NamedConfig", "is_array_metadata_v2", "is_array_metadata_v3", "is_group_metadata_v2", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index f68c1b0f5e..1f4a2ded4c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -25,30 +25,30 @@ if TYPE_CHECKING: from zarr_metadata._common import JSONValue from zarr_metadata.v2.array import ( - ArrayDimensionSeparatorV2, - ArrayMetadataV2, - ArrayOrderV2, - DataTypeMetadataV2, + ZarrV2ArrayDimensionSeparator, + ZarrV2ArrayMetadataJSON, + ZarrV2ArrayOrder, + ZarrV2DataTypeMetadata, ) - from zarr_metadata.v2.codec import CodecMetadataV2 - from zarr_metadata.v3._common import MetadataV3 - from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 + from zarr_metadata.v2.codec import ZarrV2CodecMetadata + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON, ZarrV3ExtensionField -ArrayMetadataStoreKeyV3 = Literal["zarr.json"] -ARRAY_METADATA_STORE_KEY_V3: Final[ArrayMetadataStoreKeyV3] = "zarr.json" +ZarrV3ArrayMetadataStoreKey = Literal["zarr.json"] +ARRAY_METADATA_STORE_KEY_V3: Final[ZarrV3ArrayMetadataStoreKey] = "zarr.json" -ArrayMetadataStoreKeyV2 = Literal[".zarray"] -ARRAY_METADATA_STORE_KEY_V2: Final[ArrayMetadataStoreKeyV2] = ".zarray" +ZarrV2ArrayMetadataStoreKey = Literal[".zarray"] +ARRAY_METADATA_STORE_KEY_V2: Final[ZarrV2ArrayMetadataStoreKey] = ".zarray" -AttributesStoreKeyV2 = Literal[".zattrs"] -ATTRIBUTES_STORE_KEY_V2: Final[AttributesStoreKeyV2] = ".zattrs" +ZarrV2AttributesStoreKey = Literal[".zattrs"] +ATTRIBUTES_STORE_KEY_V2: Final[ZarrV2AttributesStoreKey] = ".zattrs" @dataclass(frozen=True, slots=True, kw_only=True) -class NamedConfigModelV3: +class ZarrV3NamedConfig: """A v3 metadata field in normalized form: a name plus a configuration. - This is the in-memory model of `MetadataV3` (a bare name string or a + This is the in-memory model of `ZarrV3MetadataFieldJSON` (a bare name string or a `{name, configuration}` mapping): the bare-name and missing-configuration forms normalize to an empty configuration. """ @@ -56,11 +56,11 @@ class NamedConfigModelV3: name: str configuration: dict[str, JSONValue] - def to_json(self) -> MetadataV3: + def to_json(self) -> ZarrV3MetadataFieldJSON: return {"name": self.name, "configuration": self.configuration} @classmethod - def from_json(cls, data: object) -> NamedConfigModelV3: + def from_json(cls, data: object) -> ZarrV3NamedConfig: field = parse_metadata_field_v3(data) if isinstance(field, str): return cls(name=field, configuration={}) @@ -73,23 +73,23 @@ def from_json(cls, data: object) -> NamedConfigModelV3: return cls(name=field["name"], configuration=configuration) -MetadataFieldModelV3: TypeAlias = NamedConfigModelV3 +ZarrV3MetadataField: TypeAlias = ZarrV3NamedConfig """The in-memory model of one field of a v3 metadata document. This is the role-named alias for annotation positions: model fields and -consumer signatures should say `MetadataFieldModelV3` (the logical meaning) -rather than `NamedConfigModelV3` (the serialized form the field currently +consumer signatures should say `ZarrV3MetadataField` (the logical meaning) +rather than `ZarrV3NamedConfig` (the serialized form the field currently takes). Today every metadata field normalizes to a named configuration, so -the alias is exactly `NamedConfigModelV3`; if a future spec revision adds a +the alias is exactly `ZarrV3NamedConfig`; if a future spec revision adds a field form that cannot be normalized to name + configuration, this alias widens to a union and annotation sites do not change. Mirrors the raw-layer -split between `NamedConfigV3` (shape) and `MetadataV3` (field union). +split between `ZarrV3NamedConfigJSON` (shape) and `ZarrV3MetadataFieldJSON` (field union). """ def must_understand_subset( - extra_fields: Mapping[str, ExtensionFieldV3], -) -> dict[str, ExtensionFieldV3]: + extra_fields: Mapping[str, ZarrV3ExtensionField], +) -> dict[str, ZarrV3ExtensionField]: """The subset of `extra_fields` the reader is obligated to understand. Per the v3 spec, an extension field is implicitly `must_understand: True` @@ -98,11 +98,11 @@ def must_understand_subset( explicitly `must_understand: false`. A non-mapping field value cannot carry the explicit waiver, so it always requires understanding (the runtime isinstance check defends against values looser than the declared - `ExtensionFieldV3`). + `ZarrV3ExtensionField`). """ fields = cast("Mapping[str, object]", extra_fields) return cast( - "dict[str, ExtensionFieldV3]", + "dict[str, ZarrV3ExtensionField]", { name: value for name, value in fields.items() @@ -114,13 +114,13 @@ def must_understand_subset( ) -class ArrayMetadataModelV3Partial(TypedDict, total=False): +class ZarrV3ArrayMetadataPartial(TypedDict, total=False): """ - Partial form of the constructor-settable fields of `ArrayMetadataModelV3`. + Partial form of the constructor-settable fields of `ZarrV3ArrayMetadata`. Every key is optional and typed with the model's own (not serialized) value types, so it describes valid keyword arguments to - `ArrayMetadataModelV3.update`. The `init=False` fields `zarr_format` and + `ZarrV3ArrayMetadata.update`. The `init=False` fields `zarr_format` and `node_type` are intentionally excluded, since they cannot be passed to `dataclasses.replace`. @@ -130,24 +130,24 @@ class ArrayMetadataModelV3Partial(TypedDict, total=False): shape: tuple[int, ...] fill_value: JSONValue - data_type: MetadataFieldModelV3 - chunk_grid: MetadataFieldModelV3 - codecs: tuple[MetadataFieldModelV3, ...] - chunk_key_encoding: MetadataFieldModelV3 + data_type: ZarrV3MetadataField + chunk_grid: ZarrV3MetadataField + codecs: tuple[ZarrV3MetadataField, ...] + chunk_key_encoding: ZarrV3MetadataField dimension_names: tuple[str | None, ...] | UNSET attributes: dict[str, JSONValue] - storage_transformers: tuple[MetadataFieldModelV3, ...] - extra_fields: dict[str, ExtensionFieldV3] + storage_transformers: tuple[ZarrV3MetadataField, ...] + extra_fields: dict[str, ZarrV3ExtensionField] @dataclass(frozen=True, slots=True, kw_only=True) -class ArrayMetadataModelV3: +class ZarrV3ArrayMetadata: """In-memory model of a v3 array metadata document. A canonical, lossless representation of the `zarr.json` content for an array. Extension points (`data_type`, `chunk_grid`, `chunk_key_encoding`, - `codecs`, `storage_transformers`) are held as `MetadataFieldModelV3` - values (currently always `NamedConfigModelV3` name + configuration pairs) + `codecs`, `storage_transformers`) are held as `ZarrV3MetadataField` + values (currently always `ZarrV3NamedConfig` name + configuration pairs) and are never interpreted; `fill_value` is held verbatim in its JSON form. """ @@ -156,19 +156,17 @@ class ArrayMetadataModelV3: node_type: Literal["array"] = field(default="array", init=False) shape: tuple[int, ...] fill_value: JSONValue - data_type: MetadataFieldModelV3 - chunk_grid: MetadataFieldModelV3 - codecs: tuple[MetadataFieldModelV3, ...] - chunk_key_encoding: MetadataFieldModelV3 + data_type: ZarrV3MetadataField + chunk_grid: ZarrV3MetadataField + codecs: tuple[ZarrV3MetadataField, ...] + chunk_key_encoding: ZarrV3MetadataField dimension_names: tuple[str | None, ...] | UNSET attributes: dict[str, JSONValue] - storage_transformers: tuple[MetadataFieldModelV3, ...] - extra_fields: dict[str, ExtensionFieldV3] + storage_transformers: tuple[ZarrV3MetadataField, ...] + extra_fields: dict[str, ZarrV3ExtensionField] @classmethod - def create_default( - cls, **overrides: Unpack[ArrayMetadataModelV3Partial] - ) -> ArrayMetadataModelV3: + def create_default(cls, **overrides: Unpack[ZarrV3ArrayMetadataPartial]) -> ZarrV3ArrayMetadata: """ Create a default (empty) v3 array metadata model, with optional overrides. @@ -187,16 +185,16 @@ def create_default( responsibility. """ if "shape" in overrides and "chunk_grid" not in overrides: - overrides["chunk_grid"] = NamedConfigModelV3( + overrides["chunk_grid"] = ZarrV3NamedConfig( name="regular", configuration={"chunk_shape": tuple(overrides["shape"])} ) default = cls( shape=(), fill_value=0, - data_type=NamedConfigModelV3(name="uint8", configuration={}), - chunk_grid=NamedConfigModelV3(name="regular", configuration={"chunk_shape": ()}), - codecs=(NamedConfigModelV3(name="bytes", configuration={}),), - chunk_key_encoding=NamedConfigModelV3(name="default", configuration={}), + data_type=ZarrV3NamedConfig(name="uint8", configuration={}), + chunk_grid=ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": ()}), + codecs=(ZarrV3NamedConfig(name="bytes", configuration={}),), + chunk_key_encoding=ZarrV3NamedConfig(name="default", configuration={}), dimension_names=UNSET, attributes={}, storage_transformers=(), @@ -204,12 +202,12 @@ def create_default( ) return default.update(**overrides) - def update(self, **kwargs: Unpack[ArrayMetadataModelV3Partial]) -> ArrayMetadataModelV3: + def update(self, **kwargs: Unpack[ZarrV3ArrayMetadataPartial]) -> ZarrV3ArrayMetadata: """ - Return a new `ArrayMetadataModelV3` with the given fields updated. + Return a new `ZarrV3ArrayMetadata` with the given fields updated. Only the constructor-settable fields listed in - `ArrayMetadataModelV3Partial` can be updated; any attempt to update + `ZarrV3ArrayMetadataPartial` can be updated; any attempt to update other fields (including the fixed `zarr_format` / `node_type`) is rejected at the type level. Each given field fully replaces its previous value, including `extra_fields`. @@ -230,14 +228,14 @@ def __post_init__(self) -> None: [ ValidationProblem( ("extra_fields",), - "Extra fields cannot overlap with standard ArrayMetadataV3 fields", + "Extra fields cannot overlap with standard Zarr V3 array metadata fields", "invalid_value", ) ] ) - def to_json(self) -> ArrayMetadataV3: - out: ArrayMetadataV3 = { + def to_json(self) -> ZarrV3ArrayMetadataJSON: + out: ZarrV3ArrayMetadataJSON = { "zarr_format": self.zarr_format, "node_type": self.node_type, "shape": self.shape, @@ -264,32 +262,32 @@ def to_json(self) -> ArrayMetadataV3: return out @classmethod - def from_json(cls, data: object) -> ArrayMetadataModelV3: + def from_json(cls, data: object) -> ZarrV3ArrayMetadata: parsed = parse_array_metadata_v3(arrays_to_tuples(data)) # Sound cast: the TypedDict types all non-standard keys as its - # `extra_items` (`ExtensionFieldV3`); the comprehension's inferred value + # `extra_items` (`ZarrV3ExtensionField`); the comprehension's inferred value # type is the union over ALL keys because the key filter cannot narrow it. extra_fields = cast( - "dict[str, ExtensionFieldV3]", + "dict[str, ZarrV3ExtensionField]", {k: v for k, v in parsed.items() if k not in ARRAY_METADATA_STANDARD_KEYS_V3}, ) return cls( shape=parsed["shape"], fill_value=parsed["fill_value"], - data_type=NamedConfigModelV3.from_json(parsed["data_type"]), - chunk_grid=NamedConfigModelV3.from_json(parsed["chunk_grid"]), - codecs=tuple(NamedConfigModelV3.from_json(c) for c in parsed["codecs"]), - chunk_key_encoding=NamedConfigModelV3.from_json(parsed["chunk_key_encoding"]), + data_type=ZarrV3NamedConfig.from_json(parsed["data_type"]), + chunk_grid=ZarrV3NamedConfig.from_json(parsed["chunk_grid"]), + codecs=tuple(ZarrV3NamedConfig.from_json(c) for c in parsed["codecs"]), + chunk_key_encoding=ZarrV3NamedConfig.from_json(parsed["chunk_key_encoding"]), dimension_names=parsed.get("dimension_names", UNSET), attributes=dict(parsed.get("attributes", {})), storage_transformers=tuple( - NamedConfigModelV3.from_json(t) for t in parsed.get("storage_transformers", ()) + ZarrV3NamedConfig.from_json(t) for t in parsed.get("storage_transformers", ()) ), extra_fields=extra_fields, ) @property - def must_understand_fields(self) -> dict[str, ExtensionFieldV3]: + def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: """Extra fields the reader is obligated to understand. Everything in `extra_fields` not explicitly waived with @@ -301,7 +299,7 @@ def must_understand_fields(self) -> dict[str, ExtensionFieldV3]: return must_understand_subset(self.extra_fields) @classmethod - def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV3: + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3ArrayMetadata: return cls.from_json(load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V3)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: @@ -310,12 +308,12 @@ def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes } -class ArrayMetadataModelV2Partial(TypedDict, total=False): +class ZarrV2ArrayMetadataPartial(TypedDict, total=False): """ - Partial form of the constructor-settable fields of `ArrayMetadataModelV2`. + Partial form of the constructor-settable fields of `ZarrV2ArrayMetadata`. Every key is optional and typed with the model's own value types, so it - describes valid keyword arguments to `ArrayMetadataModelV2.update` and + describes valid keyword arguments to `ZarrV2ArrayMetadata.update` and `create_default`. The `init=False` field `zarr_format` is intentionally excluded, since it cannot be passed to `dataclasses.replace`. @@ -324,18 +322,18 @@ class ArrayMetadataModelV2Partial(TypedDict, total=False): """ shape: tuple[int, ...] - dtype: DataTypeMetadataV2 + dtype: ZarrV2DataTypeMetadata chunks: tuple[int, ...] fill_value: JSONValue - order: ArrayOrderV2 - compressor: CodecMetadataV2 | None - filters: tuple[CodecMetadataV2, ...] | None - dimension_separator: ArrayDimensionSeparatorV2 + order: ZarrV2ArrayOrder + compressor: ZarrV2CodecMetadata | None + filters: tuple[ZarrV2CodecMetadata, ...] | None + dimension_separator: ZarrV2ArrayDimensionSeparator attributes: dict[str, JSONValue] | UNSET @dataclass(frozen=True, slots=True, kw_only=True) -class ArrayMetadataModelV2: +class ZarrV2ArrayMetadata: """In-memory model of a v2 array metadata document. A canonical, lossless representation of the `.zarray` content plus the @@ -351,34 +349,32 @@ class ArrayMetadataModelV2: zarr_format: Literal[2] = field(default=2, init=False) shape: tuple[int, ...] - dtype: DataTypeMetadataV2 + dtype: ZarrV2DataTypeMetadata chunks: tuple[int, ...] fill_value: JSONValue - order: ArrayOrderV2 - compressor: CodecMetadataV2 | None - filters: tuple[CodecMetadataV2, ...] | None + order: ZarrV2ArrayOrder + compressor: ZarrV2CodecMetadata | None + filters: tuple[ZarrV2CodecMetadata, ...] | None # "." is the v2 convention's default for an ABSENT dimension_separator key; # from_json normalizes absence to it (a semantics-preserving spelling # normalization, like the v3 bare-string metadata-field form). The value # is never None: the document grammar has no null spelling for this field. - dimension_separator: ArrayDimensionSeparatorV2 = field(default=".") + dimension_separator: ZarrV2ArrayDimensionSeparator = field(default=".") attributes: dict[str, JSONValue] | UNSET - def update(self, **kwargs: Unpack[ArrayMetadataModelV2Partial]) -> ArrayMetadataModelV2: + def update(self, **kwargs: Unpack[ZarrV2ArrayMetadataPartial]) -> ZarrV2ArrayMetadata: """ - Return a new `ArrayMetadataModelV2` with the given fields updated. + Return a new `ZarrV2ArrayMetadata` with the given fields updated. Only the constructor-settable fields listed in - `ArrayMetadataModelV2Partial` can be updated; the fixed `zarr_format` is + `ZarrV2ArrayMetadataPartial` can be updated; the fixed `zarr_format` is rejected at the type level. Each given field fully replaces its previous value. """ return dataclasses.replace(self, **kwargs) @classmethod - def create_default( - cls, **overrides: Unpack[ArrayMetadataModelV2Partial] - ) -> ArrayMetadataModelV2: + def create_default(cls, **overrides: Unpack[ZarrV2ArrayMetadataPartial]) -> ZarrV2ArrayMetadata: """ Create a default (empty) v2 array metadata model, with optional overrides. @@ -407,7 +403,7 @@ def create_default( ) return default.update(**overrides) - def to_json(self) -> ArrayMetadataV2: + def to_json(self) -> ZarrV2ArrayMetadataJSON: """Return the merged in-memory document form. `attributes` is included when set (even empty). This is not the @@ -415,7 +411,7 @@ def to_json(self) -> ArrayMetadataV2: `attributes` (they live in the sibling `.zattrs` file). Use `to_key_value` to produce the spec-conforming split for storage. """ - out: ArrayMetadataV2 = { + out: ZarrV2ArrayMetadataJSON = { "zarr_format": self.zarr_format, "shape": self.shape, "dtype": self.dtype, @@ -431,7 +427,7 @@ def to_json(self) -> ArrayMetadataV2: return out @classmethod - def from_json(cls, data: object) -> ArrayMetadataModelV2: + def from_json(cls, data: object) -> ZarrV2ArrayMetadata: parsed = parse_array_metadata_v2(arrays_to_tuples(data)) return cls( shape=parsed["shape"], @@ -446,7 +442,7 @@ def from_json(cls, data: object) -> ArrayMetadataModelV2: ) @classmethod - def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV2: + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ArrayMetadata: zarray = load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V2) if ATTRIBUTES_STORE_KEY_V2 in mapping: zattrs = load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 222b0b3bf9..dc544bd19e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -12,7 +12,7 @@ from zarr_metadata.model._array import ( ATTRIBUTES_STORE_KEY_V2, - ArrayMetadataModelV3, + ZarrV3ArrayMetadata, must_understand_subset, ) from zarr_metadata.model._sentinel import UNSET @@ -29,19 +29,19 @@ if TYPE_CHECKING: from zarr_metadata._common import JSONValue - from zarr_metadata.v2.group import GroupMetadataV2 - from zarr_metadata.v3.array import ExtensionFieldV3 - from zarr_metadata.v3.consolidated import ConsolidatedMetadataV3 - from zarr_metadata.v3.group import GroupMetadataV3 + from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON + from zarr_metadata.v3.array import ZarrV3ExtensionField + from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON + from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON -GroupMetadataStoreKeyV3 = Literal["zarr.json"] -GROUP_METADATA_STORE_KEY_V3: Final[GroupMetadataStoreKeyV3] = "zarr.json" +ZarrV3GroupMetadataStoreKey = Literal["zarr.json"] +GROUP_METADATA_STORE_KEY_V3: Final[ZarrV3GroupMetadataStoreKey] = "zarr.json" -GroupMetadataStoreKeyV2 = Literal[".zgroup"] -GROUP_METADATA_STORE_KEY_V2: Final[GroupMetadataStoreKeyV2] = ".zgroup" +ZarrV2GroupMetadataStoreKey = Literal[".zgroup"] +GROUP_METADATA_STORE_KEY_V2: Final[ZarrV2GroupMetadataStoreKey] = ".zgroup" -ConsolidatedMetadataStoreKeyV2 = Literal[".zmetadata"] -CONSOLIDATED_METADATA_STORE_KEY_V2: Final[ConsolidatedMetadataStoreKeyV2] = ".zmetadata" +ZarrV2ConsolidatedMetadataStoreKey = Literal[".zmetadata"] +CONSOLIDATED_METADATA_STORE_KEY_V2: Final[ZarrV2ConsolidatedMetadataStoreKey] = ".zmetadata" # The key under which consolidated metadata is embedded in a v3 group document. # This is a reference-implementation convention (not a spec artifact), stored @@ -49,12 +49,12 @@ CONSOLIDATED_METADATA_KEY_V3: Final = "consolidated_metadata" -class GroupMetadataModelV3Partial(TypedDict, total=False): +class ZarrV3GroupMetadataPartial(TypedDict, total=False): """ - Partial form of the constructor-settable fields of `GroupMetadataModelV3`. + Partial form of the constructor-settable fields of `ZarrV3GroupMetadata`. Every key is optional and typed with the model's own value types, so it - describes valid keyword arguments to `GroupMetadataModelV3.update` and + describes valid keyword arguments to `ZarrV3GroupMetadata.update` and `create_default`. The `init=False` fields `zarr_format` and `node_type` are intentionally excluded, since they cannot be passed to `dataclasses.replace`. @@ -64,12 +64,12 @@ class GroupMetadataModelV3Partial(TypedDict, total=False): """ attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | UNSET - extra_fields: dict[str, ExtensionFieldV3] + consolidated_metadata: ZarrV3ConsolidatedMetadata | UNSET + extra_fields: dict[str, ZarrV3ExtensionField] @dataclass(frozen=True, slots=True, kw_only=True) -class GroupMetadataModelV3: +class ZarrV3GroupMetadata: """In-memory model of a v3 group metadata document. A canonical, lossless representation of the `zarr.json` content for a @@ -81,8 +81,8 @@ class GroupMetadataModelV3: zarr_format: Literal[3] = field(default=3, init=False) node_type: Literal["group"] = field(default="group", init=False) attributes: dict[str, JSONValue] - consolidated_metadata: ConsolidatedMetadataModelV3 | UNSET - extra_fields: dict[str, ExtensionFieldV3] + consolidated_metadata: ZarrV3ConsolidatedMetadata | UNSET + extra_fields: dict[str, ZarrV3ExtensionField] def __post_init__(self) -> None: reserved = GROUP_METADATA_STANDARD_KEYS_V3 | {CONSOLIDATED_METADATA_KEY_V3} @@ -91,16 +91,14 @@ def __post_init__(self) -> None: [ ValidationProblem( ("extra_fields",), - "Extra fields cannot overlap with standard GroupMetadataV3 fields", + "Extra fields cannot overlap with standard Zarr V3 group metadata fields", "invalid_value", ) ] ) @classmethod - def create_default( - cls, **overrides: Unpack[GroupMetadataModelV3Partial] - ) -> GroupMetadataModelV3: + def create_default(cls, **overrides: Unpack[ZarrV3GroupMetadataPartial]) -> ZarrV3GroupMetadata: """ Create a default (empty) v3 group metadata model, with optional overrides. @@ -111,19 +109,19 @@ def create_default( default = cls(attributes={}, consolidated_metadata=UNSET, extra_fields={}) return default.update(**overrides) - def update(self, **kwargs: Unpack[GroupMetadataModelV3Partial]) -> GroupMetadataModelV3: + def update(self, **kwargs: Unpack[ZarrV3GroupMetadataPartial]) -> ZarrV3GroupMetadata: """ - Return a new `GroupMetadataModelV3` with the given fields updated. + Return a new `ZarrV3GroupMetadata` with the given fields updated. Only the constructor-settable fields listed in - `GroupMetadataModelV3Partial` can be updated; the fixed `zarr_format` / + `ZarrV3GroupMetadataPartial` can be updated; the fixed `zarr_format` / `node_type` are rejected at the type level. Each given field fully replaces its previous value, including `extra_fields`. """ return dataclasses.replace(self, **kwargs) - def to_json(self) -> GroupMetadataV3: - out: GroupMetadataV3 = { + def to_json(self) -> ZarrV3GroupMetadataJSON: + out: ZarrV3GroupMetadataJSON = { "zarr_format": self.zarr_format, "node_type": self.node_type, } @@ -132,34 +130,34 @@ def to_json(self) -> GroupMetadataV3: if self.consolidated_metadata is not UNSET: # The consolidated-metadata shape ({kind, must_understand, metadata}, # no `name`) predates the strict v3.1 extension-field rules, so it is - # not assignable to `ExtensionFieldV3`; see the discussion on + # not assignable to `ZarrV3ExtensionField`; see the discussion on # `zarr_metadata.v3.consolidated`. out[CONSOLIDATED_METADATA_KEY_V3] = cast( - "ExtensionFieldV3", self.consolidated_metadata.to_json() + "ZarrV3ExtensionField", self.consolidated_metadata.to_json() ) for key, value in self.extra_fields.items(): out[key] = value return out @classmethod - def from_json(cls, data: object) -> GroupMetadataModelV3: + def from_json(cls, data: object) -> ZarrV3GroupMetadata: parsed = parse_group_metadata_v3(arrays_to_tuples(data)) # Cast to object: the TypedDict's extra_items type does not admit null, # but wild documents (historical zarr-python) contain it. consolidated_raw = cast("object", parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET)) - consolidated: ConsolidatedMetadataModelV3 | UNSET + consolidated: ZarrV3ConsolidatedMetadata | UNSET if consolidated_raw is UNSET or consolidated_raw is None: # consolidated_metadata: null was written by a historical # zarr-python bug; it gets no model representation. It is read as # absence and never written back — repaired, not preserved. consolidated = UNSET else: - consolidated = ConsolidatedMetadataModelV3.from_json(consolidated_raw) + consolidated = ZarrV3ConsolidatedMetadata.from_json(consolidated_raw) # Sound cast: the TypedDict types all non-standard keys as its - # `extra_items` (`ExtensionFieldV3`); the comprehension's inferred value + # `extra_items` (`ZarrV3ExtensionField`); the comprehension's inferred value # type is the union over ALL keys because the key filter cannot narrow it. extra_fields = cast( - "dict[str, ExtensionFieldV3]", + "dict[str, ZarrV3ExtensionField]", { k: v for k, v in parsed.items() @@ -173,7 +171,7 @@ def from_json(cls, data: object) -> GroupMetadataModelV3: ) @property - def must_understand_fields(self) -> dict[str, ExtensionFieldV3]: + def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: """Extra fields the reader is obligated to understand. Everything in `extra_fields` not explicitly waived with @@ -185,7 +183,7 @@ def must_understand_fields(self) -> dict[str, ExtensionFieldV3]: return must_understand_subset(self.extra_fields) @classmethod - def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV3: + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3GroupMetadata: return cls.from_json(load_store_json(mapping, GROUP_METADATA_STORE_KEY_V3)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: @@ -195,7 +193,7 @@ def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes @dataclass(frozen=True, slots=True, kw_only=True) -class ConsolidatedMetadataModelV3: +class ZarrV3ConsolidatedMetadata: """In-memory model of v3 inline consolidated metadata. Models the reference-implementation convention where consolidated metadata @@ -207,7 +205,7 @@ class ConsolidatedMetadataModelV3: kind: Literal["inline"] = field(default="inline", init=False) must_understand: bool = False - metadata: dict[str, ArrayMetadataModelV3 | GroupMetadataModelV3] + metadata: dict[str, ZarrV3ArrayMetadata | ZarrV3GroupMetadata] def __post_init__(self) -> None: if self.must_understand is not False: @@ -222,7 +220,7 @@ def __post_init__(self) -> None: ] ) - def to_json(self) -> ConsolidatedMetadataV3: + def to_json(self) -> ZarrV3ConsolidatedMetadataJSON: # `must_understand` is emitted as the literal False: the field is typed # permissively as `bool`, but `__post_init__` guarantees the value. return { @@ -232,27 +230,27 @@ def to_json(self) -> ConsolidatedMetadataV3: } @classmethod - def from_json(cls, data: object) -> ConsolidatedMetadataModelV3: + def from_json(cls, data: object) -> ZarrV3ConsolidatedMetadata: problems = validate_consolidated_metadata_v3(data) if problems: raise MetadataValidationError(problems) env = cast("Mapping[str, object]", data) - entries: dict[str, ArrayMetadataModelV3 | GroupMetadataModelV3] = {} + entries: dict[str, ZarrV3ArrayMetadata | ZarrV3GroupMetadata] = {} for key, entry in cast("Mapping[str, object]", env["metadata"]).items(): node_type = cast("Mapping[str, object]", entry).get("node_type") if node_type == "array": - entries[key] = ArrayMetadataModelV3.from_json(entry) + entries[key] = ZarrV3ArrayMetadata.from_json(entry) else: - entries[key] = GroupMetadataModelV3.from_json(entry) + entries[key] = ZarrV3GroupMetadata.from_json(entry) return cls(metadata=entries) -class GroupMetadataModelV2Partial(TypedDict, total=False): +class ZarrV2GroupMetadataPartial(TypedDict, total=False): """ - Partial form of the constructor-settable fields of `GroupMetadataModelV2`. + Partial form of the constructor-settable fields of `ZarrV2GroupMetadata`. Every key is optional and typed with the model's own value types, so it - describes valid keyword arguments to `GroupMetadataModelV2.update` and + describes valid keyword arguments to `ZarrV2GroupMetadata.update` and `create_default`. The `init=False` field `zarr_format` is intentionally excluded, since it cannot be passed to `dataclasses.replace`. @@ -264,12 +262,12 @@ class GroupMetadataModelV2Partial(TypedDict, total=False): @dataclass(frozen=True, slots=True, kw_only=True) -class GroupMetadataModelV2: +class ZarrV2GroupMetadata: """In-memory model of a v2 group metadata document. A canonical, lossless representation of the `.zgroup` content plus the sibling `.zattrs` attributes, folded into a single in-memory value - (mirroring the merged `GroupMetadataV2` document form). `attributes` is + (mirroring the merged `ZarrV2GroupMetadataJSON` document form). `attributes` is `UNSET` when no `.zattrs` file (or merged `attributes` key) exists — distinct from an explicit empty `.zattrs`, which is `{}` and round-trips as a file. @@ -279,9 +277,7 @@ class GroupMetadataModelV2: attributes: dict[str, JSONValue] | UNSET @classmethod - def create_default( - cls, **overrides: Unpack[GroupMetadataModelV2Partial] - ) -> GroupMetadataModelV2: + def create_default(cls, **overrides: Unpack[ZarrV2GroupMetadataPartial]) -> ZarrV2GroupMetadata: """ Create a default (empty) v2 group metadata model, with optional overrides. @@ -292,18 +288,18 @@ def create_default( default = cls(attributes=UNSET) return default.update(**overrides) - def update(self, **kwargs: Unpack[GroupMetadataModelV2Partial]) -> GroupMetadataModelV2: + def update(self, **kwargs: Unpack[ZarrV2GroupMetadataPartial]) -> ZarrV2GroupMetadata: """ - Return a new `GroupMetadataModelV2` with the given fields updated. + Return a new `ZarrV2GroupMetadata` with the given fields updated. Only the constructor-settable fields listed in - `GroupMetadataModelV2Partial` can be updated; the fixed `zarr_format` + `ZarrV2GroupMetadataPartial` can be updated; the fixed `zarr_format` is rejected at the type level. Each given field fully replaces its previous value. """ return dataclasses.replace(self, **kwargs) - def to_json(self) -> GroupMetadataV2: + def to_json(self) -> ZarrV2GroupMetadataJSON: """Return the merged in-memory document form. `attributes` is included when set (even empty). This is not the @@ -311,18 +307,18 @@ def to_json(self) -> GroupMetadataV2: `attributes` (they live in the sibling `.zattrs` file). Use `to_key_value` to produce the spec-conforming split for storage. """ - out: GroupMetadataV2 = {"zarr_format": self.zarr_format} + out: ZarrV2GroupMetadataJSON = {"zarr_format": self.zarr_format} if self.attributes is not UNSET: out["attributes"] = self.attributes return out @classmethod - def from_json(cls, data: object) -> GroupMetadataModelV2: + def from_json(cls, data: object) -> ZarrV2GroupMetadata: parsed = parse_group_metadata_v2(arrays_to_tuples(data)) return cls(attributes=(dict(parsed["attributes"]) if "attributes" in parsed else UNSET)) @classmethod - def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV2: + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2GroupMetadata: zgroup = load_store_json(mapping, GROUP_METADATA_STORE_KEY_V2) if ATTRIBUTES_STORE_KEY_V2 in mapping: zattrs = load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) @@ -343,7 +339,7 @@ def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes @dataclass(frozen=True, slots=True, kw_only=True) -class ConsolidatedMetadataModelV2: +class ZarrV2ConsolidatedMetadata: """In-memory model of a v2 `.zmetadata` document. The `metadata` map holds the flat file-keyed entries (`"path/.zarray"`, @@ -363,7 +359,7 @@ def to_json(self) -> dict[str, JSONValue]: } @classmethod - def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: + def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata: if not isinstance(data, Mapping): raise MetadataValidationError( [ValidationProblem((), "expected a mapping", "invalid_type")] @@ -393,7 +389,7 @@ def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: return cls(metadata=entries_tupled) @classmethod - def from_key_value(cls, mapping: Mapping[str, bytes]) -> ConsolidatedMetadataModelV2: + def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ConsolidatedMetadata: return cls.from_json(load_store_json(mapping, CONSOLIDATED_METADATA_STORE_KEY_V2)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index fae09187a4..d2ef94da52 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -21,11 +21,11 @@ from typing_extensions import TypeIs from zarr_metadata._common import JSONValue -from zarr_metadata.v2.array import ArrayMetadataV2 -from zarr_metadata.v2.group import GroupMetadataV2 -from zarr_metadata.v3._common import MetadataV3 -from zarr_metadata.v3.array import ArrayMetadataV3 -from zarr_metadata.v3.group import GroupMetadataV3 +from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON ProblemKind = Literal["missing_key", "invalid_type", "invalid_value", "invalid_json"] """Machine-readable classification of a `ValidationProblem`. @@ -112,33 +112,33 @@ def parse_json(value: object) -> JSONValue: # this set is an extension field. Built from the TypedDict's required/optional # key sets (which resolve inherited keys, unlike `__annotations__`). ARRAY_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( - ArrayMetadataV3.__required_keys__ + ZarrV3ArrayMetadataJSON.__required_keys__ ) ARRAY_METADATA_OPTIONAL_KEYS_V3: Final[frozenset[str]] = frozenset( - ArrayMetadataV3.__optional_keys__ + ZarrV3ArrayMetadataJSON.__optional_keys__ ) ARRAY_METADATA_STANDARD_KEYS_V3: Final[frozenset[str]] = ( ARRAY_METADATA_REQUIRED_KEYS_V3 | ARRAY_METADATA_OPTIONAL_KEYS_V3 ) ARRAY_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( - ArrayMetadataV2.__required_keys__ + ZarrV2ArrayMetadataJSON.__required_keys__ ) # The standard top-level keys of a v3 group metadata document. Anything outside # this set is an extension field. GROUP_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( - GroupMetadataV3.__required_keys__ + ZarrV3GroupMetadataJSON.__required_keys__ ) GROUP_METADATA_OPTIONAL_KEYS_V3: Final[frozenset[str]] = frozenset( - GroupMetadataV3.__optional_keys__ + ZarrV3GroupMetadataJSON.__optional_keys__ ) GROUP_METADATA_STANDARD_KEYS_V3: Final[frozenset[str]] = ( GROUP_METADATA_REQUIRED_KEYS_V3 | GROUP_METADATA_OPTIONAL_KEYS_V3 ) GROUP_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( - GroupMetadataV2.__required_keys__ + ZarrV2GroupMetadataJSON.__required_keys__ ) @@ -196,17 +196,17 @@ def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: return problems -def is_metadata_field_v3(value: object) -> TypeIs[MetadataV3]: +def is_metadata_field_v3(value: object) -> TypeIs[ZarrV3MetadataFieldJSON]: """Whether `value` is a v3 metadata field: a bare name or a named config.""" return not validate_metadata_field_v3(value) -def parse_metadata_field_v3(value: object) -> MetadataV3: - """Return `value` narrowed to `MetadataV3`, or raise `MetadataValidationError`.""" +def parse_metadata_field_v3(value: object) -> ZarrV3MetadataFieldJSON: + """Return `value` narrowed to `ZarrV3MetadataFieldJSON`, or raise `MetadataValidationError`.""" problems = validate_metadata_field_v3(value) if problems: raise MetadataValidationError(problems) - return cast(MetadataV3, value) + return cast(ZarrV3MetadataFieldJSON, value) def _is_int_sequence(value: object) -> bool: @@ -355,17 +355,17 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: return problems -def is_array_metadata_v3(value: object) -> TypeIs[ArrayMetadataV3]: +def is_array_metadata_v3(value: object) -> TypeIs[ZarrV3ArrayMetadataJSON]: """Whether `value` is a structurally-valid v3 array metadata document.""" return not validate_array_metadata_v3(value) -def parse_array_metadata_v3(value: object) -> ArrayMetadataV3: - """Return `value` narrowed to `ArrayMetadataV3`, or raise `MetadataValidationError`.""" +def parse_array_metadata_v3(value: object) -> ZarrV3ArrayMetadataJSON: + """Return `value` narrowed to `ZarrV3ArrayMetadataJSON`, or raise `MetadataValidationError`.""" problems = validate_array_metadata_v3(value) if problems: raise MetadataValidationError(problems) - return cast(ArrayMetadataV3, value) + return cast(ZarrV3ArrayMetadataJSON, value) def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: @@ -436,17 +436,17 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: return problems -def is_array_metadata_v2(value: object) -> TypeIs[ArrayMetadataV2]: +def is_array_metadata_v2(value: object) -> TypeIs[ZarrV2ArrayMetadataJSON]: """Whether `value` is a structurally-valid v2 array metadata document.""" return not validate_array_metadata_v2(value) -def parse_array_metadata_v2(value: object) -> ArrayMetadataV2: - """Return `value` narrowed to `ArrayMetadataV2`, or raise `MetadataValidationError`.""" +def parse_array_metadata_v2(value: object) -> ZarrV2ArrayMetadataJSON: + """Return `value` narrowed to `ZarrV2ArrayMetadataJSON`, or raise `MetadataValidationError`.""" problems = validate_array_metadata_v2(value) if problems: raise MetadataValidationError(problems) - return cast(ArrayMetadataV2, value) + return cast(ZarrV2ArrayMetadataJSON, value) def validate_consolidated_metadata_v3(value: object) -> list[ValidationProblem]: @@ -455,7 +455,7 @@ def validate_consolidated_metadata_v3(value: object) -> list[ValidationProblem]: Locs are value-relative (the caller prefixes with `consolidated_metadata` where appropriate). Entries recurse into the array and group document validators, so a validator verdict always agrees with what - `ConsolidatedMetadataModelV3.from_json` accepts. + `ZarrV3ConsolidatedMetadata.from_json` accepts. """ if not isinstance(value, Mapping): return [ValidationProblem((), "expected a mapping", "invalid_type")] @@ -531,17 +531,17 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: return problems -def is_group_metadata_v3(value: object) -> TypeIs[GroupMetadataV3]: +def is_group_metadata_v3(value: object) -> TypeIs[ZarrV3GroupMetadataJSON]: """Whether `value` is a structurally-valid v3 group metadata document.""" return not validate_group_metadata_v3(value) -def parse_group_metadata_v3(value: object) -> GroupMetadataV3: - """Return `value` narrowed to `GroupMetadataV3`, or raise `MetadataValidationError`.""" +def parse_group_metadata_v3(value: object) -> ZarrV3GroupMetadataJSON: + """Return `value` narrowed to `ZarrV3GroupMetadataJSON`, or raise `MetadataValidationError`.""" problems = validate_group_metadata_v3(value) if problems: raise MetadataValidationError(problems) - return cast(GroupMetadataV3, value) + return cast(ZarrV3GroupMetadataJSON, value) def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: @@ -560,17 +560,17 @@ def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: return problems -def is_group_metadata_v2(value: object) -> TypeIs[GroupMetadataV2]: +def is_group_metadata_v2(value: object) -> TypeIs[ZarrV2GroupMetadataJSON]: """Whether `value` is a structurally-valid v2 group metadata document.""" return not validate_group_metadata_v2(value) -def parse_group_metadata_v2(value: object) -> GroupMetadataV2: - """Return `value` narrowed to `GroupMetadataV2`, or raise `MetadataValidationError`.""" +def parse_group_metadata_v2(value: object) -> ZarrV2GroupMetadataJSON: + """Return `value` narrowed to `ZarrV2GroupMetadataJSON`, or raise `MetadataValidationError`.""" problems = validate_group_metadata_v2(value) if problems: raise MetadataValidationError(problems) - return cast(GroupMetadataV2, value) + return cast(ZarrV2GroupMetadataJSON, value) def load_store_json(mapping: Mapping[str, bytes], key: str) -> Any: diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index 3eada76316..8b88b49128 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -20,10 +20,10 @@ class ArrayManifest(BaseModel): path: str - metadata: zmp.ArrayMetadataV3 + metadata: zmp.ZarrV3ArrayMetadata Static type checkers see each field type as its core model class, so -`manifest.metadata` is an `ArrayMetadataModelV3`. +`manifest.metadata` is a `zarr_metadata.model.ZarrV3ArrayMetadata`. """ from __future__ import annotations @@ -32,15 +32,7 @@ class ArrayManifest(BaseModel): from pydantic import BeforeValidator, InstanceOf, PlainSerializer, WithJsonSchema -from zarr_metadata.model import ( - ArrayMetadataModelV2, - ArrayMetadataModelV3, - ConsolidatedMetadataModelV2, - ConsolidatedMetadataModelV3, - GroupMetadataModelV2, - GroupMetadataModelV3, - NamedConfigModelV3, -) +from zarr_metadata import model as _model if TYPE_CHECKING: from collections.abc import Callable @@ -64,68 +56,72 @@ def coerce(value: object) -> _M: _DOCUMENT_SCHEMA = {"type": "object"} _FIELD_SCHEMA = {"anyOf": [{"type": "string"}, {"type": "object"}]} -ArrayMetadataV3 = Annotated[ - InstanceOf[ArrayMetadataModelV3], - BeforeValidator(_coerce_to(ArrayMetadataModelV3, ArrayMetadataModelV3.from_json)), - PlainSerializer(ArrayMetadataModelV3.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ArrayMetadataV3"}), +ZarrV3ArrayMetadata = Annotated[ + InstanceOf[_model.ZarrV3ArrayMetadata], + BeforeValidator(_coerce_to(_model.ZarrV3ArrayMetadata, _model.ZarrV3ArrayMetadata.from_json)), + PlainSerializer(_model.ZarrV3ArrayMetadata.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV3ArrayMetadata"}), ] """Field type for a v3 array metadata document (`zarr.json` content).""" -ArrayMetadataV2 = Annotated[ - InstanceOf[ArrayMetadataModelV2], - BeforeValidator(_coerce_to(ArrayMetadataModelV2, ArrayMetadataModelV2.from_json)), - PlainSerializer(ArrayMetadataModelV2.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ArrayMetadataV2"}), +ZarrV2ArrayMetadata = Annotated[ + InstanceOf[_model.ZarrV2ArrayMetadata], + BeforeValidator(_coerce_to(_model.ZarrV2ArrayMetadata, _model.ZarrV2ArrayMetadata.from_json)), + PlainSerializer(_model.ZarrV2ArrayMetadata.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV2ArrayMetadata"}), ] """Field type for a v2 array metadata document (merged `.zarray` + `.zattrs` form).""" -GroupMetadataV3 = Annotated[ - InstanceOf[GroupMetadataModelV3], - BeforeValidator(_coerce_to(GroupMetadataModelV3, GroupMetadataModelV3.from_json)), - PlainSerializer(GroupMetadataModelV3.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "GroupMetadataV3"}), +ZarrV3GroupMetadata = Annotated[ + InstanceOf[_model.ZarrV3GroupMetadata], + BeforeValidator(_coerce_to(_model.ZarrV3GroupMetadata, _model.ZarrV3GroupMetadata.from_json)), + PlainSerializer(_model.ZarrV3GroupMetadata.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV3GroupMetadata"}), ] """Field type for a v3 group metadata document (`zarr.json` content).""" -GroupMetadataV2 = Annotated[ - InstanceOf[GroupMetadataModelV2], - BeforeValidator(_coerce_to(GroupMetadataModelV2, GroupMetadataModelV2.from_json)), - PlainSerializer(GroupMetadataModelV2.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "GroupMetadataV2"}), +ZarrV2GroupMetadata = Annotated[ + InstanceOf[_model.ZarrV2GroupMetadata], + BeforeValidator(_coerce_to(_model.ZarrV2GroupMetadata, _model.ZarrV2GroupMetadata.from_json)), + PlainSerializer(_model.ZarrV2GroupMetadata.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV2GroupMetadata"}), ] """Field type for a v2 group metadata document (merged `.zgroup` + `.zattrs` form).""" -ConsolidatedMetadataV3 = Annotated[ - InstanceOf[ConsolidatedMetadataModelV3], - BeforeValidator(_coerce_to(ConsolidatedMetadataModelV3, ConsolidatedMetadataModelV3.from_json)), - PlainSerializer(ConsolidatedMetadataModelV3.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ConsolidatedMetadataV3"}), +ZarrV3ConsolidatedMetadata = Annotated[ + InstanceOf[_model.ZarrV3ConsolidatedMetadata], + BeforeValidator( + _coerce_to(_model.ZarrV3ConsolidatedMetadata, _model.ZarrV3ConsolidatedMetadata.from_json) + ), + PlainSerializer(_model.ZarrV3ConsolidatedMetadata.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV3ConsolidatedMetadata"}), ] """Field type for v3 inline consolidated metadata.""" -ConsolidatedMetadataV2 = Annotated[ - InstanceOf[ConsolidatedMetadataModelV2], - BeforeValidator(_coerce_to(ConsolidatedMetadataModelV2, ConsolidatedMetadataModelV2.from_json)), - PlainSerializer(ConsolidatedMetadataModelV2.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ConsolidatedMetadataV2"}), +ZarrV2ConsolidatedMetadata = Annotated[ + InstanceOf[_model.ZarrV2ConsolidatedMetadata], + BeforeValidator( + _coerce_to(_model.ZarrV2ConsolidatedMetadata, _model.ZarrV2ConsolidatedMetadata.from_json) + ), + PlainSerializer(_model.ZarrV2ConsolidatedMetadata.to_json, return_type=dict), + WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV2ConsolidatedMetadata"}), ] """Field type for a v2 `.zmetadata` document.""" -MetadataFieldV3 = Annotated[ - InstanceOf[NamedConfigModelV3], - BeforeValidator(_coerce_to(NamedConfigModelV3, NamedConfigModelV3.from_json)), - PlainSerializer(NamedConfigModelV3.to_json, return_type=dict), - WithJsonSchema(_FIELD_SCHEMA | {"title": "MetadataFieldV3"}), +ZarrV3MetadataField = Annotated[ + InstanceOf[_model.ZarrV3NamedConfig], + BeforeValidator(_coerce_to(_model.ZarrV3NamedConfig, _model.ZarrV3NamedConfig.from_json)), + PlainSerializer(_model.ZarrV3NamedConfig.to_json, return_type=dict), + WithJsonSchema(_FIELD_SCHEMA | {"title": "ZarrV3MetadataField"}), ] """Field type for one v3 metadata field (bare name string or name + configuration).""" __all__ = [ - "ArrayMetadataV2", - "ArrayMetadataV3", - "ConsolidatedMetadataV2", - "ConsolidatedMetadataV3", - "GroupMetadataV2", - "GroupMetadataV3", - "MetadataFieldV3", + "ZarrV2ArrayMetadata", + "ZarrV2ConsolidatedMetadata", + "ZarrV2GroupMetadata", + "ZarrV3ArrayMetadata", + "ZarrV3ConsolidatedMetadata", + "ZarrV3GroupMetadata", + "ZarrV3MetadataField", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py index 4e9a76125b..62dd8168ea 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py @@ -1,26 +1,26 @@ """Zarr v2 metadata types.""" from zarr_metadata.v2.array import ( - ArrayDimensionSeparatorV2, - ArrayMetadataV2, - ArrayOrderV2, - DataTypeMetadataV2, ZArrayMetadata, + ZarrV2ArrayDimensionSeparator, + ZarrV2ArrayMetadataJSON, + ZarrV2ArrayOrder, + ZarrV2DataTypeMetadata, ) from zarr_metadata.v2.attributes import ZAttrsMetadata -from zarr_metadata.v2.codec import CodecMetadataV2 -from zarr_metadata.v2.consolidated import ConsolidatedMetadataV2 -from zarr_metadata.v2.group import GroupMetadataV2, ZGroupMetadata +from zarr_metadata.v2.codec import ZarrV2CodecMetadata +from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON, ZGroupMetadata __all__ = [ - "ArrayDimensionSeparatorV2", - "ArrayMetadataV2", - "ArrayOrderV2", - "CodecMetadataV2", - "ConsolidatedMetadataV2", - "DataTypeMetadataV2", - "GroupMetadataV2", "ZArrayMetadata", "ZAttrsMetadata", "ZGroupMetadata", + "ZarrV2ArrayDimensionSeparator", + "ZarrV2ArrayMetadataJSON", + "ZarrV2ArrayOrder", + "ZarrV2CodecMetadata", + "ZarrV2ConsolidatedMetadataJSON", + "ZarrV2DataTypeMetadata", + "ZarrV2GroupMetadataJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/array.py b/packages/zarr-metadata/src/zarr_metadata/v2/array.py index 999c341dc7..741f37acba 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/array.py @@ -6,9 +6,9 @@ from typing_extensions import TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v2.codec import CodecMetadataV2 +from zarr_metadata.v2.codec import ZarrV2CodecMetadata -DataTypeMetadataV2 = str | tuple[tuple[str, str] | tuple[str, str, tuple[int, ...]], ...] +ZarrV2DataTypeMetadata = str | tuple[tuple[str, str] | tuple[str, str, tuple[int, ...]], ...] """The v2 dtype representation. Either a numpy-style dtype string (e.g. `"CodecConfiguration`, etc., import directly from the leaf submodule. For the field-level "any codec entry" alias (used in array metadata's -`codecs` list and in sharding's inner pipelines), import `MetadataV3` +`codecs` list and in sharding's inner pipelines), import `ZarrV3MetadataFieldJSON` from `zarr_metadata.v3`. See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/index.html diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 7e9b071669..96c39e5916 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -9,7 +9,7 @@ from typing_extensions import TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v3._common import MetadataV3 +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON CAST_VALUE_CODEC_NAME: Final = "cast_value" """The `name` field value of the `cast_value` codec.""" @@ -71,7 +71,7 @@ class CastValueCodecConfiguration(TypedDict): bare-string primitive name or a `{name, configuration}` envelope. """ - data_type: MetadataV3 + data_type: ZarrV3MetadataFieldJSON rounding: NotRequired[CastRoundingMode] out_of_range: NotRequired[CastOutOfRangeMode] scalar_map: NotRequired[ScalarMap] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index a1488f7c30..a8c9247ec4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -8,7 +8,7 @@ from typing_extensions import TypedDict -from zarr_metadata.v3._common import MetadataV3 +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON SHARDING_INDEXED_CODEC_NAME: Final = "sharding_indexed" """The `name` field value of the `sharding_indexed` codec.""" @@ -40,8 +40,8 @@ class ShardingIndexedCodecConfiguration(TypedDict): """ chunk_shape: tuple[int, ...] - codecs: tuple[MetadataV3, ...] - index_codecs: tuple[MetadataV3, ...] + codecs: tuple[ZarrV3MetadataFieldJSON, ...] + index_codecs: tuple[ZarrV3MetadataFieldJSON, ...] index_location: NotRequired[ShardingIndexLocation] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py index 486a0897a5..5419049bad 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py @@ -10,7 +10,7 @@ rules. Under the strict Zarr v3.1 reading, every extension field must also include a `name: str` key, which would make this shape — and every real-world consolidated metadata document in the wild — out of spec. -See `ExtensionFieldV3` and +See `ZarrV3ExtensionField` and https://github.com/zarr-developers/zarr-specs/issues/371 for the ongoing discussion. """ @@ -20,11 +20,11 @@ from typing_extensions import TypedDict -from zarr_metadata.v3.array import ArrayMetadataV3 -from zarr_metadata.v3.group import GroupMetadataV3 +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON -class ConsolidatedMetadataV3(TypedDict): +class ZarrV3ConsolidatedMetadataJSON(TypedDict): """ Inline consolidated metadata embedded in a v3 group. @@ -35,9 +35,9 @@ class ConsolidatedMetadataV3(TypedDict): kind: Literal["inline"] must_understand: Literal[False] - metadata: Mapping[str, ArrayMetadataV3 | GroupMetadataV3] + metadata: Mapping[str, ZarrV3ArrayMetadataJSON | ZarrV3GroupMetadataJSON] __all__ = [ - "ConsolidatedMetadataV3", + "ZarrV3ConsolidatedMetadataJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 5291e5c309..b1b6b50308 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -10,7 +10,7 @@ from typing_extensions import ReadOnly, TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v3._common import MetadataV3 +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON STRUCT_DATA_TYPE_NAME: Final = "struct" """The `name` field value of the `struct` data type.""" @@ -33,7 +33,7 @@ class StructField(TypedDict): """ name: ReadOnly[str] - data_type: ReadOnly[MetadataV3] + data_type: ReadOnly[ZarrV3MetadataFieldJSON] class StructConfiguration(TypedDict): diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/group.py b/packages/zarr-metadata/src/zarr_metadata/v3/group.py index be990b1ae7..029a530e6a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/group.py @@ -9,14 +9,14 @@ from typing_extensions import TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.v3.array import ExtensionFieldV3 +from zarr_metadata.v3.array import ZarrV3ExtensionField -class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): +class ZarrV3GroupMetadataJSON(TypedDict, extra_items=ZarrV3ExtensionField): """ Zarr v3 group metadata document (the `zarr.json` content for a group). - Extra keys are permitted if they conform to `ExtensionFieldV3`. + Extra keys are permitted if they conform to `ZarrV3ExtensionField`. See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#group-metadata """ @@ -26,11 +26,11 @@ class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): attributes: NotRequired[Mapping[str, JSONValue]] -class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): +class ZarrV3GroupMetadataJSONPartial(TypedDict, total=False, extra_items=ZarrV3ExtensionField): """ - Partial form of `GroupMetadataV3`: every field is `NotRequired`. + Partial form of `ZarrV3GroupMetadataJSON`: every field is `NotRequired`. - Field annotations and `extra_items=` mirror `GroupMetadataV3` exactly. + Field annotations and `extra_items=` mirror `ZarrV3GroupMetadataJSON` exactly. The only difference is `total=False`, which makes every key optional at the type level. @@ -40,12 +40,12 @@ class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV into a complete document elsewhere. The `NotRequired[...]` wrapper on `attributes` is intentional: keeping it - preserves byte-identical `__annotations__` with `GroupMetadataV3` so the + preserves byte-identical `__annotations__` with `ZarrV3GroupMetadataJSON` so the `==` check in `tests/test_partial_equivalence.py` passes without special-casing that field (PEP 655 explicitly permits `NotRequired` inside `total=False`). - Drift between this type and `GroupMetadataV3` is prevented by + Drift between this type and `ZarrV3GroupMetadataJSON` is prevented by `tests/test_partial_equivalence.py`. """ @@ -55,6 +55,6 @@ class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV __all__ = [ - "GroupMetadataV3", - "GroupMetadataV3Partial", + "ZarrV3GroupMetadataJSON", + "ZarrV3GroupMetadataJSONPartial", ] diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index ff27923635..6e5029a8fd 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -14,14 +14,14 @@ ARRAY_METADATA_REQUIRED_KEYS_V3, ARRAY_METADATA_STANDARD_KEYS_V3, UNSET, - ArrayMetadataModelV2, - ArrayMetadataModelV2Partial, - ArrayMetadataModelV3, - ArrayMetadataModelV3Partial, - MetadataFieldModelV3, MetadataValidationError, - NamedConfigModelV3, ValidationProblem, + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadataPartial, + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadataPartial, + ZarrV3MetadataField, + ZarrV3NamedConfig, is_array_metadata_v2, is_array_metadata_v3, is_json, @@ -39,7 +39,7 @@ if TYPE_CHECKING: from zarr_metadata._common import JSONValue - from zarr_metadata.v2 import CodecMetadataV2 + from zarr_metadata.v2 import ZarrV2CodecMetadata # --- public exports -------------------------------------------------------- @@ -89,11 +89,11 @@ def test_expect_expectfail_smoke() -> None: def test_v3_from_json_error_lists_all_problems() -> None: """A malformed v3 document surfaces every problem via MetadataValidationError.problems.""" - doc: dict[str, object] = dict(ArrayMetadataModelV3.create_default().to_json()) + doc: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) del doc["shape"] doc["data_type"] = 5 with pytest.raises(MetadataValidationError) as exc_info: - ArrayMetadataModelV3.from_json(doc) + ZarrV3ArrayMetadata.from_json(doc) locs = {p.loc for p in exc_info.value.problems} assert ("shape",) in locs assert ("data_type",) in locs @@ -117,21 +117,21 @@ def test_string_nan_fill_value_roundtrips() -> None: # unlike a raw float('nan') (which is an invalid fill_value the caller must # not pass). """A string 'NaN' fill_value round-trips cleanly (non-finite floats are the caller's responsibility).""" - m = ArrayMetadataModelV3.create_default(fill_value="NaN") - assert ArrayMetadataModelV3.from_json(m.to_json()) == m - assert ArrayMetadataModelV3.from_json(m.to_json()).fill_value == "NaN" + m = ZarrV3ArrayMetadata.create_default(fill_value="NaN") + assert ZarrV3ArrayMetadata.from_json(m.to_json()) == m + assert ZarrV3ArrayMetadata.from_json(m.to_json()).fill_value == "NaN" -# --- NamedConfigModelV3.to_json ------------------------------------------------ +# --- ZarrV3NamedConfig.to_json ------------------------------------------------ ZARR_TO_JSON_CASES = [ Expect( - NamedConfigModelV3(name="regular", configuration={"chunk_shape": [1]}), + ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": [1]}), {"name": "regular", "configuration": {"chunk_shape": [1]}}, id="with-configuration", ), Expect( - NamedConfigModelV3(name="bytes", configuration={}), + ZarrV3NamedConfig(name="bytes", configuration={}), {"name": "bytes", "configuration": {}}, id="without-configuration", ), @@ -139,32 +139,32 @@ def test_string_nan_fill_value_roundtrips() -> None: @pytest.mark.parametrize("case", ZARR_TO_JSON_CASES, ids=lambda c: c.id) -def test_zarr_metadata_v3_to_json(case: Expect[NamedConfigModelV3, dict[str, object]]) -> None: - """NamedConfigModelV3.to_json emits the canonical object form.""" +def test_zarr_metadata_v3_to_json(case: Expect[ZarrV3NamedConfig, dict[str, object]]) -> None: + """ZarrV3NamedConfig.to_json emits the canonical object form.""" assert case.input.to_json() == case.output -# --- NamedConfigModelV3.from_json ----------------------------------------------- +# --- ZarrV3NamedConfig.from_json ----------------------------------------------- ZARR_FROM_JSON_CASES = [ - Expect("bytes", NamedConfigModelV3(name="bytes", configuration={}), id="bare-string"), + Expect("bytes", ZarrV3NamedConfig(name="bytes", configuration={}), id="bare-string"), Expect( {"name": "regular", "configuration": {"chunk_shape": [1]}}, - NamedConfigModelV3(name="regular", configuration={"chunk_shape": (1,)}), + ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": (1,)}), id="object-with-config", ), Expect( {"name": "bytes"}, - NamedConfigModelV3(name="bytes", configuration={}), + ZarrV3NamedConfig(name="bytes", configuration={}), id="object-without-config", ), ] @pytest.mark.parametrize("case", ZARR_FROM_JSON_CASES, ids=lambda c: c.id) -def test_zarr_metadata_v3_from_json(case: Expect[object, NamedConfigModelV3]) -> None: - """NamedConfigModelV3.from_json parses both the bare-string and object forms.""" - assert NamedConfigModelV3.from_json(case.input) == case.output +def test_zarr_metadata_v3_from_json(case: Expect[object, ZarrV3NamedConfig]) -> None: + """ZarrV3NamedConfig.from_json parses both the bare-string and object forms.""" + assert ZarrV3NamedConfig.from_json(case.input) == case.output # --- V3 baseline ----------------------------------------------------------- @@ -173,8 +173,8 @@ def test_zarr_metadata_v3_from_json(case: Expect[object, NamedConfigModelV3]) -> def test_v3_to_json_emits_canonical_document() -> None: """V3 to_json emits exactly the expected document (which covers every spec-required key by construction).""" - out = ArrayMetadataModelV3.create_default( - shape=(10,), data_type=NamedConfigModelV3(name="int32", configuration={}) + out = ZarrV3ArrayMetadata.create_default( + shape=(10,), data_type=ZarrV3NamedConfig(name="int32", configuration={}) ).to_json() assert out == { "zarr_format": 3, @@ -191,14 +191,14 @@ def test_v3_to_json_emits_canonical_document() -> None: def test_v3_dimension_names_included_when_present() -> None: """V3 to_json includes dimension_names when they are set.""" out: dict[str, object] = dict( - ArrayMetadataModelV3.create_default(dimension_names=("x",)).to_json() + ZarrV3ArrayMetadata.create_default(dimension_names=("x",)).to_json() ) assert out["dimension_names"] == ("x",) def test_v3_dimension_names_omitted_when_none() -> None: """V3 to_json omits dimension_names when they are UNSET.""" - out = ArrayMetadataModelV3.create_default(dimension_names=UNSET).to_json() + out = ZarrV3ArrayMetadata.create_default(dimension_names=UNSET).to_json() assert "dimension_names" not in out @@ -213,7 +213,7 @@ def test_v3_attributes_included_when_dimension_names_is_none() -> None: dimension names. """ out: dict[str, object] = dict( - ArrayMetadataModelV3.create_default( + ZarrV3ArrayMetadata.create_default( dimension_names=UNSET, attributes={"foo": "bar"} ).to_json() ) @@ -229,16 +229,16 @@ def test_v3_single_storage_transformer_included() -> None: Regression: the guard used ``> 1`` instead of ``> 0``, dropping a lone storage transformer. """ - st = NamedConfigModelV3(name="some_transformer", configuration={}) + st = ZarrV3NamedConfig(name="some_transformer", configuration={}) out: dict[str, object] = dict( - ArrayMetadataModelV3.create_default(storage_transformers=(st,)).to_json() + ZarrV3ArrayMetadata.create_default(storage_transformers=(st,)).to_json() ) assert out["storage_transformers"] == ({"name": "some_transformer", "configuration": {}},) def test_v3_no_storage_transformers_omitted() -> None: """V3 to_json omits storage_transformers when empty.""" - out = ArrayMetadataModelV3.create_default(storage_transformers=()).to_json() + out = ZarrV3ArrayMetadata.create_default(storage_transformers=()).to_json() assert "storage_transformers" not in out @@ -247,7 +247,7 @@ def test_v3_no_storage_transformers_omitted() -> None: def test_v3_extra_fields_merged() -> None: """V3 to_json merges extra_fields into the top-level document.""" - out = ArrayMetadataModelV3.create_default( + out = ZarrV3ArrayMetadata.create_default( extra_fields={"my_ext": {"must_understand": False}} ).to_json() assert out["my_ext"] == {"must_understand": False} @@ -256,7 +256,7 @@ def test_v3_extra_fields_merged() -> None: def test_v3_extra_fields_overlapping_standard_field_rejected() -> None: """Constructing a V3 model with an extra field that collides with a standard key is rejected.""" with pytest.raises(ValueError): - ArrayMetadataModelV3.create_default(extra_fields={"shape": {"must_understand": False}}) + ZarrV3ArrayMetadata.create_default(extra_fields={"shape": {"must_understand": False}}) # --- V3 key/value ---------------------------------------------------------- @@ -264,7 +264,7 @@ def test_v3_extra_fields_overlapping_standard_field_rejected() -> None: def test_v3_to_key_value_is_valid_json_under_zarr_json() -> None: """V3 to_key_value produces valid JSON bytes under the zarr.json key.""" - kv = ArrayMetadataModelV3.create_default(attributes={"a": 1}).to_key_value() + kv = ZarrV3ArrayMetadata.create_default(attributes={"a": 1}).to_key_value() assert set(kv) == {"zarr.json"} parsed = json.loads(kv["zarr.json"].decode("utf-8")) assert parsed["zarr_format"] == 3 @@ -298,29 +298,29 @@ def test_standard_keys_contains_known_fields_and_excludes_extensions() -> None: def test_v3_create_default_is_valid_empty_array() -> None: """V3 create_default builds a structurally valid empty array that round-trips.""" - m = ArrayMetadataModelV3.create_default() + m = ZarrV3ArrayMetadata.create_default() assert m.shape == () - assert m.data_type == NamedConfigModelV3(name="uint8", configuration={}) + assert m.data_type == ZarrV3NamedConfig(name="uint8", configuration={}) assert m.fill_value == 0 assert m.attributes == {} assert m.extra_fields == {} # the default document is structurally valid and round-trips assert validate_array_metadata_v3(m.to_json()) == [] - assert ArrayMetadataModelV3.from_json(m.to_json()) == m + assert ZarrV3ArrayMetadata.from_json(m.to_json()) == m def test_v3_create_default_applies_overrides() -> None: """V3 create_default applies keyword overrides over the defaults.""" - m = ArrayMetadataModelV3.create_default(shape=(4, 4), attributes={"a": 1}) + m = ZarrV3ArrayMetadata.create_default(shape=(4, 4), attributes={"a": 1}) assert m.shape == (4, 4) assert m.attributes == {"a": 1} # un-overridden fields keep their defaults - assert m.data_type == NamedConfigModelV3(name="uint8", configuration={}) + assert m.data_type == ZarrV3NamedConfig(name="uint8", configuration={}) def test_v2_create_default_is_valid_empty_array() -> None: """V2 create_default builds a structurally valid empty array that round-trips.""" - m = ArrayMetadataModelV2.create_default() + m = ZarrV2ArrayMetadata.create_default() assert m.shape == () assert m.chunks == () assert m.fill_value == 0 @@ -328,12 +328,12 @@ def test_v2_create_default_is_valid_empty_array() -> None: assert m.filters is None assert m.attributes is UNSET assert validate_array_metadata_v2(m.to_json()) == [] - assert ArrayMetadataModelV2.from_json(m.to_json()) == m + assert ZarrV2ArrayMetadata.from_json(m.to_json()) == m def test_v2_create_default_applies_overrides() -> None: """V2 create_default applies keyword overrides over the defaults.""" - m = ArrayMetadataModelV2.create_default(shape=(8,), attributes={"k": "v"}) + m = ZarrV2ArrayMetadata.create_default(shape=(8,), attributes={"k": "v"}) assert m.shape == (8,) assert m.attributes == {"k": "v"} assert m.dtype == "|u1" # default dtype unchanged @@ -344,14 +344,14 @@ def test_v2_create_default_applies_overrides() -> None: # Cluster 3: update same-shape pairs across versions — parametrized UPDATE_NEW_INSTANCE_PARAMS = [ - pytest.param(ArrayMetadataModelV3, id="v3"), - pytest.param(ArrayMetadataModelV2, id="v2"), + pytest.param(ZarrV3ArrayMetadata, id="v3"), + pytest.param(ZarrV2ArrayMetadata, id="v2"), ] @pytest.mark.parametrize("model_cls", UPDATE_NEW_INSTANCE_PARAMS) def test_update_returns_new_instance( - model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], ) -> None: """update returns a new instance with the field replaced, leaving the original unchanged.""" base = model_cls.create_default(shape=(10,)) @@ -362,14 +362,14 @@ def test_update_returns_new_instance( UPDATE_NO_ARGS_PARAMS = [ - pytest.param(ArrayMetadataModelV3, id="v3"), - pytest.param(ArrayMetadataModelV2, id="v2"), + pytest.param(ZarrV3ArrayMetadata, id="v3"), + pytest.param(ZarrV2ArrayMetadata, id="v2"), ] @pytest.mark.parametrize("model_cls", UPDATE_NO_ARGS_PARAMS) def test_update_no_args_returns_equal_model( - model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], ) -> None: """update with no arguments returns a model equal to the original.""" base = model_cls.create_default() @@ -381,14 +381,14 @@ def test_update_no_args_returns_equal_model( def test_update_can_replace_extra_fields() -> None: """update can replace the extra_fields mapping.""" - base = ArrayMetadataModelV3.create_default(extra_fields={}) + base = ZarrV3ArrayMetadata.create_default(extra_fields={}) updated = base.update(extra_fields={"my_ext": {"must_understand": False}}) assert updated.extra_fields == {"my_ext": {"must_understand": False}} def test_update_replaces_extra_fields_rather_than_merging() -> None: """update replaces extra_fields wholesale rather than merging.""" - base = ArrayMetadataModelV3.create_default(extra_fields={"a": {"must_understand": False}}) + base = ZarrV3ArrayMetadata.create_default(extra_fields={"a": {"must_understand": False}}) updated = base.update(extra_fields={"b": {"must_understand": True}}) assert updated.extra_fields == {"b": {"must_understand": True}} @@ -397,10 +397,10 @@ def test_partial_keys_match_settable_model_fields() -> None: """The partial TypedDict must list exactly the constructor-settable fields. Guards against drift: adding/removing a settable field on the model - without updating ``ArrayMetadataModelV3Partial`` fails here. + without updating ``ZarrV3ArrayMetadataPartial`` fails here. """ - settable = {f.name for f in dataclasses.fields(ArrayMetadataModelV3) if f.init} - assert set(ArrayMetadataModelV3Partial.__annotations__) == settable + settable = {f.name for f in dataclasses.fields(ZarrV3ArrayMetadata) if f.init} + assert set(ZarrV3ArrayMetadataPartial.__annotations__) == settable # --- V2 model -------------------------------------------------------------- @@ -408,13 +408,13 @@ def test_partial_keys_match_settable_model_fields() -> None: def test_v2_partial_keys_match_settable_model_fields() -> None: """The v2 partial TypedDict must list exactly the settable fields.""" - settable = {f.name for f in dataclasses.fields(ArrayMetadataModelV2) if f.init} - assert set(ArrayMetadataModelV2Partial.__annotations__) == settable + settable = {f.name for f in dataclasses.fields(ZarrV2ArrayMetadata) if f.init} + assert set(ZarrV2ArrayMetadataPartial.__annotations__) == settable def test_v2_to_key_value_splits_zarray_and_zattrs() -> None: """V2 to_key_value splits the document into .zarray and .zattrs.""" - kv = ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_key_value() + kv = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_key_value() assert set(kv) == {".zarray", ".zattrs"} zarray = json.loads(kv[".zarray"].decode("utf-8")) zattrs = json.loads(kv[".zattrs"].decode("utf-8")) @@ -426,19 +426,17 @@ def test_v2_zarray_excludes_attributes() -> None: """The on-disk ``.zarray`` document must not contain user attributes. In v2, attributes live only in the sibling ``.zattrs`` file. The bundled - ``ArrayMetadataV2`` / ``to_json()`` carry attributes for convenience, but + ``ZarrV2ArrayMetadataJSON`` / ``to_json()`` carry attributes for convenience, but ``to_key_value()`` must split them out. """ - kv = ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_key_value() + kv = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_key_value() zarray = json.loads(kv[".zarray"].decode("utf-8")) assert "attributes" not in zarray def test_v2_to_json_still_includes_attributes() -> None: """``to_json()`` is the bundled in-memory form and keeps attributes.""" - out: dict[str, object] = dict( - ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_json() - ) + out: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_json()) assert out["attributes"] == {"a": 1} @@ -464,29 +462,29 @@ def test_arrays_to_tuples(case: Expect[object, object]) -> None: assert arrays_to_tuples(case.input) == case.output -# --- ArrayMetadataModelV3.from_json ---------------------------------------- +# --- ZarrV3ArrayMetadata.from_json ---------------------------------------- def test_v3_from_json_reconstructs_required_fields() -> None: """V3 from_json reconstructs the required fields from a document.""" - doc = ArrayMetadataModelV3.create_default( + doc = ZarrV3ArrayMetadata.create_default( shape=(7,), attributes={"a": 1}, - data_type=NamedConfigModelV3(name="int32", configuration={}), + data_type=ZarrV3NamedConfig(name="int32", configuration={}), ).to_json() - model = ArrayMetadataModelV3.from_json(doc) + model = ZarrV3ArrayMetadata.from_json(doc) assert model.shape == (7,) - assert model.data_type == NamedConfigModelV3(name="int32", configuration={}) + assert model.data_type == ZarrV3NamedConfig(name="int32", configuration={}) assert model.attributes == {"a": 1} def test_v3_from_json_defaults_for_omitted_optionals() -> None: """V3 from_json supplies defaults for omitted optional fields.""" - doc = ArrayMetadataModelV3.create_default( + doc = ZarrV3ArrayMetadata.create_default( attributes={}, storage_transformers=(), dimension_names=UNSET ).to_json() # to_json omits these entirely; from_json must restore defaults - model = ArrayMetadataModelV3.from_json(doc) + model = ZarrV3ArrayMetadata.from_json(doc) assert model.attributes == {} assert model.storage_transformers == () assert model.dimension_names is UNSET @@ -494,36 +492,36 @@ def test_v3_from_json_defaults_for_omitted_optionals() -> None: def test_v3_from_json_routes_unknown_keys_to_extra_fields() -> None: """V3 from_json routes unknown top-level keys into extra_fields.""" - doc = ArrayMetadataModelV3.create_default( + doc = ZarrV3ArrayMetadata.create_default( extra_fields={"my_ext": {"must_understand": False}} ).to_json() - model = ArrayMetadataModelV3.from_json(doc) + model = ZarrV3ArrayMetadata.from_json(doc) assert model.extra_fields == {"my_ext": {"must_understand": False}} def test_v3_from_json_standard_keys_not_in_extra_fields() -> None: """V3 from_json keeps standard keys out of extra_fields.""" - doc = ArrayMetadataModelV3.create_default( + doc = ZarrV3ArrayMetadata.create_default( shape=(10,), attributes={"a": 1}, dimension_names=("x",) ).to_json() - model = ArrayMetadataModelV3.from_json(doc) + model = ZarrV3ArrayMetadata.from_json(doc) assert model.extra_fields == {} def test_v3_from_json_nested_arrays_in_attributes_become_tuples() -> None: """V3 from_json converts nested arrays in attributes into tuples.""" - doc = ArrayMetadataModelV3.create_default(attributes={"scale": [[1, 2], [3, 4]]}).to_json() - model = ArrayMetadataModelV3.from_json(doc) + doc = ZarrV3ArrayMetadata.create_default(attributes={"scale": [[1, 2], [3, 4]]}).to_json() + model = ZarrV3ArrayMetadata.from_json(doc) assert model.attributes == {"scale": ((1, 2), (3, 4))} -# --- ArrayMetadataModelV3.from_key_value ---------------------------------- +# --- ZarrV3ArrayMetadata.from_key_value ---------------------------------- def test_v3_from_key_value_parses_zarr_json() -> None: """V3 from_key_value parses the zarr.json entry into a model.""" - kv = ArrayMetadataModelV3.create_default(shape=(3,)).to_key_value() - model = ArrayMetadataModelV3.from_key_value(kv) + kv = ZarrV3ArrayMetadata.create_default(shape=(3,)).to_key_value() + model = ZarrV3ArrayMetadata.from_key_value(kv) assert model.shape == (3,) @@ -531,12 +529,12 @@ def test_v3_from_key_value_parses_zarr_json() -> None: FROM_KEY_VALUE_MISSING_PARAMS = [ pytest.param( - ArrayMetadataModelV3, + ZarrV3ArrayMetadata, ExpectFail({}, MetadataValidationError, id="v3-missing-zarr-json", msg="missing store key"), id="v3-missing-zarr-json", ), pytest.param( - ArrayMetadataModelV2, + ZarrV2ArrayMetadata, ExpectFail({}, MetadataValidationError, id="v2-missing-zarray", msg="missing store key"), id="v2-missing-zarray", ), @@ -545,7 +543,7 @@ def test_v3_from_key_value_parses_zarr_json() -> None: @pytest.mark.parametrize(("model_cls", "case"), FROM_KEY_VALUE_MISSING_PARAMS) def test_from_key_value_missing_key_raises( - model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], case: ExpectFail[dict[str, bytes]], ) -> None: """from_key_value raises MetadataValidationError when the required store key is absent.""" @@ -557,19 +555,19 @@ def test_from_key_value_missing_key_raises( ROUNDTRIP_MODEL_JSON_PARAMS = [ pytest.param( - ArrayMetadataModelV3, - ArrayMetadataModelV3.create_default( + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadata.create_default( shape=(10,), attributes={"a": 1}, dimension_names=("x",), - storage_transformers=(NamedConfigModelV3(name="t", configuration={}),), + storage_transformers=(ZarrV3NamedConfig(name="t", configuration={}),), extra_fields={"ext": {"must_understand": False}}, ), id="v3-full", ), pytest.param( - ArrayMetadataModelV3, - ArrayMetadataModelV3.create_default( + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadata.create_default( attributes={}, dimension_names=UNSET, storage_transformers=(), @@ -578,8 +576,8 @@ def test_from_key_value_missing_key_raises( id="v3-empty-optionals", ), pytest.param( - ArrayMetadataModelV2, - ArrayMetadataModelV2.create_default(attributes={"a": 1}, filters=None, compressor=None), + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadata.create_default(attributes={"a": 1}, filters=None, compressor=None), id="v2-basic", ), ] @@ -587,8 +585,8 @@ def test_from_key_value_missing_key_raises( @pytest.mark.parametrize(("model_cls", "model"), ROUNDTRIP_MODEL_JSON_PARAMS) def test_roundtrip_model_json_model( - model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], - model: ArrayMetadataModelV3 | ArrayMetadataModelV2, + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], + model: ZarrV3ArrayMetadata | ZarrV2ArrayMetadata, ) -> None: """A model round-trips through to_json/from_json back to an equal model.""" assert model_cls.from_json(model.to_json()) == model @@ -598,13 +596,13 @@ def test_roundtrip_model_json_model( ROUNDTRIP_KEY_VALUE_PARAMS = [ pytest.param( - ArrayMetadataModelV3, - ArrayMetadataModelV3.create_default(attributes={"a": 1}), + ZarrV3ArrayMetadata, + ZarrV3ArrayMetadata.create_default(attributes={"a": 1}), id="v3", ), pytest.param( - ArrayMetadataModelV2, - ArrayMetadataModelV2.create_default(attributes={"a": 1}), + ZarrV2ArrayMetadata, + ZarrV2ArrayMetadata.create_default(attributes={"a": 1}), id="v2", ), ] @@ -612,8 +610,8 @@ def test_roundtrip_model_json_model( @pytest.mark.parametrize(("model_cls", "model"), ROUNDTRIP_KEY_VALUE_PARAMS) def test_roundtrip_via_key_value( - model_cls: type[ArrayMetadataModelV3 | ArrayMetadataModelV2], - model: ArrayMetadataModelV3 | ArrayMetadataModelV2, + model_cls: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata], + model: ZarrV3ArrayMetadata | ZarrV2ArrayMetadata, ) -> None: """A model round-trips through to_key_value/from_key_value back to an equal model.""" assert model_cls.from_key_value(model.to_key_value()) == model @@ -624,48 +622,46 @@ def test_roundtrip_via_key_value( def test_v3_roundtrip_json_model_json() -> None: """A v3 document round-trips through from_json/to_json back to an equal document.""" - doc = ArrayMetadataModelV3.create_default( + doc = ZarrV3ArrayMetadata.create_default( shape=(10,), attributes={"a": 1}, dimension_names=("x",) ).to_json() - assert ArrayMetadataModelV3.from_json(doc).to_json() == doc + assert ZarrV3ArrayMetadata.from_json(doc).to_json() == doc def test_v2_roundtrip_json_model_json() -> None: """A v2 document round-trips through from_json/to_json back to an equal document.""" - doc = ArrayMetadataModelV2.create_default(attributes={"a": 1}).to_json() - assert ArrayMetadataModelV2.from_json(doc).to_json() == doc + doc = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}).to_json() + assert ZarrV2ArrayMetadata.from_json(doc).to_json() == doc def test_v3_parser_accepts_bare_string_data_type() -> None: """V3 from_json accepts a bare-string data_type and re-serializes it canonically.""" - doc = ArrayMetadataModelV3.create_default().to_json() + doc = ZarrV3ArrayMetadata.create_default().to_json() doc["data_type"] = "int32" # bare-string form, not canonical object form - model = ArrayMetadataModelV3.from_json(doc) + model = ZarrV3ArrayMetadata.from_json(doc) # parses correctly, re-serializes to canonical object form - assert model.data_type == NamedConfigModelV3(name="int32", configuration={}) + assert model.data_type == ZarrV3NamedConfig(name="int32", configuration={}) assert model.to_json()["data_type"] == {"name": "int32", "configuration": {}} def test_v2_roundtrip_with_compressor_and_filters() -> None: # Non-None compressor/filters must round-trip; extra assertion on .compressor. """A v2 model with non-None compressor and filters round-trips.""" - compressor: CodecMetadataV2 = {"id": "blosc", "clevel": 5} - filters: tuple[CodecMetadataV2, ...] = ({"id": "delta"},) - m = ArrayMetadataModelV2.create_default(compressor=compressor, filters=filters) - restored = ArrayMetadataModelV2.from_json(m.to_json()) + compressor: ZarrV2CodecMetadata = {"id": "blosc", "clevel": 5} + filters: tuple[ZarrV2CodecMetadata, ...] = ({"id": "delta"},) + m = ZarrV2ArrayMetadata.create_default(compressor=compressor, filters=filters) + restored = ZarrV2ArrayMetadata.from_json(m.to_json()) assert restored == m assert restored.compressor == {"id": "blosc", "clevel": 5} -# --- ArrayMetadataModelV2.from_json ---------------------------------------- +# --- ZarrV2ArrayMetadata.from_json ---------------------------------------- def test_v2_from_json_reconstructs_fields() -> None: """V2 from_json reconstructs the fields from a document.""" - doc = ArrayMetadataModelV2.create_default( - shape=(4,), attributes={"a": 1}, dtype=" None: def test_v2_from_json_attributes_absent_is_unset() -> None: """V2 from_json reads an absent attributes key as UNSET, distinct from an explicit empty mapping.""" - absent = ArrayMetadataModelV2.from_json(ArrayMetadataModelV2.create_default().to_json()) - explicit = ArrayMetadataModelV2.from_json( - ArrayMetadataModelV2.create_default(attributes={}).to_json() + absent = ZarrV2ArrayMetadata.from_json(ZarrV2ArrayMetadata.create_default().to_json()) + explicit = ZarrV2ArrayMetadata.from_json( + ZarrV2ArrayMetadata.create_default(attributes={}).to_json() ) assert absent.attributes is UNSET assert explicit.attributes == {} assert absent != explicit -# --- ArrayMetadataModelV2.from_key_value -------------------------------- +# --- ZarrV2ArrayMetadata.from_key_value -------------------------------- def test_v2_from_key_value_remerges_zattrs() -> None: """V2 from_key_value re-merges .zattrs back into attributes.""" - kv = ArrayMetadataModelV2.create_default(attributes={"a": 1}, shape=(10,)).to_key_value() - model = ArrayMetadataModelV2.from_key_value(kv) + kv = ZarrV2ArrayMetadata.create_default(attributes={"a": 1}, shape=(10,)).to_key_value() + model = ZarrV2ArrayMetadata.from_key_value(kv) assert model.attributes == {"a": 1} assert model.shape == (10,) @@ -698,13 +694,13 @@ def test_v2_zattrs_presence_round_trips() -> None: """The .zattrs file's presence is part of the store: an absent file reads as UNSET and emits no .zattrs; an explicit empty file reads as {} and emits .zattrs — the two stores stay distinct through a round-trip.""" - explicit_kv = dict(ArrayMetadataModelV2.create_default(attributes={}).to_key_value()) + explicit_kv = dict(ZarrV2ArrayMetadata.create_default(attributes={}).to_key_value()) assert ".zattrs" in explicit_kv absent_kv = dict(explicit_kv) del absent_kv[".zattrs"] - absent = ArrayMetadataModelV2.from_key_value(absent_kv) - explicit = ArrayMetadataModelV2.from_key_value(explicit_kv) + absent = ZarrV2ArrayMetadata.from_key_value(absent_kv) + explicit = ZarrV2ArrayMetadata.from_key_value(explicit_kv) assert absent.attributes is UNSET assert explicit.attributes == {} assert ".zattrs" not in absent.to_key_value() @@ -713,8 +709,8 @@ def test_v2_zattrs_presence_round_trips() -> None: def test_v2_from_json_nested_arrays_in_attributes_become_tuples() -> None: """V2 from_json converts nested arrays in attributes into tuples.""" - doc = ArrayMetadataModelV2.create_default(attributes={"axes": [[0, 1], [2, 3]]}).to_json() - model = ArrayMetadataModelV2.from_json(doc) + doc = ZarrV2ArrayMetadata.create_default(attributes={"axes": [[0, 1], [2, 3]]}).to_json() + model = ZarrV2ArrayMetadata.from_json(doc) assert model.attributes == {"axes": ((0, 1), (2, 3))} @@ -829,12 +825,12 @@ def test_parse_metadata_field_v3( # invalid cases (a subset check, so accumulation of OTHER problems is allowed). -def _build_v3(**overrides: Unpack[ArrayMetadataModelV3Partial]) -> dict[str, object]: - return dict(ArrayMetadataModelV3.create_default(**overrides).to_json()) +def _build_v3(**overrides: Unpack[ZarrV3ArrayMetadataPartial]) -> dict[str, object]: + return dict(ZarrV3ArrayMetadata.create_default(**overrides).to_json()) -def _build_v2(**overrides: Unpack[ArrayMetadataModelV2Partial]) -> dict[str, object]: - return dict(ArrayMetadataModelV2.create_default(**overrides).to_json()) +def _build_v2(**overrides: Unpack[ZarrV2ArrayMetadataPartial]) -> dict[str, object]: + return dict(ZarrV2ArrayMetadata.create_default(**overrides).to_json()) def _mutate(build: Callable[[], dict], mutate: Callable[[dict], object]) -> Callable[[], dict]: @@ -971,22 +967,22 @@ def test_array_metadata_guards( FROM_JSON_REJECT_PARAMS = [ pytest.param( - ArrayMetadataModelV3, + ZarrV3ArrayMetadata, ExpectFail(lambda: {"zarr_format": 3}, MetadataValidationError, id="x"), id="v3-missing-required", ), pytest.param( - ArrayMetadataModelV3, + ZarrV3ArrayMetadata, ExpectFail(_mutate(_build_v3, _set("data_type", 5)), MetadataValidationError, id="x"), id="v3-bad-field-type", ), pytest.param( - ArrayMetadataModelV2, + ZarrV2ArrayMetadata, ExpectFail(lambda: {"zarr_format": 2}, MetadataValidationError, id="x"), id="v2-missing-required", ), pytest.param( - NamedConfigModelV3, + ZarrV3NamedConfig, ExpectFail(lambda: 5, MetadataValidationError, id="x"), id="zarr-metadata-bad-input", ), @@ -995,7 +991,7 @@ def test_array_metadata_guards( @pytest.mark.parametrize(("model", "case"), FROM_JSON_REJECT_PARAMS) def test_from_json_rejects_malformed( - model: type[ArrayMetadataModelV3 | ArrayMetadataModelV2 | NamedConfigModelV3], + model: type[ZarrV3ArrayMetadata | ZarrV2ArrayMetadata | ZarrV3NamedConfig], case: ExpectFail[Callable[[], object]], ) -> None: """from_json raises MetadataValidationError on a malformed document.""" @@ -1056,7 +1052,7 @@ def test_prefix_prepends_loc_head() -> None: def test_v2_dtype_must_be_string_or_records() -> None: """A non-string, non-records v2 dtype is rejected with an invalid_type problem.""" - doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dtype": 42} + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dtype": 42} problems = validate_array_metadata_v2(doc) assert [(p.loc, p.kind) for p in problems] == [(("dtype",), "invalid_type")] @@ -1064,34 +1060,34 @@ def test_v2_dtype_must_be_string_or_records() -> None: def test_v2_structured_dtype_records_accepted() -> None: """A structured v2 dtype (field records, optionally nested/shaped) validates.""" dtype = (("a", " None: """A field record with the wrong arity is rejected.""" - doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dtype": (("a",),)} + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dtype": (("a",),)} problems = validate_array_metadata_v2(doc) assert [p.loc for p in problems] == [("dtype",)] def test_v2_order_literal_enforced() -> None: """An order other than 'C' or 'F' is rejected with an invalid_value problem.""" - doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"order": "Q"} + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"order": "Q"} problems = validate_array_metadata_v2(doc) assert [(p.loc, p.kind) for p in problems] == [(("order",), "invalid_value")] def test_v2_compressor_must_be_codec_or_none() -> None: """A compressor that is not null or a codec config mapping is rejected.""" - doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"compressor": "zlib"} + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"compressor": "zlib"} problems = validate_array_metadata_v2(doc) assert [(p.loc, p.kind) for p in problems] == [(("compressor",), "invalid_type")] def test_v2_compressor_requires_string_id() -> None: """A compressor mapping without a string id is rejected.""" - doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"compressor": {"level": 3}} + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"compressor": {"level": 3}} problems = validate_array_metadata_v2(doc) assert [p.loc for p in problems] == [("compressor",)] @@ -1099,42 +1095,42 @@ def test_v2_compressor_requires_string_id() -> None: def test_v2_filters_must_be_codec_sequence_or_none() -> None: """Filters that are not null or a sequence of codec configs are rejected.""" for bad in (7, (5,), "gzip"): - doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"filters": bad} + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"filters": bad} problems = validate_array_metadata_v2(doc) assert [(p.loc, p.kind) for p in problems] == [(("filters",), "invalid_type")], bad def test_v2_dimension_separator_literal_enforced() -> None: """A dimension_separator other than '.' or '/' is rejected.""" - doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dimension_separator": "-"} + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dimension_separator": "-"} problems = validate_array_metadata_v2(doc) assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] def test_v2_zarr_format_literal_enforced() -> None: """A v2 document claiming zarr_format 3 is rejected with an invalid_value problem.""" - doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"zarr_format": 3} + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"zarr_format": 3} problems = validate_array_metadata_v2(doc) assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] def test_v3_zarr_format_literal_enforced() -> None: """A v3 document claiming zarr_format 2 is rejected with an invalid_value problem.""" - doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"zarr_format": 2} + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"zarr_format": 2} problems = validate_array_metadata_v3(doc) assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] def test_v3_node_type_literal_enforced() -> None: """A v3 array document claiming node_type 'group' is rejected.""" - doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"node_type": "group"} + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"node_type": "group"} problems = validate_array_metadata_v3(doc) assert [(p.loc, p.kind) for p in problems] == [(("node_type",), "invalid_value")] def test_missing_key_kind_is_machine_readable() -> None: """A missing required key is distinguishable by kind, without message matching.""" - doc = dict(ArrayMetadataModelV3.create_default().to_json()) + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) del doc["chunk_key_encoding"] problems = validate_array_metadata_v3(doc) assert problems == [ @@ -1148,14 +1144,14 @@ def test_missing_key_kind_is_machine_readable() -> None: def test_from_key_value_invalid_json_raises_metadata_error() -> None: """Undecodable store bytes raise MetadataValidationError (kind invalid_json), not JSONDecodeError.""" with pytest.raises(MetadataValidationError) as exc_info: - ArrayMetadataModelV3.from_key_value({"zarr.json": b"{not json"}) + ZarrV3ArrayMetadata.from_key_value({"zarr.json": b"{not json"}) assert [p.kind for p in exc_info.value.problems] == ["invalid_json"] def test_from_key_value_missing_key_kind() -> None: """A missing store key surfaces as a missing_key problem at the store-key loc.""" with pytest.raises(MetadataValidationError) as exc_info: - ArrayMetadataModelV2.from_key_value({}) + ZarrV2ArrayMetadata.from_key_value({}) assert exc_info.value.problems == [ ValidationProblem((".zarray",), "missing store key", "missing_key") ] @@ -1164,20 +1160,20 @@ def test_from_key_value_missing_key_kind() -> None: def test_extra_fields_overlap_raises_metadata_error() -> None: """The extra-fields overlap invariant raises MetadataValidationError (a ValueError).""" with pytest.raises(MetadataValidationError, match="Extra fields") as exc_info: - ArrayMetadataModelV3.create_default(extra_fields={"shape": {"must_understand": False}}) + ZarrV3ArrayMetadata.create_default(extra_fields={"shape": {"must_understand": False}}) assert [p.kind for p in exc_info.value.problems] == ["invalid_value"] def test_extension_point_fields_annotated_with_role_alias() -> None: - """Extension-point fields are annotated with MetadataFieldModelV3 (the - logical role), not NamedConfigModelV3 (the current serialized form), so a + """Extension-point fields are annotated with ZarrV3MetadataField (the + logical role), not ZarrV3NamedConfig (the current serialized form), so a future widening of the field union does not move annotation sites.""" - assert MetadataFieldModelV3 is NamedConfigModelV3 - annotations = ArrayMetadataModelV3.__annotations__ + assert ZarrV3MetadataField is ZarrV3NamedConfig + annotations = ZarrV3ArrayMetadata.__annotations__ for field_name in ("data_type", "chunk_grid", "chunk_key_encoding"): - assert annotations[field_name] == "MetadataFieldModelV3" + assert annotations[field_name] == "ZarrV3MetadataField" for field_name in ("codecs", "storage_transformers"): - assert annotations[field_name] == "tuple[MetadataFieldModelV3, ...]" + assert annotations[field_name] == "tuple[ZarrV3MetadataField, ...]" # --- Adversarial-probe fixes: documents that used to pass validation --------- @@ -1186,19 +1182,19 @@ def test_extension_point_fields_annotated_with_role_alias() -> None: def test_shape_rejects_json_booleans() -> None: """JSON booleans are not integers: shape/chunks containing true/false are rejected (bool is an int subclass in Python, so isinstance alone passes).""" - v3 = dict(ArrayMetadataModelV3.create_default().to_json()) | {"shape": (True, True)} + v3 = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"shape": (True, True)} assert [p.loc for p in validate_array_metadata_v3(v3)] == [("shape",)] - v2 = dict(ArrayMetadataModelV2.create_default().to_json()) | {"chunks": (True,)} + v2 = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"chunks": (True,)} assert [p.loc for p in validate_array_metadata_v2(v2)] == [("chunks",)] def test_shape_rejects_negative_dimensions() -> None: """Dimension lengths must be non-negative; a negative entry is invalid_value.""" - v3 = dict(ArrayMetadataModelV3.create_default().to_json()) | {"shape": (-1,)} + v3 = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"shape": (-1,)} assert [(p.loc, p.kind) for p in validate_array_metadata_v3(v3)] == [ (("shape",), "invalid_value") ] - v2 = dict(ArrayMetadataModelV2.create_default().to_json()) | {"chunks": (-5,)} + v2 = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"chunks": (-5,)} assert [(p.loc, p.kind) for p in validate_array_metadata_v2(v2)] == [ (("chunks",), "invalid_value") ] @@ -1206,7 +1202,7 @@ def test_shape_rejects_negative_dimensions() -> None: def test_dimension_names_length_must_match_shape() -> None: """dimension_names must have one entry per dimension of shape.""" - doc = dict(ArrayMetadataModelV3.create_default(shape=(10,)).to_json()) | { + doc = dict(ZarrV3ArrayMetadata.create_default(shape=(10,)).to_json()) | { "dimension_names": ("x", "y", "z") } assert [(p.loc, p.kind) for p in validate_array_metadata_v3(doc)] == [ @@ -1217,7 +1213,7 @@ def test_dimension_names_length_must_match_shape() -> None: def test_attributes_values_must_be_json() -> None: """Attribute values are JSON-checked recursively (like fill_value), so a non-serializable value is a validation problem, not a later TypeError.""" - doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"attributes": {"a": {1, 2}}} + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"attributes": {"a": {1, 2}}} problems = validate_array_metadata_v3(doc) assert [(p.loc, p.kind) for p in problems] == [(("attributes", "a"), "invalid_type")] @@ -1225,7 +1221,7 @@ def test_attributes_values_must_be_json() -> None: def test_configuration_values_must_be_json() -> None: """Configuration values are JSON-checked recursively, so an int-keyed dict cannot pass validation and be silently rewritten by json.dumps.""" - doc = dict(ArrayMetadataModelV3.create_default().to_json()) | { + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | { "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": {1: 2}}} } problems = validate_array_metadata_v3(doc) @@ -1242,7 +1238,7 @@ def test_must_understand_fields_partition() -> None: with must_understand: false, including implicitly-true and non-mapping fields, so a reader can discharge the spec's fail-to-open duty by subtracting the extensions it recognizes.""" - model = ArrayMetadataModelV3.create_default( + model = ZarrV3ArrayMetadata.create_default( extra_fields={ "ext_a": {"name": "a", "must_understand": False}, "ext_b": {"name": "b"}, @@ -1257,7 +1253,7 @@ def test_must_understand_fields_partition() -> None: def test_must_understand_fields_empty_when_all_waived() -> None: """must_understand_fields is empty when every extra field is explicitly waived.""" - model = ArrayMetadataModelV3.create_default( + model = ZarrV3ArrayMetadata.create_default( extra_fields={"ext_a": {"name": "a", "must_understand": False}} ) assert model.must_understand_fields == {} @@ -1268,13 +1264,12 @@ def test_dimension_names_null_field_rejected() -> None: null as an element (an unnamed dimension), never as the field value — "not specified" is spelled by omitting the key. Consumers bridging from an in-memory None sentinel must drop the key, not write null.""" - doc = dict(ArrayMetadataModelV3.create_default().to_json()) | {"dimension_names": None} + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"dimension_names": None} problems = validate_array_metadata_v3(doc) assert [(p.loc, p.kind) for p in problems] == [(("dimension_names",), "invalid_type")] # and the model's own None spelling correctly maps to key absence assert ( - "dimension_names" - not in ArrayMetadataModelV3.create_default(dimension_names=UNSET).to_json() + "dimension_names" not in ZarrV3ArrayMetadata.create_default(dimension_names=UNSET).to_json() ) @@ -1285,28 +1280,28 @@ def test_v3_create_default_chunk_grid_follows_shape() -> None: """Overriding shape without chunk_grid derives a consistent default grid: one chunk covering the array (chunk_shape == shape), instead of silently keeping the scalar default's 0-d grid.""" - model = ArrayMetadataModelV3.create_default(shape=(100, 100)) - assert model.chunk_grid == NamedConfigModelV3( + model = ZarrV3ArrayMetadata.create_default(shape=(100, 100)) + assert model.chunk_grid == ZarrV3NamedConfig( name="regular", configuration={"chunk_shape": (100, 100)} ) def test_v3_create_default_explicit_chunk_grid_respected() -> None: """An explicit chunk_grid override wins over the shape-derived default.""" - grid = NamedConfigModelV3(name="regular", configuration={"chunk_shape": (10, 10)}) - model = ArrayMetadataModelV3.create_default(shape=(100, 100), chunk_grid=grid) + grid = ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": (10, 10)}) + model = ZarrV3ArrayMetadata.create_default(shape=(100, 100), chunk_grid=grid) assert model.chunk_grid == grid def test_v2_create_default_chunks_follow_shape() -> None: """Overriding shape without chunks derives chunks == shape.""" - model = ArrayMetadataModelV2.create_default(shape=(100, 100)) + model = ZarrV2ArrayMetadata.create_default(shape=(100, 100)) assert model.chunks == (100, 100) def test_v2_create_default_explicit_chunks_respected() -> None: """An explicit chunks override wins over the shape-derived default.""" - model = ArrayMetadataModelV2.create_default(shape=(100, 100), chunks=(10, 10)) + model = ZarrV2ArrayMetadata.create_default(shape=(100, 100), chunks=(10, 10)) assert model.chunks == (10, 10) @@ -1315,7 +1310,7 @@ def test_v3_create_default_zero_length_dimensions() -> None: 'The chunk shape elements are non-zero when the corresponding dimensions of the arrays have non-zero length' — the constraint is conditional, so a zero chunk length is permitted exactly where the dimension is empty.""" - model = ArrayMetadataModelV3.create_default(shape=(0, 3)) + model = ZarrV3ArrayMetadata.create_default(shape=(0, 3)) assert model.chunk_grid.configuration["chunk_shape"] == (0, 3) @@ -1324,11 +1319,11 @@ def test_create_default_derivation_is_one_way() -> None: scalar default shape=() untouched: a user-supplied chunk_grid is an extension point taken verbatim, and deriving shape from it would require interpreting grid configurations, which the model layer never does.""" - grid = NamedConfigModelV3(name="regular", configuration={"chunk_shape": (10, 10)}) - v3 = ArrayMetadataModelV3.create_default(chunk_grid=grid) + grid = ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": (10, 10)}) + v3 = ZarrV3ArrayMetadata.create_default(chunk_grid=grid) assert v3.shape == () assert v3.chunk_grid == grid - v2 = ArrayMetadataModelV2.create_default(chunks=(10, 10)) + v2 = ZarrV2ArrayMetadata.create_default(chunks=(10, 10)) assert v2.shape == () assert v2.chunks == (10, 10) @@ -1341,9 +1336,9 @@ def test_v2_absent_dimension_separator_means_dot() -> None: '.', not '/': chunk keys of real-world default-separator v2 arrays look like '0.0'. The model normalizes the absent key to an explicit '.' — a semantics-preserving spelling normalization.""" - doc = dict(ArrayMetadataModelV2.create_default().to_json()) + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) del doc["dimension_separator"] - model = ArrayMetadataModelV2.from_json(doc) + model = ZarrV2ArrayMetadata.from_json(doc) assert model.dimension_separator == "." assert model.to_json()["dimension_separator"] == "." @@ -1353,19 +1348,19 @@ def test_v2_from_key_value_without_separator_means_dot() -> None: dimension_separator key.""" doc = { k: v - for k, v in ArrayMetadataModelV2.create_default().to_json().items() + for k, v in ZarrV2ArrayMetadata.create_default().to_json().items() if k not in ("dimension_separator", "attributes") } import json as _json - model = ArrayMetadataModelV2.from_key_value({".zarray": _json.dumps(doc).encode()}) + model = ZarrV2ArrayMetadata.from_key_value({".zarray": _json.dumps(doc).encode()}) assert model.dimension_separator == "." def test_v2_null_dimension_separator_rejected() -> None: """dimension_separator may be absent, '.', or '/' — never null: the document grammar has no null spelling for this field.""" - doc = dict(ArrayMetadataModelV2.create_default().to_json()) | {"dimension_separator": None} + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dimension_separator": None} problems = validate_array_metadata_v2(doc) assert [(p.loc, p.kind) for p in problems] == [(("dimension_separator",), "invalid_value")] @@ -1376,11 +1371,11 @@ def test_dimension_names_absent_and_all_null_are_distinct() -> None: has a name, which is null; absence says there are no dimension names. The model preserves the distinction (UNSET vs a tuple of Nones), and both spellings round-trip faithfully.""" - absent_doc = dict(ArrayMetadataModelV3.create_default(shape=(2, 3)).to_json()) + absent_doc = dict(ZarrV3ArrayMetadata.create_default(shape=(2, 3)).to_json()) explicit_doc = absent_doc | {"dimension_names": (None, None)} - absent = ArrayMetadataModelV3.from_json(absent_doc) - explicit = ArrayMetadataModelV3.from_json(explicit_doc) + absent = ZarrV3ArrayMetadata.from_json(absent_doc) + explicit = ZarrV3ArrayMetadata.from_json(explicit_doc) assert absent.dimension_names is UNSET assert explicit.dimension_names == (None, None) diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 58b92f5d71..2259a3d8ca 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -6,14 +6,14 @@ import pytest from zarr_metadata.model import UNSET -from zarr_metadata.model._array import ArrayMetadataModelV3 +from zarr_metadata.model._array import ZarrV3ArrayMetadata from zarr_metadata.model._group import ( - ConsolidatedMetadataModelV2, - ConsolidatedMetadataModelV3, - GroupMetadataModelV2, - GroupMetadataModelV2Partial, - GroupMetadataModelV3, - GroupMetadataModelV3Partial, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV2GroupMetadataPartial, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, + ZarrV3GroupMetadataPartial, ) from zarr_metadata.model._validation import ( MetadataValidationError, @@ -23,26 +23,26 @@ validate_group_metadata_v3, ) -# --- GroupMetadataModelV3 --------------------------------------------------- +# --- ZarrV3GroupMetadata --------------------------------------------------- def test_group_v3_roundtrip() -> None: """A v3 group document round-trips through the model unchanged.""" doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": (1, 2)}} - model = GroupMetadataModelV3.from_json(doc) + model = ZarrV3GroupMetadata.from_json(doc) assert model.to_json() == doc def test_group_v3_omits_empty_attributes() -> None: """to_json omits the attributes key when attributes is empty.""" - model = GroupMetadataModelV3.create_default() + model = ZarrV3GroupMetadata.create_default() assert "attributes" not in model.to_json() def test_group_v3_lists_become_tuples() -> None: """from_json converts JSON arrays in attributes to tuples.""" doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": [1, 2]}} - model = GroupMetadataModelV3.from_json(doc) + model = ZarrV3GroupMetadata.from_json(doc) assert model.attributes == {"a": (1, 2)} @@ -53,7 +53,7 @@ def test_group_v3_extra_fields_roundtrip() -> None: "node_type": "group", "my_extension": {"name": "thing", "must_understand": False}, } - model = GroupMetadataModelV3.from_json(doc) + model = ZarrV3GroupMetadata.from_json(doc) assert model.extra_fields == {"my_extension": {"name": "thing", "must_understand": False}} assert model.to_json() == doc @@ -61,7 +61,7 @@ def test_group_v3_extra_fields_roundtrip() -> None: def test_group_v3_extra_fields_overlap_rejected() -> None: """Constructing a v3 group model with extra_fields shadowing a standard key raises.""" with pytest.raises(ValueError, match="Extra fields"): - GroupMetadataModelV3( + ZarrV3GroupMetadata( attributes={}, consolidated_metadata=UNSET, extra_fields={"node_type": {"name": "x", "must_understand": False}}, @@ -71,7 +71,7 @@ def test_group_v3_extra_fields_overlap_rejected() -> None: def test_group_v3_consolidated_extra_field_rejected() -> None: """extra_fields may not shadow the consolidated_metadata convention key.""" with pytest.raises(ValueError, match="Extra fields"): - GroupMetadataModelV3( + ZarrV3GroupMetadata( attributes={}, consolidated_metadata=UNSET, extra_fields={"consolidated_metadata": {"name": "x", "must_understand": False}}, @@ -92,38 +92,38 @@ def test_group_v3_bad_attributes() -> None: def test_group_v3_key_value_roundtrip() -> None: """from_key_value(to_key_value()) is the identity for v3 groups.""" - model = GroupMetadataModelV3.create_default(attributes={"a": 1}) - assert GroupMetadataModelV3.from_key_value(model.to_key_value()) == model + model = ZarrV3GroupMetadata.create_default(attributes={"a": 1}) + assert ZarrV3GroupMetadata.from_key_value(model.to_key_value()) == model def test_group_v3_update() -> None: """update replaces the given fields and returns a new instance.""" - base = GroupMetadataModelV3.create_default() + base = ZarrV3GroupMetadata.create_default() updated = base.update(attributes={"a": 1}) assert updated.attributes == {"a": 1} assert base.attributes == {} -# --- GroupMetadataModelV2 --------------------------------------------------- +# --- ZarrV2GroupMetadata --------------------------------------------------- def test_group_v2_key_value_split() -> None: """v2 to_key_value writes .zgroup and .zattrs; from_key_value merges them.""" - model = GroupMetadataModelV2.create_default(attributes={"a": 1}) + model = ZarrV2GroupMetadata.create_default(attributes={"a": 1}) kv = model.to_key_value() assert set(kv) == {".zgroup", ".zattrs"} assert json.loads(kv[".zgroup"]) == {"zarr_format": 2} - assert GroupMetadataModelV2.from_key_value(kv) == model + assert ZarrV2GroupMetadata.from_key_value(kv) == model def test_group_v2_zattrs_presence_round_trips() -> None: """A v2 group with no .zattrs file parses with UNSET attributes and emits no .zattrs; an explicit empty .zattrs stays a file — the stores remain distinct through a round-trip.""" - absent = GroupMetadataModelV2.from_key_value({".zgroup": b'{"zarr_format": 2}'}) + absent = ZarrV2GroupMetadata.from_key_value({".zgroup": b'{"zarr_format": 2}'}) assert absent.attributes is UNSET assert ".zattrs" not in absent.to_key_value() - explicit = GroupMetadataModelV2.from_key_value( + explicit = ZarrV2GroupMetadata.from_key_value( {".zgroup": b'{"zarr_format": 2}', ".zattrs": b"{}"} ) assert explicit.attributes == {} @@ -134,13 +134,13 @@ def test_group_v2_zattrs_presence_round_trips() -> None: def test_group_v2_json_roundtrip() -> None: """A merged-form v2 group document round-trips through the model unchanged.""" doc = {"zarr_format": 2, "attributes": {"a": 1}} - model = GroupMetadataModelV2.from_json(doc) + model = ZarrV2GroupMetadata.from_json(doc) assert model.to_json() == doc def test_group_v2_omits_empty_attributes() -> None: """to_json omits the attributes key when attributes is empty.""" - assert "attributes" not in GroupMetadataModelV2.create_default().to_json() + assert "attributes" not in ZarrV2GroupMetadata.create_default().to_json() def test_group_v2_not_a_mapping() -> None: @@ -165,20 +165,20 @@ def test_group_partial_keys_match_settable_model_fields() -> None: without updating its `*Partial` TypedDict fails here. """ for model_cls, partial_cls in ( - (GroupMetadataModelV3, GroupMetadataModelV3Partial), - (GroupMetadataModelV2, GroupMetadataModelV2Partial), + (ZarrV3GroupMetadata, ZarrV3GroupMetadataPartial), + (ZarrV2GroupMetadata, ZarrV2GroupMetadataPartial), ): settable = {f.name for f in dataclasses.fields(model_cls) if f.init} assert set(partial_cls.__annotations__) == settable -# --- ConsolidatedMetadataModelV3 -------------------------------------------- +# --- ZarrV3ConsolidatedMetadata -------------------------------------------- def test_consolidated_v3_roundtrip() -> None: """A v3 group with inline consolidated metadata round-trips, with child entries parsed into array/group models.""" - child = ArrayMetadataModelV3.create_default(shape=(2,)).to_json() + child = ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json() doc = { "zarr_format": 3, "node_type": "group", @@ -188,23 +188,23 @@ def test_consolidated_v3_roundtrip() -> None: "metadata": {"a": child, "g": {"zarr_format": 3, "node_type": "group"}}, }, } - model = GroupMetadataModelV3.from_json(doc) - assert isinstance(model.consolidated_metadata, ConsolidatedMetadataModelV3) - assert isinstance(model.consolidated_metadata.metadata["a"], ArrayMetadataModelV3) - assert isinstance(model.consolidated_metadata.metadata["g"], GroupMetadataModelV3) + model = ZarrV3GroupMetadata.from_json(doc) + assert isinstance(model.consolidated_metadata, ZarrV3ConsolidatedMetadata) + assert isinstance(model.consolidated_metadata.metadata["a"], ZarrV3ArrayMetadata) + assert isinstance(model.consolidated_metadata.metadata["g"], ZarrV3GroupMetadata) assert model.to_json() == doc def test_consolidated_v3_must_understand_true_rejected() -> None: - """ConsolidatedMetadataModelV3 enforces must_understand=False at runtime.""" + """ZarrV3ConsolidatedMetadata enforces must_understand=False at runtime.""" with pytest.raises(ValueError, match="must_understand"): - ConsolidatedMetadataModelV3(must_understand=True, metadata={}) + ZarrV3ConsolidatedMetadata(must_understand=True, metadata={}) def test_consolidated_v3_from_json_must_understand_true_rejected() -> None: """from_json rejects a consolidated document carrying must_understand=true.""" with pytest.raises(MetadataValidationError, match="must_understand"): - ConsolidatedMetadataModelV3.from_json( + ZarrV3ConsolidatedMetadata.from_json( {"kind": "inline", "must_understand": True, "metadata": {}} ) @@ -212,7 +212,7 @@ def test_consolidated_v3_from_json_must_understand_true_rejected() -> None: def test_consolidated_v3_entry_without_node_type_rejected() -> None: """from_json rejects a consolidated entry lacking a recognizable node_type.""" with pytest.raises(MetadataValidationError, match="node_type"): - ConsolidatedMetadataModelV3.from_json( + ZarrV3ConsolidatedMetadata.from_json( {"kind": "inline", "must_understand": False, "metadata": {"a": {"zarr_format": 3}}} ) @@ -220,10 +220,10 @@ def test_consolidated_v3_entry_without_node_type_rejected() -> None: def test_consolidated_v3_not_a_mapping() -> None: """from_json rejects a non-mapping consolidated document.""" with pytest.raises(MetadataValidationError, match="expected a mapping"): - ConsolidatedMetadataModelV3.from_json(5) + ZarrV3ConsolidatedMetadata.from_json(5) -# --- ConsolidatedMetadataModelV2 -------------------------------------------- +# --- ZarrV2ConsolidatedMetadata -------------------------------------------- def test_consolidated_v2_verbatim_roundtrip() -> None: @@ -245,16 +245,16 @@ def test_consolidated_v2_verbatim_roundtrip() -> None: }, }, } - model = ConsolidatedMetadataModelV2.from_json(doc) + model = ZarrV2ConsolidatedMetadata.from_json(doc) assert model.to_json() == doc def test_consolidated_v2_key_value_roundtrip() -> None: """from_key_value(to_key_value()) is the identity for .zmetadata documents.""" - model = ConsolidatedMetadataModelV2.from_json( + model = ZarrV2ConsolidatedMetadata.from_json( {"zarr_consolidated_format": 1, "metadata": {".zgroup": {"zarr_format": 2}}} ) - assert ConsolidatedMetadataModelV2.from_key_value(model.to_key_value()) == model + assert ZarrV2ConsolidatedMetadata.from_key_value(model.to_key_value()) == model def test_consolidated_v2_lists_become_tuples() -> None: @@ -263,20 +263,20 @@ def test_consolidated_v2_lists_become_tuples() -> None: "zarr_consolidated_format": 1, "metadata": {"a/.zarray": {"shape": [2, 3]}}, } - model = ConsolidatedMetadataModelV2.from_json(doc) + model = ZarrV2ConsolidatedMetadata.from_json(doc) assert model.metadata == {"a/.zarray": {"shape": (2, 3)}} def test_consolidated_v2_envelope_validation() -> None: """from_json rejects a .zmetadata document missing the metadata key.""" with pytest.raises(MetadataValidationError, match="metadata"): - ConsolidatedMetadataModelV2.from_json({"zarr_consolidated_format": 1}) + ZarrV2ConsolidatedMetadata.from_json({"zarr_consolidated_format": 1}) def test_consolidated_v2_not_a_mapping() -> None: """from_json rejects a non-mapping .zmetadata document.""" with pytest.raises(MetadataValidationError, match="expected a mapping"): - ConsolidatedMetadataModelV2.from_json([1]) + ZarrV2ConsolidatedMetadata.from_json([1]) # --- Literal-value enforcement ----------------------------------------------- @@ -284,7 +284,7 @@ def test_consolidated_v2_not_a_mapping() -> None: def test_group_v3_literals_enforced() -> None: """A v3 group doc with wrong zarr_format or node_type is rejected with invalid_value.""" - base = GroupMetadataModelV3.create_default().to_json() + base = ZarrV3GroupMetadata.create_default().to_json() for key, bad in (("zarr_format", 2), ("node_type", "array")): problems = validate_group_metadata_v3(dict(base) | {key: bad}) assert [(p.loc, p.kind) for p in problems] == [((key,), "invalid_value")], key @@ -329,12 +329,12 @@ def test_group_v3_validator_agrees_with_from_json_on_consolidated() -> None: for doc in bad_docs: assert validate_group_metadata_v3(doc) != [], doc with pytest.raises(MetadataValidationError): - GroupMetadataModelV3.from_json(doc) + ZarrV3GroupMetadata.from_json(doc) def test_group_v3_valid_consolidated_passes_validator() -> None: """A well-formed consolidated group validates cleanly (control case).""" - child = ArrayMetadataModelV3.create_default(shape=(2,)).to_json() + child = ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json() doc = { "zarr_format": 3, "node_type": "group", @@ -353,7 +353,7 @@ def test_group_v3_valid_consolidated_passes_validator() -> None: def test_group_must_understand_fields_partition() -> None: """The group model partitions extra fields by the spec's implicit-true rule, like the array model.""" - model = GroupMetadataModelV3.create_default( + model = ZarrV3GroupMetadata.create_default( extra_fields={ "waived": {"name": "w", "must_understand": False}, "implicit": {"name": "i"}, @@ -369,7 +369,7 @@ def test_group_v3_null_consolidated_metadata_repaired_to_absence() -> None: deliberately repairs the document rather than preserving the bug.""" null_doc = {"zarr_format": 3, "node_type": "group", "consolidated_metadata": None} assert validate_group_metadata_v3(null_doc) == [] - model = GroupMetadataModelV3.from_json(null_doc) + model = ZarrV3GroupMetadata.from_json(null_doc) assert model.consolidated_metadata is UNSET assert "consolidated_metadata" not in model.to_json() - assert model == GroupMetadataModelV3.from_json({"zarr_format": 3, "node_type": "group"}) + assert model == ZarrV3GroupMetadata.from_json({"zarr_format": 3, "node_type": "group"}) diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index 7e80d52ee8..d3e6bc6358 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -3,7 +3,7 @@ Pydantic's native dataclass introspection CAN be made to work (see `test_native_dataclass_introspection_is_possible_but_diverges`): the models keep their annotation-only imports behind `TYPE_CHECKING`, so a bare -`TypeAdapter(ArrayMetadataModelV3)` raises `class-not-fully-defined`, but +`TypeAdapter(ZarrV3ArrayMetadata)` raises `class-not-fully-defined`, but `rebuild(_types_namespace=...)` with the names supplied resolves the schema. It is still the wrong tool: it validates the MODEL SHAPE, not the DOCUMENT — no `from_json` normalization (a bare-string `data_type` is rejected), and @@ -42,24 +42,24 @@ ) from zarr_metadata import JSONValue -from zarr_metadata.model import ArrayMetadataModelV3, NamedConfigModelV3 +from zarr_metadata.model import ZarrV3ArrayMetadata, ZarrV3NamedConfig # --- the integration (this is the example) ----------------------------------- -def _as_array_metadata_v3(value: object) -> ArrayMetadataModelV3: +def _as_array_metadata_v3(value: object) -> ZarrV3ArrayMetadata: """Accept an existing model instance or a raw metadata document.""" - if isinstance(value, ArrayMetadataModelV3): + if isinstance(value, ZarrV3ArrayMetadata): return value - return ArrayMetadataModelV3.from_json(value) + return ZarrV3ArrayMetadata.from_json(value) -# return_type is explicit because to_json's own annotation (`ArrayMetadataV3`) +# return_type is explicit because to_json's own annotation (`ZarrV3ArrayMetadataJSON`) # is a TYPE_CHECKING-only name pydantic cannot resolve at runtime. ArrayMetadataV3Field = Annotated[ - InstanceOf[ArrayMetadataModelV3], + InstanceOf[ZarrV3ArrayMetadata], BeforeValidator(_as_array_metadata_v3), - PlainSerializer(ArrayMetadataModelV3.to_json, return_type=dict), + PlainSerializer(ZarrV3ArrayMetadata.to_json, return_type=dict), ] """A pydantic-ready field type for v3 array metadata. @@ -93,14 +93,14 @@ def test_raw_document_is_validated_into_a_model() -> None: """A raw metadata document on a pydantic field is parsed by from_json, with the library's normalization applied (tuples, canonical field form).""" manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) - assert isinstance(manifest.metadata, ArrayMetadataModelV3) + assert isinstance(manifest.metadata, ZarrV3ArrayMetadata) assert manifest.metadata.shape == (10,) assert manifest.metadata.data_type.name == "uint8" def test_model_instance_passes_through() -> None: """An already-constructed model instance is accepted unchanged.""" - model = ArrayMetadataModelV3.from_json(VALID_DOC) + model = ZarrV3ArrayMetadata.from_json(VALID_DOC) manifest = ArrayManifest(path="a/b", metadata=model) assert manifest.metadata is model @@ -137,7 +137,7 @@ def test_type_adapter_standalone() -> None: """The annotated alias also works without a BaseModel, via TypeAdapter.""" adapter = TypeAdapter(ArrayMetadataV3Field) model = adapter.validate_python(VALID_DOC) - assert isinstance(model, ArrayMetadataModelV3) + assert isinstance(model, ZarrV3ArrayMetadata) assert adapter.dump_python(model) == model.to_json() @@ -157,20 +157,20 @@ def test_native_dataclass_introspection_is_not_supported() -> None: path needs its divergences documented again.""" from zarr_metadata._common import JSONValue from zarr_metadata.model import UNSET - from zarr_metadata.v3._common import MetadataV3 - from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3 + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON, ZarrV3ExtensionField def build_and_use() -> None: - adapter = TypeAdapter(ArrayMetadataModelV3) + adapter = TypeAdapter(ZarrV3ArrayMetadata) adapter.rebuild( force=True, _types_namespace={ "JSONValue": JSONValue, - "ExtensionFieldV3": ExtensionFieldV3, - "MetadataV3": MetadataV3, - "ArrayMetadataV3": ArrayMetadataV3, - "NamedConfigModelV3": NamedConfigModelV3, - "MetadataFieldModelV3": NamedConfigModelV3, + "ZarrV3ExtensionField": ZarrV3ExtensionField, + "ZarrV3MetadataFieldJSON": ZarrV3MetadataFieldJSON, + "ZarrV3ArrayMetadataJSON": ZarrV3ArrayMetadataJSON, + "ZarrV3NamedConfig": ZarrV3NamedConfig, + "ZarrV3MetadataField": ZarrV3NamedConfig, "UNSET": UNSET, }, ) @@ -204,7 +204,7 @@ class ArrayMetadataV3Spec(BaseModel, Generic[AttrsT]): """A pydantic-native, attribute-typed view of a v3 array metadata document. The library is the engine: every input is canonicalized and structurally - validated by `ArrayMetadataModelV3.from_json` before pydantic sees the + validated by `ZarrV3ArrayMetadata.from_json` before pydantic sees the fields, and `to_document` / `to_metadata_model` emit through the library. """ @@ -228,12 +228,12 @@ def _canonicalize(cls, data: object) -> object: """Route every input document through the library's validation and normalization; pydantic then parses only canonical documents.""" if isinstance(data, Mapping): - doc = dict(ArrayMetadataModelV3.from_json(data).to_json()) + doc = dict(ZarrV3ArrayMetadata.from_json(data).to_json()) doc.setdefault("attributes", {}) return doc return data - def to_metadata_model(self) -> ArrayMetadataModelV3: + def to_metadata_model(self) -> ZarrV3ArrayMetadata: """Bridge back to the canonical model, via the document form. In the document, "no dimension names" is key-absence, not null; the @@ -242,7 +242,7 @@ def to_metadata_model(self) -> ArrayMetadataModelV3: doc = self.model_dump() if doc["dimension_names"] is None: del doc["dimension_names"] - return ArrayMetadataModelV3.from_json(doc) + return ZarrV3ArrayMetadata.from_json(doc) def to_document(self) -> dict[str, object]: """The canonical document (omit-empty conventions applied).""" @@ -280,7 +280,7 @@ def test_spec_bridges_to_canonical_model_and_document() -> None: doc = dict(VALID_DOC) | {"attributes": {"resolution_um": 0.5}} spec = ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(doc) model = spec.to_metadata_model() - assert isinstance(model, ArrayMetadataModelV3) + assert isinstance(model, ZarrV3ArrayMetadata) assert spec.to_document() == dict(model.to_json()) # and back: the document revalidates to an equal spec assert ArrayMetadataV3Spec[MicroscopyAttrs].model_validate(spec.to_document()) == spec diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 81b9b962af..56be055c77 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -10,17 +10,17 @@ import zarr_metadata.pydantic as zmp from zarr_metadata.model import ( - ArrayMetadataModelV2, - ArrayMetadataModelV3, - ConsolidatedMetadataModelV2, - ConsolidatedMetadataModelV3, - GroupMetadataModelV2, - GroupMetadataModelV3, - NamedConfigModelV3, + ZarrV2ArrayMetadata, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV3ArrayMetadata, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, + ZarrV3NamedConfig, ) -V3_ARRAY_DOC = dict(ArrayMetadataModelV3.create_default(shape=(4,)).to_json()) -V2_ARRAY_DOC = dict(ArrayMetadataModelV2.create_default(shape=(4,), chunks=(2,)).to_json()) +V3_ARRAY_DOC = dict(ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()) +V2_ARRAY_DOC = dict(ZarrV2ArrayMetadata.create_default(shape=(4,), chunks=(2,)).to_json()) V3_GROUP_DOC = {"zarr_format": 3, "node_type": "group", "attributes": {"a": 1}} V2_GROUP_DOC = {"zarr_format": 2, "attributes": {"a": 1}} V3_CONSOLIDATED_DOC = { @@ -34,23 +34,23 @@ } FIELD_CASES = [ - pytest.param(zmp.ArrayMetadataV3, ArrayMetadataModelV3, V3_ARRAY_DOC, id="array-v3"), - pytest.param(zmp.ArrayMetadataV2, ArrayMetadataModelV2, V2_ARRAY_DOC, id="array-v2"), - pytest.param(zmp.GroupMetadataV3, GroupMetadataModelV3, V3_GROUP_DOC, id="group-v3"), - pytest.param(zmp.GroupMetadataV2, GroupMetadataModelV2, V2_GROUP_DOC, id="group-v2"), + pytest.param(zmp.ZarrV3ArrayMetadata, ZarrV3ArrayMetadata, V3_ARRAY_DOC, id="array-v3"), + pytest.param(zmp.ZarrV2ArrayMetadata, ZarrV2ArrayMetadata, V2_ARRAY_DOC, id="array-v2"), + pytest.param(zmp.ZarrV3GroupMetadata, ZarrV3GroupMetadata, V3_GROUP_DOC, id="group-v3"), + pytest.param(zmp.ZarrV2GroupMetadata, ZarrV2GroupMetadata, V2_GROUP_DOC, id="group-v2"), pytest.param( - zmp.ConsolidatedMetadataV3, - ConsolidatedMetadataModelV3, + zmp.ZarrV3ConsolidatedMetadata, + ZarrV3ConsolidatedMetadata, V3_CONSOLIDATED_DOC, id="consolidated-v3", ), pytest.param( - zmp.ConsolidatedMetadataV2, - ConsolidatedMetadataModelV2, + zmp.ZarrV2ConsolidatedMetadata, + ZarrV2ConsolidatedMetadata, V2_CONSOLIDATED_DOC, id="consolidated-v2", ), - pytest.param(zmp.MetadataFieldV3, NamedConfigModelV3, {"name": "bytes"}, id="field-v3"), + pytest.param(zmp.ZarrV3MetadataField, ZarrV3NamedConfig, {"name": "bytes"}, id="field-v3"), ] @@ -74,9 +74,9 @@ def test_core_instances_interoperate() -> None: the core classes rather than pydantic-aware subclasses.""" class Manifest(BaseModel): - metadata: zmp.ArrayMetadataV3 + metadata: zmp.ZarrV3ArrayMetadata - core = ArrayMetadataModelV3.from_json(V3_ARRAY_DOC) + core = ZarrV3ArrayMetadata.from_json(V3_ARRAY_DOC) manifest = Manifest(metadata=core) assert manifest.metadata is core @@ -85,7 +85,7 @@ def test_validation_error_carries_problems() -> None: """A defective document fails with the library's loc-annotated messages.""" class Manifest(BaseModel): - metadata: zmp.ArrayMetadataV3 + metadata: zmp.ZarrV3ArrayMetadata doc = dict(V3_ARRAY_DOC) del doc["chunk_key_encoding"] @@ -97,11 +97,11 @@ def test_json_schema_generation() -> None: """model_json_schema works, describing the document form each field accepts.""" class Manifest(BaseModel): - metadata: zmp.ArrayMetadataV3 - codec: zmp.MetadataFieldV3 + metadata: zmp.ZarrV3ArrayMetadata + codec: zmp.ZarrV3MetadataField schema = Manifest.model_json_schema() - assert schema["properties"]["metadata"] == {"type": "object", "title": "ArrayMetadataV3"} + assert schema["properties"]["metadata"] == {"type": "object", "title": "ZarrV3ArrayMetadata"} assert schema["properties"]["codec"]["anyOf"] == [{"type": "string"}, {"type": "object"}] @@ -109,7 +109,7 @@ def test_json_roundtrip() -> None: """model_dump_json output re-validates to an equal pydantic model.""" class Manifest(BaseModel): - metadata: zmp.ArrayMetadataV3 + metadata: zmp.ZarrV3ArrayMetadata manifest = Manifest.model_validate({"metadata": V3_ARRAY_DOC}) assert Manifest.model_validate_json(manifest.model_dump_json()) == manifest diff --git a/packages/zarr-metadata/tests/model/test_sentinel.py b/packages/zarr-metadata/tests/model/test_sentinel.py index 78aa60e208..252a4cdd30 100644 --- a/packages/zarr-metadata/tests/model/test_sentinel.py +++ b/packages/zarr-metadata/tests/model/test_sentinel.py @@ -21,11 +21,11 @@ from zarr_metadata.model import ( UNSET, - ArrayMetadataModelV2, - ArrayMetadataModelV3, - ConsolidatedMetadataModelV3, - GroupMetadataModelV2, - GroupMetadataModelV3, + ZarrV2ArrayMetadata, + ZarrV2GroupMetadata, + ZarrV3ArrayMetadata, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, ) # Whole-model cases covering the states we know are problematic for @@ -33,20 +33,20 @@ # same fields in the present state (including present-but-empty, which must # stay distinct from absent), and UNSET nested inside consolidated metadata. MODEL_CASES = { - "array-v3-dimension-names-unset": ArrayMetadataModelV3.create_default(shape=(4,)), - "array-v3-dimension-names-set": ArrayMetadataModelV3.create_default(shape=(2, 2)).update( + "array-v3-dimension-names-unset": ZarrV3ArrayMetadata.create_default(shape=(4,)), + "array-v3-dimension-names-set": ZarrV3ArrayMetadata.create_default(shape=(2, 2)).update( dimension_names=("x", None) ), - "array-v2-attributes-unset": ArrayMetadataModelV2.create_default(shape=(4,)), - "array-v2-attributes-empty": ArrayMetadataModelV2.create_default(shape=(4,), attributes={}), - "group-v2-attributes-unset": GroupMetadataModelV2.create_default(), - "group-v2-attributes-set": GroupMetadataModelV2.create_default(attributes={"a": 1}), - "group-v3-consolidated-unset": GroupMetadataModelV3.create_default(), - "group-v3-consolidated-with-unset-inside": GroupMetadataModelV3.create_default( - consolidated_metadata=ConsolidatedMetadataModelV3( + "array-v2-attributes-unset": ZarrV2ArrayMetadata.create_default(shape=(4,)), + "array-v2-attributes-empty": ZarrV2ArrayMetadata.create_default(shape=(4,), attributes={}), + "group-v2-attributes-unset": ZarrV2GroupMetadata.create_default(), + "group-v2-attributes-set": ZarrV2GroupMetadata.create_default(attributes={"a": 1}), + "group-v3-consolidated-unset": ZarrV3GroupMetadata.create_default(), + "group-v3-consolidated-with-unset-inside": ZarrV3GroupMetadata.create_default( + consolidated_metadata=ZarrV3ConsolidatedMetadata( metadata={ - "child": ArrayMetadataModelV3.create_default(shape=(4,)), - "subgroup": GroupMetadataModelV3.create_default(), + "child": ZarrV3ArrayMetadata.create_default(shape=(4,)), + "subgroup": ZarrV3GroupMetadata.create_default(), } ) ), @@ -65,10 +65,7 @@ def test_unset_copy_preserves_identity() -> None: @pytest.mark.parametrize("model", MODEL_CASES.values(), ids=MODEL_CASES.keys()) def test_model_pickle_round_trip( - model: ArrayMetadataModelV2 - | ArrayMetadataModelV3 - | GroupMetadataModelV2 - | GroupMetadataModelV3, + model: ZarrV2ArrayMetadata | ZarrV3ArrayMetadata | ZarrV2GroupMetadata | ZarrV3GroupMetadata, ) -> None: restored = pickle.loads(pickle.dumps(model)) assert restored == model @@ -76,10 +73,7 @@ def test_model_pickle_round_trip( @pytest.mark.parametrize("model", MODEL_CASES.values(), ids=MODEL_CASES.keys()) def test_model_deepcopy( - model: ArrayMetadataModelV2 - | ArrayMetadataModelV3 - | GroupMetadataModelV2 - | GroupMetadataModelV3, + model: ZarrV2ArrayMetadata | ZarrV3ArrayMetadata | ZarrV2GroupMetadata | ZarrV3GroupMetadata, ) -> None: assert copy.deepcopy(model) == model diff --git a/packages/zarr-metadata/tests/test_partial_equivalence.py b/packages/zarr-metadata/tests/test_partial_equivalence.py index 995a6a21e1..33492b2356 100644 --- a/packages/zarr-metadata/tests/test_partial_equivalence.py +++ b/packages/zarr-metadata/tests/test_partial_equivalence.py @@ -14,17 +14,17 @@ import pytest -from zarr_metadata.v2.array import ArrayMetadataV2, ArrayMetadataV2Partial -from zarr_metadata.v2.group import GroupMetadataV2, GroupMetadataV2Partial -from zarr_metadata.v3.array import ArrayMetadataV3, ArrayMetadataV3Partial -from zarr_metadata.v3.group import GroupMetadataV3, GroupMetadataV3Partial +from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON, ZarrV2ArrayMetadataJSONPartial +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON, ZarrV2GroupMetadataJSONPartial +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON, ZarrV3ArrayMetadataJSONPartial +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON, ZarrV3GroupMetadataJSONPartial # (full, partial) pairs to check. Add new pairs here as more are introduced. PAIRS: list[tuple[type, type]] = [ - (ArrayMetadataV3, ArrayMetadataV3Partial), - (GroupMetadataV3, GroupMetadataV3Partial), - (ArrayMetadataV2, ArrayMetadataV2Partial), - (GroupMetadataV2, GroupMetadataV2Partial), + (ZarrV3ArrayMetadataJSON, ZarrV3ArrayMetadataJSONPartial), + (ZarrV3GroupMetadataJSON, ZarrV3GroupMetadataJSONPartial), + (ZarrV2ArrayMetadataJSON, ZarrV2ArrayMetadataJSONPartial), + (ZarrV2GroupMetadataJSON, ZarrV2GroupMetadataJSONPartial), ] diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index ee26e80c5a..de8e271d28 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -21,43 +21,43 @@ def _group_rank(s: str) -> int: EXPECTED = [ # Category A — metadata-document types - "ArrayMetadataV2", - "ArrayMetadataV2Partial", + "ZarrV2ArrayMetadataJSON", + "ZarrV2ArrayMetadataJSONPartial", "ZArrayMetadata", - "GroupMetadataV2", - "GroupMetadataV2Partial", + "ZarrV2GroupMetadataJSON", + "ZarrV2GroupMetadataJSONPartial", "ZGroupMetadata", - "ConsolidatedMetadataV2", + "ZarrV2ConsolidatedMetadataJSON", "ZAttrsMetadata", - "CodecMetadataV2", - "ArrayMetadataV3", - "ArrayMetadataV3Partial", - "ExtensionFieldV3", - "GroupMetadataV3", - "GroupMetadataV3Partial", - "ConsolidatedMetadataV3", - "NamedConfigV3", - "MetadataV3", + "ZarrV2CodecMetadata", + "ZarrV3ArrayMetadataJSON", + "ZarrV3ArrayMetadataJSONPartial", + "ZarrV3ExtensionField", + "ZarrV3GroupMetadataJSON", + "ZarrV3GroupMetadataJSONPartial", + "ZarrV3ConsolidatedMetadataJSON", + "ZarrV3NamedConfigJSON", + "ZarrV3MetadataFieldJSON", "JSONValue", # Category A' — metadata models (in-memory dataclasses over the documents) - "ArrayMetadataModelV2", - "ArrayMetadataModelV2Partial", - "ArrayMetadataModelV3", - "ArrayMetadataModelV3Partial", - "GroupMetadataModelV2", - "GroupMetadataModelV2Partial", - "GroupMetadataModelV3", - "GroupMetadataModelV3Partial", - "ConsolidatedMetadataModelV2", - "ConsolidatedMetadataModelV3", - "NamedConfigModelV3", - "MetadataFieldModelV3", + "ZarrV2ArrayMetadata", + "ZarrV2ArrayMetadataPartial", + "ZarrV3ArrayMetadata", + "ZarrV3ArrayMetadataPartial", + "ZarrV2GroupMetadata", + "ZarrV2GroupMetadataPartial", + "ZarrV3GroupMetadata", + "ZarrV3GroupMetadataPartial", + "ZarrV2ConsolidatedMetadata", + "ZarrV3ConsolidatedMetadata", + "ZarrV3NamedConfig", + "ZarrV3MetadataField", "ValidationProblem", "MetadataValidationError", "ProblemKind", "UNSET", # v2 data-type encoding union - "DataTypeMetadataV2", + "ZarrV2DataTypeMetadata", # Category B — codec canonical unions "BloscCodecMetadata", "BytesCodecMetadata", @@ -146,9 +146,9 @@ def _group_rank(s: str) -> int: "RawBytesFillValue", # Category E — constant+Literal pairs "ARRAY_ORDER_V2", - "ArrayOrderV2", + "ZarrV2ArrayOrder", "ARRAY_DIMENSION_SEPARATOR_V2", - "ArrayDimensionSeparatorV2", + "ZarrV2ArrayDimensionSeparator", "ENDIANNESS", "Endianness", "BYTES_CODEC_NAME", @@ -226,7 +226,7 @@ def test_promoted_pairs_drift() -> None: (zm.NUMPY_TIME_UNIT, zm.NumpyTimeUnit), (zm.CAST_ROUNDING_MODE, zm.CastRoundingMode), (zm.CAST_OUT_OF_RANGE_MODE, zm.CastOutOfRangeMode), - (zm.ARRAY_ORDER_V2, zm.ArrayOrderV2), + (zm.ARRAY_ORDER_V2, zm.ZarrV2ArrayOrder), ] for const, lit in pairs: assert set(const) == set(get_args(lit)) diff --git a/packages/zarr-metadata/tests/v2/consolidated/test_fixtures.py b/packages/zarr-metadata/tests/v2/consolidated/test_fixtures.py index 9dad66d074..e802c5bef8 100644 --- a/packages/zarr-metadata/tests/v2/consolidated/test_fixtures.py +++ b/packages/zarr-metadata/tests/v2/consolidated/test_fixtures.py @@ -8,11 +8,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v2.consolidated import ConsolidatedMetadataV2 +from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(ConsolidatedMetadataV2) +ADAPTER = TypeAdapter(ZarrV2ConsolidatedMetadataJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) diff --git a/packages/zarr-metadata/tests/v3/array/test_fixtures.py b/packages/zarr-metadata/tests/v3/array/test_fixtures.py index fccd00d481..c84cc4042b 100644 --- a/packages/zarr-metadata/tests/v3/array/test_fixtures.py +++ b/packages/zarr-metadata/tests/v3/array/test_fixtures.py @@ -1,7 +1,7 @@ """Decode v3 array metadata fixtures via pydantic. Each `*.json` file in this directory is a representative on-disk -`zarr.json` that should validate cleanly as `ArrayMetadataV3`. +`zarr.json` that should validate cleanly as `ZarrV3ArrayMetadataJSON`. Fixtures are named for the variant they exercise (regular vs rectilinear grid, blosc/gzip/zstd/sharding_indexed codecs, named-config dtypes, optional fields, extra fields). @@ -15,11 +15,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v3.array import ArrayMetadataV3 +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(ArrayMetadataV3) +ADAPTER = TypeAdapter(ZarrV3ArrayMetadataJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) diff --git a/packages/zarr-metadata/tests/v3/array/with_extra_field.json b/packages/zarr-metadata/tests/v3/array/with_extra_field.json index 46a7f0f235..bd7a9f5b45 100644 --- a/packages/zarr-metadata/tests/v3/array/with_extra_field.json +++ b/packages/zarr-metadata/tests/v3/array/with_extra_field.json @@ -16,6 +16,6 @@ ], "my_custom_extension": { "must_understand": false, - "purpose": "exercise the extra_items=ExtensionFieldV3 path" + "purpose": "exercise the extra_items=ZarrV3ExtensionField path" } } diff --git a/packages/zarr-metadata/tests/v3/consolidated/test_fixtures.py b/packages/zarr-metadata/tests/v3/consolidated/test_fixtures.py index 4d9e300bae..d052b16986 100644 --- a/packages/zarr-metadata/tests/v3/consolidated/test_fixtures.py +++ b/packages/zarr-metadata/tests/v3/consolidated/test_fixtures.py @@ -8,11 +8,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v3.consolidated import ConsolidatedMetadataV3 +from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(ConsolidatedMetadataV3) +ADAPTER = TypeAdapter(ZarrV3ConsolidatedMetadataJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) diff --git a/packages/zarr-metadata/tests/v3/group/test_fixtures.py b/packages/zarr-metadata/tests/v3/group/test_fixtures.py index 2015d5ce96..ffcdedef2b 100644 --- a/packages/zarr-metadata/tests/v3/group/test_fixtures.py +++ b/packages/zarr-metadata/tests/v3/group/test_fixtures.py @@ -8,11 +8,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v3.group import GroupMetadataV3 +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(GroupMetadataV3) +ADAPTER = TypeAdapter(ZarrV3GroupMetadataJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) From a75add93e2114c9e4fdc732702178ee33537ed8a Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Mon, 13 Jul 2026 21:14:32 +0200 Subject: [PATCH 36/70] test: ignore Python 3.15 fork-with-threads DeprecationWarning in test_multiprocessing (#4143) Python 3.15 makes os.fork() emit a DeprecationWarning when the process has multiple threads (python/cpython#84559). zarr's sync event-loop thread is always running by the time test_multiprocessing forks, and the suite runs with filterwarnings = ["error"], so the fork parametrizations fail on 3.15. Fork-safety despite those threads is exactly what this test pins down (zarr registers an os.register_at_fork handler), so keep the coverage and ignore the warning with a targeted per-param filter. Fixes #4086 Assisted-by: ClaudeCode:claude-fable-5 --- changes/4086.misc.md | 1 + tests/test_array.py | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 changes/4086.misc.md diff --git a/changes/4086.misc.md b/changes/4086.misc.md new file mode 100644 index 0000000000..2564cf9ea1 --- /dev/null +++ b/changes/4086.misc.md @@ -0,0 +1 @@ +Fix `test_multiprocessing[fork]` failing on Python 3.15, where `os.fork()` in a multi-threaded process emits a `DeprecationWarning` that the test suite promotes to an error. diff --git a/tests/test_array.py b/tests/test_array.py index 4faf99eb9e..d2f324439d 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -1903,9 +1903,18 @@ def _index_array(arr: AnyArray, index: Any) -> Any: [ pytest.param( "fork", - marks=pytest.mark.skipif( - sys.platform in ("win32", "darwin"), reason="fork not supported on Windows or OSX" - ), + marks=[ + pytest.mark.skipif( + sys.platform in ("win32", "darwin"), + reason="fork not supported on Windows or OSX", + ), + # Python 3.15 deprecates fork() in multi-threaded processes, and zarr's + # sync event-loop thread is always running here. Fork-safety despite + # those threads is exactly what this test pins down, so keep running it. + pytest.mark.filterwarnings( + r"ignore:This process \(pid=\d+\) is multi-threaded, use of fork\(\):DeprecationWarning" + ), + ], ), "spawn", pytest.param( From de2cce1adc41a4d38721bf62b25eb312a52066dd Mon Sep 17 00:00:00 2001 From: Amadeus Brandes <79119643+AMBRA7592@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:48:16 +0200 Subject: [PATCH 37/70] fix: skip object store prefix markers in list_dir (#4117) * fix: skip object store prefix markers in list_dir Signed-off-by: ABrandes <79119643+AMBRA7592@users.noreply.github.com> * Address list_dir prefix marker review Signed-off-by: ABrandes <79119643+AMBRA7592@users.noreply.github.com> * Format prefix marker follow-up Signed-off-by: ABrandes <79119643+AMBRA7592@users.noreply.github.com> * Test S3 prefix marker handling Signed-off-by: ABrandes <79119643+AMBRA7592@users.noreply.github.com> * Filter boto3 deprecation in S3 test Signed-off-by: ABrandes <79119643+AMBRA7592@users.noreply.github.com> * Address final prefix marker review Signed-off-by: ABrandes <79119643+AMBRA7592@users.noreply.github.com> --------- Signed-off-by: ABrandes <79119643+AMBRA7592@users.noreply.github.com> Co-authored-by: Davis Bennett --- changes/4032.bugfix.md | 1 + src/zarr/core/array.py | 6 +++++- src/zarr/storage/_memory.py | 2 +- src/zarr/storage/_obstore.py | 6 +++++- tests/test_array.py | 4 +++- tests/test_store/test_object.py | 37 ++++++++++++++++++++++++++++++++- 6 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 changes/4032.bugfix.md diff --git a/changes/4032.bugfix.md b/changes/4032.bugfix.md new file mode 100644 index 0000000000..4c9476f583 --- /dev/null +++ b/changes/4032.bugfix.md @@ -0,0 +1 @@ +Fixed `ObjectStore.list_dir` for object-store listings that include a directory-marker object matching the requested non-root prefix. diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 2bf761bd3d..f75ef72415 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -4018,7 +4018,11 @@ async def _shards_initialized( x async for x in array.store_path.store.list_prefix(prefix=array.store_path.path) ] store_contents_relative = [ - _relativize_path(path=key, prefix=array.store_path.path) for key in store_contents + _relativize_path(path=key, prefix=array.store_path.path) + for key in store_contents + # obstore can include a directory marker whose key matches the listed prefix; + # it is not an initialized shard and must be excluded before relativizing. + if array.store_path.path == "" or key != array.store_path.path ] return tuple( chunk_key for chunk_key in array._iter_shard_keys() if chunk_key in store_contents_relative diff --git a/src/zarr/storage/_memory.py b/src/zarr/storage/_memory.py index e867706155..5f5b632d4a 100644 --- a/src/zarr/storage/_memory.py +++ b/src/zarr/storage/_memory.py @@ -219,7 +219,7 @@ async def list_dir(self, prefix: str) -> AsyncIterator[str]: keys_unique = { key.removeprefix(f"{prefix}/").split("/")[0] for key in self._store_dict - if key.startswith(f"{prefix}/") and key != prefix + if key.startswith(f"{prefix}/") and key not in {prefix, f"{prefix}/"} } for key in keys_unique: diff --git a/src/zarr/storage/_obstore.py b/src/zarr/storage/_obstore.py index ffea523f9f..b34a5f624d 100644 --- a/src/zarr/storage/_obstore.py +++ b/src/zarr/storage/_obstore.py @@ -267,7 +267,11 @@ async def _transform_list_dir( for path in chain( list_result["common_prefixes"], map(itemgetter("path"), list_result["objects"]) ): - yield _relativize_path(path=path, prefix=prefix) + if prefix != "" and path == prefix: + continue + relpath = _relativize_path(path=path, prefix=prefix) + if relpath: + yield relpath class _BoundedRequest(TypedDict): diff --git a/tests/test_array.py b/tests/test_array.py index d2f324439d..b1a7a3c0f2 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -45,7 +45,7 @@ default_serializer_v3, ) from zarr.core.array_spec import ArrayConfig, ArrayConfigParams -from zarr.core.buffer import NDArrayLike, NDArrayLikeOrScalar, default_buffer_prototype +from zarr.core.buffer import NDArrayLike, NDArrayLikeOrScalar, cpu, default_buffer_prototype from zarr.core.chunk_grids import ( SHARDED_INNER_CHUNK_MAX_BYTES, guess_chunks, @@ -447,6 +447,8 @@ async def test_chunks_initialized( arr = zarr.create_array( store, name=path, shape=shape, shards=shard_shape, chunks=chunk_shape, dtype="i1" ) + if path: + await store.set(path, cpu.Buffer.from_bytes(b"")) chunks_accumulated = tuple( accumulate(tuple(tuple(v.split(" ")) for v in arr._iter_shard_keys())) diff --git a/tests/test_store/test_object.py b/tests/test_store/test_object.py index 6a4b796639..cd85a48eb8 100644 --- a/tests/test_store/test_object.py +++ b/tests/test_store/test_object.py @@ -1,4 +1,5 @@ # ruff: noqa: E402 +import re from pathlib import Path from typing import TypedDict @@ -9,9 +10,10 @@ from hypothesis.stateful import ( run_state_machine_as_test, ) -from obstore.store import LocalStore, MemoryStore +from obstore.store import LocalStore, MemoryStore, S3Store from zarr.core.buffer import Buffer, cpu +from zarr.core.sync import _collect_aiterator from zarr.storage import ObjectStore from zarr.testing.stateful import ZarrHierarchyStateMachine from zarr.testing.store import StoreTests @@ -97,6 +99,39 @@ async def test_store_getsize_prefix(self, store: ObjectStore[LocalStore]) -> Non assert total_size == len(buf) * 2 +@pytest.mark.filterwarnings( + re.escape("ignore:datetime.datetime.utcnow() is deprecated:DeprecationWarning") +) +async def test_list_dir_ignores_s3_prefix_marker(moto_server: str) -> None: + """Ensure obstore's exact-prefix S3 directory marker is not listed as a child.""" + boto3 = pytest.importorskip("boto3") + bucket = "object-store-prefix-marker" + client = boto3.client( + "s3", + endpoint_url=moto_server, + region_name="us-east-1", + aws_access_key_id="x", + aws_secret_access_key="x", + ) + client.create_bucket(Bucket=bucket) + client.put_object(Bucket=bucket, Key="g/", Body=b"") + + store = ObjectStore( + S3Store( + bucket=bucket, + endpoint=moto_server, + region="us-east-1", + access_key_id="x", + secret_access_key="x", + client_options={"allow_http": True}, + virtual_hosted_style_request=False, + ) + ) + + assert await _collect_aiterator(store.list_dir("g")) == () + assert await _collect_aiterator(store.list_dir("g/")) == () + + @pytest.mark.slow_hypothesis def test_zarr_hierarchy() -> None: sync_store = ObjectStore(MemoryStore()) From bbe27d7ce9713d50e34df2cda5108f4dcf410b92 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Tue, 14 Jul 2026 13:39:53 +0200 Subject: [PATCH 38/70] fix: byte-order handling for structured dtypes in the bytes codec (#4142) * fix: byte-order handling for structured dtypes in the bytes codec The bytes codec neither byte-swapped structured-dtype fields to its configured endian on encode (numpy reports byteorder '|' for void dtypes, so the top-level byteorder comparison never detected a mismatch) nor honored its endian when decoding, silently corrupting any structured data whose field byte order differed from the stored one (e.g. virtual references to external big-endian data). Encode now detects byte-order mismatches by comparing full dtypes via newbyteorder, and decode reinterprets raw bytes in the stored byte order before converting to the data type's declared byte order, so the stored layout (codec state) and the in-memory layout (array data type) are independent. Closes #4141 Assisted-by: ClaudeCode:claude-fable-5 * test: fold structured byte-order cases into existing bytes codec tests Extend test_endian's parametrization with structured dtypes and test_bytes_codec_sync_roundtrip with endian/dtype parametrization plus stored-layout and decoded-dtype assertions, instead of adding parallel test functions for the same properties. Assisted-by: ClaudeCode:claude-fable-5 * refactor: rename stored_dtype to view_dtype in BytesCodec decode The variable is the dtype used to view the raw chunk bytes (byte order from the codec's endian configuration), not a property of the stored data or of the returned buffer, which always carries the array's declared dtype. Assisted-by: ClaudeCode:claude-fable-5 * docs: note that the decode-side byte-order conversion copies the chunk Assisted-by: ClaudeCode:claude-fable-5 --- changes/4141.bugfix.md | 1 + src/zarr/codecs/bytes.py | 33 +++++++++++----- tests/test_codecs/test_bytes.py | 69 ++++++++++++++++++++++++++------- 3 files changed, 80 insertions(+), 23 deletions(-) create mode 100644 changes/4141.bugfix.md diff --git a/changes/4141.bugfix.md b/changes/4141.bugfix.md new file mode 100644 index 0000000000..6a132da3f5 --- /dev/null +++ b/changes/4141.bugfix.md @@ -0,0 +1 @@ +Fix silent byte-order corruption for structured dtypes with the `bytes` codec: multi-byte fields are now byte-swapped to the codec's configured `endian` on write and decoded honoring it on read, so non-native-endian structured data (e.g. big-endian fields, as produced by virtual references to external data) round-trips correctly. diff --git a/src/zarr/codecs/bytes.py b/src/zarr/codecs/bytes.py index 240c077627..fae762fd08 100644 --- a/src/zarr/codecs/bytes.py +++ b/src/zarr/codecs/bytes.py @@ -100,14 +100,28 @@ def _decode_sync( chunk_spec: ArraySpec, ) -> NDBuffer: endian_str = self.endian + dtype = chunk_spec.dtype.to_native_dtype() + # The byte order of the stored data is set by this codec's `endian` + # configuration; the byte order of the decoded array is set by the array's + # data type. The two are independent: the raw bytes are viewed with a dtype + # in the stored byte order, then converted to the declared dtype if needed. if isinstance(chunk_spec.dtype, HasEndianness): - dtype = replace(chunk_spec.dtype, endianness=endian_str).to_native_dtype() # type: ignore[call-arg] + view_dtype = replace(chunk_spec.dtype, endianness=endian_str).to_native_dtype() # type: ignore[call-arg] + elif isinstance(chunk_spec.dtype, Struct) and endian_str is not None: + # Per the struct data type spec, all multi-byte fields are stored in the + # byte order configured on this codec. + view_dtype = dtype.newbyteorder(endian_str) else: - dtype = chunk_spec.dtype.to_native_dtype() + view_dtype = dtype as_array_like = chunk_bytes.as_array_like() chunk_array = chunk_spec.prototype.nd_buffer.from_ndarray_like( - as_array_like.view(dtype=dtype) # type: ignore[attr-defined] + as_array_like.view(dtype=view_dtype) # type: ignore[attr-defined] ) + if view_dtype != dtype: + # This byte-swapping conversion copies the chunk. The dtype inequality + # guard keeps the common case, where the stored and declared byte orders + # already match, on the zero-copy view path above. + chunk_array = chunk_array.astype(dtype) # ensure correct chunk shape if chunk_array.shape != chunk_spec.shape: @@ -129,13 +143,14 @@ def _encode_sync( chunk_spec: ArraySpec, ) -> Buffer | None: assert isinstance(chunk_array, NDBuffer) - if ( - chunk_array.dtype.itemsize > 1 - and self.endian is not None - and self.endian != chunk_array.byteorder - ): + if chunk_array.dtype.itemsize > 1 and self.endian is not None: + # Compare full dtypes rather than the top-level byteorder: numpy reports + # byteorder '|' for structured dtypes even when their fields are + # byte-order-sensitive, so newbyteorder is the only reliable way to + # detect (and normalize) a byte-order mismatch. new_dtype = chunk_array.dtype.newbyteorder(self.endian) - chunk_array = chunk_array.astype(new_dtype) + if new_dtype != chunk_array.dtype: + chunk_array = chunk_array.astype(new_dtype) nd_array = chunk_array.as_ndarray_like() # Flatten the nd-array (only copy if needed) and reinterpret as bytes diff --git a/tests/test_codecs/test_bytes.py b/tests/test_codecs/test_bytes.py index 03dd0b40c6..ead778f526 100644 --- a/tests/test_codecs/test_bytes.py +++ b/tests/test_codecs/test_bytes.py @@ -33,29 +33,50 @@ @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) -@pytest.mark.parametrize("input_dtype", [">u2", "u2", + "f4"), ("mask", ">i4")], + [("flux", "u2", " None: """ The `bytes` codec stores multi-byte data in the byte order configured on the codec, regardless of the input array's byte order, and reads it back to the - original values. The input-dtype/store-endian cross-product exercises the - encode-side byteswap (input byte order != store byte order) and the no-op - case alike. Compression is disabled so the stored chunk is the codec's raw - output and its byte layout can be asserted directly. + original values. For structured dtypes this applies to every multi-byte + field, per the `struct` data type spec; the struct cases guard against the + endianness bugs from + https://github.com/zarr-developers/zarr-python/issues/4141, where the + encode path never byte-swapped struct fields (numpy reports byteorder '|' + for void dtypes) and the decode path ignored the codec's endian entirely. + The input-dtype/store-endian cross-product exercises the encode-side + byteswap (input byte order != store byte order) and the no-op case alike. + Compression is disabled so the stored chunk is the codec's raw output and + its byte layout can be asserted directly. """ - data = np.arange(0, 256, dtype=input_dtype).reshape((16, 16)) + dtype = np.dtype(input_dtype) + if dtype.fields is None: + data = np.arange(0, 256, dtype=dtype).reshape((16, 16)) + else: + data = np.zeros((16, 16), dtype=dtype) + data["flux"] = np.arange(0, 256).reshape((16, 16)) + data["mask"] = np.arange(256, 512).reshape((16, 16)) path = "endian" spath = StorePath(store, path) a = await zarr.api.asynchronous.create_array( spath, shape=data.shape, chunks=(16, 16), - dtype="uint16", + dtype=dtype, fill_value=0, compressors=None, serializer=BytesCodec(endian=store_endian), @@ -66,8 +87,7 @@ async def test_endian( # The stored chunk is laid out in the byte order configured on the codec. stored = await store.get(f"{path}/c/0/0", prototype=default_buffer_prototype()) assert stored is not None - expected_dtype = ">u2" if store_endian == "big" else " None: assert isinstance(BytesCodec(), SupportsSyncCodec) -def test_bytes_codec_sync_roundtrip() -> None: - codec = BytesCodec() - arr = np.arange(100, dtype="float64") +@pytest.mark.parametrize("endian", ENDIAN) +@pytest.mark.parametrize( + "native_dtype", + [np.dtype("float64"), np.dtype(">u2"), np.dtype([("a", ">f4"), ("b", " None: + """ + The synchronous encode/decode path round-trips data, and the two byte + orders involved are independent: the codec's `endian` configuration governs + only the stored byte layout (every multi-byte value, including struct + fields, is laid out in the codec's byte order regardless of the input + array's byte order), while the decoded buffer's byte order is governed by + the array's data type regardless of the codec's. The mixed-endian struct + case pins that per-field byte order of the in-memory dtype survives a + roundtrip through a single stored byte order. + """ + if native_dtype.fields is None: + arr = np.arange(100, dtype=native_dtype) + else: + arr = np.array([(1.5, 2), (3.5, 4), (5.5, 6), (7.5, 8)], dtype=native_dtype) zdtype = get_data_type_from_native_dtype(arr.dtype) spec = ArraySpec( shape=arr.shape, @@ -91,11 +129,14 @@ def test_bytes_codec_sync_roundtrip() -> None: ) nd_buf: NDBuffer = default_buffer_prototype().nd_buffer.from_numpy_array(arr) - codec = codec.evolve_from_array_spec(spec) + codec = BytesCodec(endian=endian).evolve_from_array_spec(spec) encoded = codec._encode_sync(nd_buf, spec) assert encoded is not None + assert encoded.to_bytes() == arr.astype(native_dtype.newbyteorder(endian)).tobytes() + decoded = codec._decode_sync(encoded, spec) + assert decoded.dtype == zdtype.to_native_dtype() np.testing.assert_array_equal(arr, decoded.as_numpy_array()) From d916f759dca7ee41bfa56f51a2295b8b0e90a81a Mon Sep 17 00:00:00 2001 From: Ilan Gold Date: Tue, 14 Jul 2026 17:13:03 +0200 Subject: [PATCH 39/70] chore: add Ilan Gold to the team list (#4145) --- TEAM.md | 1 + 1 file changed, 1 insertion(+) diff --git a/TEAM.md b/TEAM.md index dc22a1ee87..ce9de1d486 100644 --- a/TEAM.md +++ b/TEAM.md @@ -11,6 +11,7 @@ - @dcherian (Deepak Cherian) - @TomAugspurger (Tom Augspurger) - @maxrjones (Max Jones) +- @ilan-gold (Ilan Gold) ## Emeritus core-developers - @alimanfoo (Alistair Miles) From 91a8ec06dc4d475fe40691e0ab668dd655d3751a Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Thu, 16 Jul 2026 16:34:50 +0200 Subject: [PATCH 40/70] docs: 3.3.0 release notes (#4148) * chore(deps): bump the actions group across 1 directory with 8 updates (#176) Bumps the actions group with 8 updates in the / directory: | Package | From | To | | --- | --- | --- | | [prefix-dev/setup-pixi](https://github.com/prefix-dev/setup-pixi) | `0.9.5` | `0.9.6` | | [codecov/codecov-action](https://github.com/codecov/codecov-action) | `6.0.0` | `6.0.1` | | [github/issue-metrics](https://github.com/github/issue-metrics) | `4.2.2` | `4.2.7` | | [j178/prek-action](https://github.com/j178/prek-action) | `2.0.3` | `2.0.4` | | [actions/upload-artifact](https://github.com/actions/upload-artifact) | `7.0.0` | `7.0.1` | | [actions/download-artifact](https://github.com/actions/download-artifact) | `7.0.0` | `8.0.1` | | [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) | `1.13.0` | `1.14.0` | | [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) | `0.5.3` | `0.5.6` | Updates `prefix-dev/setup-pixi` from 0.9.5 to 0.9.6 - [Release notes](https://github.com/prefix-dev/setup-pixi/releases) - [Commits](https://github.com/prefix-dev/setup-pixi/compare/1b2de7f3351f171c8b4dfeb558c639cb58ed4ec0...5185adfbffb4bd703da3010310260805d89ebb11) Updates `codecov/codecov-action` from 6.0.0 to 6.0.1 - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/57e3a136b779b570ffcdbf80b3bdc90e7fab3de2...e79a6962e0d4c0c17b229090214935d2e33f8354) Updates `github/issue-metrics` from 4.2.2 to 4.2.7 - [Release notes](https://github.com/github/issue-metrics/releases) - [Commits](https://github.com/github/issue-metrics/compare/c9e9838147fd355dace335ba787f01b6641a400a...1e38d5e62363e14db8019ed7d106b9855bdba6cc) Updates `j178/prek-action` from 2.0.3 to 2.0.4 - [Release notes](https://github.com/j178/prek-action/releases) - [Commits](https://github.com/j178/prek-action/compare/6ad80277337ad479fe43bd70701c3f7f8aa74db3...bdca6f102f98e2b4c7029491a53dfd366469e33d) Updates `actions/upload-artifact` from 7.0.0 to 7.0.1 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v7...043fb46d1a93c77aae656e7c1c64a875d1fc6a0a) Updates `actions/download-artifact` from 7.0.0 to 8.0.1 - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v7...3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c) Updates `pypa/gh-action-pypi-publish` from 1.13.0 to 1.14.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.13.0...cef221092ed1bacb1cc03d23a2d87d1d172e277b) Updates `zizmorcore/zizmor-action` from 0.5.3 to 0.5.6 - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/b1d7e1fb5de872772f31590499237e7cce841e8e...5f14fd08f7cf1cb1609c1e344975f152c7ee938d) --- updated-dependencies: - dependency-name: prefix-dev/setup-pixi dependency-version: 0.9.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: codecov/codecov-action dependency-version: 6.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: github/issue-metrics dependency-version: 4.2.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: j178/prek-action dependency-version: 2.0.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/upload-artifact dependency-version: 7.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/download-artifact dependency-version: 8.0.1 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: pypa/gh-action-pypi-publish dependency-version: 1.14.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: zizmorcore/zizmor-action dependency-version: 0.5.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * fix: byte-order handling for structured dtypes in the bytes codec (#220) * fix: byte-order handling for structured dtypes in the bytes codec The bytes codec neither byte-swapped structured-dtype fields to its configured endian on encode (numpy reports byteorder '|' for void dtypes, so the top-level byteorder comparison never detected a mismatch) nor honored its endian when decoding, silently corrupting any structured data whose field byte order differed from the stored one (e.g. virtual references to external big-endian data). Encode now detects byte-order mismatches by comparing full dtypes via newbyteorder, and decode reinterprets raw bytes in the stored byte order before converting to the data type's declared byte order, so the stored layout (codec state) and the in-memory layout (array data type) are independent. Closes #4141 Assisted-by: ClaudeCode:claude-fable-5 * test: fold structured byte-order cases into existing bytes codec tests Extend test_endian's parametrization with structured dtypes and test_bytes_codec_sync_roundtrip with endian/dtype parametrization plus stored-layout and decoded-dtype assertions, instead of adding parallel test functions for the same properties. Assisted-by: ClaudeCode:claude-fable-5 * refactor: rename stored_dtype to view_dtype in BytesCodec decode The variable is the dtype used to view the raw chunk bytes (byte order from the codec's endian configuration), not a property of the stored data or of the returned buffer, which always carries the array's declared dtype. Assisted-by: ClaudeCode:claude-fable-5 * docs: note that the decode-side byte-order conversion copies the chunk Assisted-by: ClaudeCode:claude-fable-5 * docs: 3.3.0 release notes Add missing changelog fragments for #3955 (datetime64/timedelta64 V3 metadata types), #3966 (writes to 0-dimensional sharded arrays), and the public alias renames that accompanied the #3963/#3968 enum deprecations, then build the 3.3.0 release notes with towncrier, consuming all fragments accumulated since v3.2.1. Co-Authored-By: Claude Fable 5 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- changes/202.bugfix.md | 1 - changes/214.misc.md | 1 - changes/215.misc.md | 1 - changes/2929.bugfix.md | 3 - changes/3004.feature.md | 4 - changes/3009.doc.md | 1 - changes/3417.bugfix.md | 4 - changes/3469.bugfix.md | 1 - changes/3826.feature.md | 1 - changes/3885.bugfix.md | 1 - changes/3885.feature.1.md | 1 - changes/3885.feature.md | 1 - changes/3899.bugfix.md | 7 -- changes/3908.misc.md | 1 - changes/3925.feature.md | 1 - changes/3954.bugfix.md | 1 - changes/3963.removal.md | 6 -- changes/3968.removal.md | 17 ----- changes/3972.misc.md | 1 - changes/3973.removal.md | 1 - changes/3975.misc.md | 1 - changes/3977.bugfix.md | 1 - changes/3979.misc.md | 1 - changes/3984.doc.md | 4 - changes/3987.feature.md | 1 - changes/3990.misc.md | 5 -- changes/3998.misc.md | 1 - changes/4000.misc.md | 1 - changes/4001.misc.md | 6 -- changes/4003.bugfix.md | 19 ----- changes/4016.bugfix.md | 1 - changes/4032.bugfix.md | 1 - changes/4046.misc.md | 1 - changes/4052.doc.md | 21 ----- changes/4053.doc.md | 1 - changes/4054.misc.md | 1 - changes/4059.bugfix.md | 1 - changes/4073.misc.md | 1 - changes/4074.bugfix.md | 7 -- changes/4086.misc.md | 1 - changes/4116.bugfix.md | 1 - changes/4132.doc.md | 6 -- changes/4133.doc.md | 1 - changes/4138.misc.md | 1 - changes/4141.bugfix.md | 1 - docs/release-notes.md | 156 ++++++++++++++++++++++++++++++++++++++ 46 files changed, 156 insertions(+), 141 deletions(-) delete mode 100644 changes/202.bugfix.md delete mode 100644 changes/214.misc.md delete mode 100644 changes/215.misc.md delete mode 100644 changes/2929.bugfix.md delete mode 100644 changes/3004.feature.md delete mode 100644 changes/3009.doc.md delete mode 100644 changes/3417.bugfix.md delete mode 100644 changes/3469.bugfix.md delete mode 100644 changes/3826.feature.md delete mode 100644 changes/3885.bugfix.md delete mode 100644 changes/3885.feature.1.md delete mode 100644 changes/3885.feature.md delete mode 100644 changes/3899.bugfix.md delete mode 100644 changes/3908.misc.md delete mode 100644 changes/3925.feature.md delete mode 100644 changes/3954.bugfix.md delete mode 100644 changes/3963.removal.md delete mode 100644 changes/3968.removal.md delete mode 100644 changes/3972.misc.md delete mode 100644 changes/3973.removal.md delete mode 100644 changes/3975.misc.md delete mode 100644 changes/3977.bugfix.md delete mode 100644 changes/3979.misc.md delete mode 100644 changes/3984.doc.md delete mode 100644 changes/3987.feature.md delete mode 100644 changes/3990.misc.md delete mode 100644 changes/3998.misc.md delete mode 100644 changes/4000.misc.md delete mode 100644 changes/4001.misc.md delete mode 100644 changes/4003.bugfix.md delete mode 100644 changes/4016.bugfix.md delete mode 100644 changes/4032.bugfix.md delete mode 100644 changes/4046.misc.md delete mode 100644 changes/4052.doc.md delete mode 100644 changes/4053.doc.md delete mode 100644 changes/4054.misc.md delete mode 100644 changes/4059.bugfix.md delete mode 100644 changes/4073.misc.md delete mode 100644 changes/4074.bugfix.md delete mode 100644 changes/4086.misc.md delete mode 100644 changes/4116.bugfix.md delete mode 100644 changes/4132.doc.md delete mode 100644 changes/4133.doc.md delete mode 100644 changes/4138.misc.md delete mode 100644 changes/4141.bugfix.md diff --git a/changes/202.bugfix.md b/changes/202.bugfix.md deleted file mode 100644 index 9c9bd40f21..0000000000 --- a/changes/202.bugfix.md +++ /dev/null @@ -1 +0,0 @@ -Stop emitting an `UnstableSpecificationWarning` when serializing the `struct` data type to Zarr V3 metadata. The `struct` data type now has a stable Zarr V3 specification. The legacy `structured` alias and the unspecified `null_terminated_bytes`, `raw_bytes`, and `variable_length_bytes` data types continue to warn. diff --git a/changes/214.misc.md b/changes/214.misc.md deleted file mode 100644 index 8611362e84..0000000000 --- a/changes/214.misc.md +++ /dev/null @@ -1 +0,0 @@ -Bind the shared moto S3 test server to an ephemeral port instead of a fixed one. The fixed port deadlocked `pytest -n auto`: session-scoped fixtures run once per xdist worker, so concurrent workers raced to bind the same port and the losers blocked forever in `ThreadedMotoServer.start()`. diff --git a/changes/215.misc.md b/changes/215.misc.md deleted file mode 100644 index f8df7cc632..0000000000 --- a/changes/215.misc.md +++ /dev/null @@ -1 +0,0 @@ -Enable pytest's `strict = true` config option (strict config, markers, xfail, and parametrization ids), replacing the `--strict-config`/`--strict-markers` addopts flags that pytest silently ignored before 9.1, and fix the seven duplicate parametrization ids it surfaced — including restoring float-JSON roundtrip cases that were meant to cover numpy scalars but ran plain-float cases twice instead. Also arm pytest's faulthandler watchdog (`faulthandler_timeout = 600` with `faulthandler_exit_on_timeout`) so a deadlocked test dumps every thread's traceback and fails the run instead of hanging indefinitely. diff --git a/changes/2929.bugfix.md b/changes/2929.bugfix.md deleted file mode 100644 index 533a5f86c1..0000000000 --- a/changes/2929.bugfix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fix equality comparison of `ArrayV2Metadata` and `ArrayV3Metadata` objects with a -`NaN` fill value. Such objects are now compared by their JSON-serialized form, so two -otherwise-identical metadata objects with a `NaN` (or infinite) fill value compare equal. diff --git a/changes/3004.feature.md b/changes/3004.feature.md deleted file mode 100644 index 9d3816a50c..0000000000 --- a/changes/3004.feature.md +++ /dev/null @@ -1,4 +0,0 @@ -Optimizes reading multiple chunks from a shard. Serial calls to `Store.get()` -in the sharding codec have been replaced with a single call to -`Store.get_ranges()`, which coalesces nearby byte ranges and fetches them -concurrently. diff --git a/changes/3009.doc.md b/changes/3009.doc.md deleted file mode 100644 index 777672b77b..0000000000 --- a/changes/3009.doc.md +++ /dev/null @@ -1 +0,0 @@ -Document the changes to `zarr.errors` in the 3.0 migration guide, including the removal of v2 exception classes and the introduction of `NodeNotFoundError`. diff --git a/changes/3417.bugfix.md b/changes/3417.bugfix.md deleted file mode 100644 index be5b44f3e9..0000000000 --- a/changes/3417.bugfix.md +++ /dev/null @@ -1,4 +0,0 @@ -Fixed `BytesCodec.from_dict` so that `BytesCodec` instances roundtrip to / from -their dict representation. `BytesCodec.from_dict` now interprets a missing -`endian` configuration as `endian=None` (matching what `BytesCodec.to_dict` -emits), instead of falling back to the system's native byte order. diff --git a/changes/3469.bugfix.md b/changes/3469.bugfix.md deleted file mode 100644 index eb56e87476..0000000000 --- a/changes/3469.bugfix.md +++ /dev/null @@ -1 +0,0 @@ -Fixed `save_array`, `Group.__setitem__`, and `load` for 0-dimensional arrays. diff --git a/changes/3826.feature.md b/changes/3826.feature.md deleted file mode 100644 index 8909e777c5..0000000000 --- a/changes/3826.feature.md +++ /dev/null @@ -1 +0,0 @@ -Added a `subchunk_write_order` option to `ShardingCodec` to control the physical order of subchunks within a shard. Supported values are `morton`, `unordered`, `lexicographic`, and `colexicographic`. `unordered` makes no guarantee about subchunk layout. This setting affects only on-disk layout, not the data read back, and is not persisted in array metadata: it applies per codec instance and is not recovered when reopening a sharded array. diff --git a/changes/3885.bugfix.md b/changes/3885.bugfix.md deleted file mode 100644 index d4c1dd5c03..0000000000 --- a/changes/3885.bugfix.md +++ /dev/null @@ -1 +0,0 @@ -Fixed inner-codec spec evolution for sharded arrays. The sharding codec now threads the array spec through its inner codec chain when evolving codecs, so a codec that changes the dtype upstream of `BytesCodec` no longer leaves the inner chain evolved against the wrong spec (which previously failed at decode time). This runs on the default `BatchedCodecPipeline` as well. Standard inner chains (`[BytesCodec]`, `[BytesCodec, ZstdCodec]`, transpose + bytes) are byte-identical to before. Restores the behavior of #2179. diff --git a/changes/3885.feature.1.md b/changes/3885.feature.1.md deleted file mode 100644 index 0a51ba6844..0000000000 --- a/changes/3885.feature.1.md +++ /dev/null @@ -1 +0,0 @@ -Added `SyncByteGetter` and `SyncByteSetter` runtime-checkable protocols and a `get_ranges_sync` method on the `Store` ABC. These let custom byte getters/setters opt into the synchronous codec pipeline's fast path for in-memory IO, which the sharding codec uses for its inner chunks. diff --git a/changes/3885.feature.md b/changes/3885.feature.md deleted file mode 100644 index 13010f521e..0000000000 --- a/changes/3885.feature.md +++ /dev/null @@ -1 +0,0 @@ -Added `FusedCodecPipeline`, an opt-in codec pipeline that runs codec compute synchronously and in bulk (avoiding the per-chunk async scheduling overhead of the default `BatchedCodecPipeline`), giving large speedups for sharded arrays (up to ~24x writes / ~14x reads on many-chunks-per-shard layouts, more with compression) and no regressions on compute-bound workloads. The default `BatchedCodecPipeline` is unchanged for standard configurations, so existing code keeps working unless you opt in; enable the new pipeline with `zarr.config.set({"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"})`. diff --git a/changes/3899.bugfix.md b/changes/3899.bugfix.md deleted file mode 100644 index e2b2177a39..0000000000 --- a/changes/3899.bugfix.md +++ /dev/null @@ -1,7 +0,0 @@ -Make chunk normalization properly handle `-1` as a compact representation of the -length of an entire axis. Reject several previously-accepted but ill-defined -chunk specifications: `chunks=True` (previously silently produced size-1 chunks), -chunk tuples shorter than the array's number of dimensions (previously padded to -the array's shape), and `None` as a per-dimension chunk size. These all now -raise informative errors. Also fix chunk handling for 0-length array dimensions, -and add explicit rejection of 0-length chunks. diff --git a/changes/3908.misc.md b/changes/3908.misc.md deleted file mode 100644 index 66717e8444..0000000000 --- a/changes/3908.misc.md +++ /dev/null @@ -1 +0,0 @@ -Reuse a constant `ArraySpec` during indexing when possible. \ No newline at end of file diff --git a/changes/3925.feature.md b/changes/3925.feature.md deleted file mode 100644 index ed07be309c..0000000000 --- a/changes/3925.feature.md +++ /dev/null @@ -1 +0,0 @@ -Add `zarr.abc.store.Store.get_ranges` for concurrent, coalesced multi-range reads from a single key. The method is defined on the `Store` ABC with a default implementation built on `Store.get`, so every store inherits a working version; stores with native multi-range backends (e.g. `FsspecStore`) can override for efficiency. Coalescing knobs (`max_concurrency`, `max_gap_bytes`, `max_coalesced_bytes`) are passed as keyword arguments to `get_ranges`. Failures from underlying fetches surface as a `BaseExceptionGroup` (PEP 654); callers should use `except*` to filter for specific exception types such as `FileNotFoundError`. diff --git a/changes/3954.bugfix.md b/changes/3954.bugfix.md deleted file mode 100644 index 1a6c93a8ac..0000000000 --- a/changes/3954.bugfix.md +++ /dev/null @@ -1 +0,0 @@ -Handle missing consolidated metadata in leaf Group nodes. diff --git a/changes/3963.removal.md b/changes/3963.removal.md deleted file mode 100644 index a83fd53853..0000000000 --- a/changes/3963.removal.md +++ /dev/null @@ -1,6 +0,0 @@ -The ``BloscShuffle`` and ``BloscCname`` enums (``zarr.codecs.BloscShuffle``, -``zarr.codecs.BloscCname``) are now deprecated. Pass the equivalent literal -string (e.g. ``"zstd"``, ``"bitshuffle"``) when constructing a ``BloscCodec``. -The enum classes remain importable but emit ``DeprecationWarning`` on member -access, and will be removed in a future release. ``BloscCodec.cname`` and -``BloscCodec.shuffle`` are now plain strings rather than enum members. diff --git a/changes/3968.removal.md b/changes/3968.removal.md deleted file mode 100644 index 075e4a15a9..0000000000 --- a/changes/3968.removal.md +++ /dev/null @@ -1,17 +0,0 @@ -The ``Endian`` (``zarr.codecs.bytes.Endian``) and ``ShardingCodecIndexLocation`` -(``zarr.codecs.ShardingCodecIndexLocation``) enums are now deprecated. Pass the -equivalent literal string instead (e.g. ``"little"`` / ``"big"``, ``"start"`` / -``"end"``). The enum classes remain importable but emit ``DeprecationWarning`` -on member access, and will be removed in a future release. ``BytesCodec.endian`` -and ``ShardingCodec.index_location`` are now plain strings rather than enum -members. - -Two follow-on changes from this deprecation: - -- ``NDBuffer.byteorder`` now returns a literal string (``"little"`` or - ``"big"``) rather than an ``Endian`` member. Subclasses overriding this - property should update their return type. -- The module-level binding ``zarr.codecs.bytes.default_system_endian`` was - removed. ``BytesCodec()`` continues to default to ``sys.byteorder``; - external callers that imported ``default_system_endian`` should use - ``sys.byteorder`` directly. diff --git a/changes/3972.misc.md b/changes/3972.misc.md deleted file mode 100644 index 60e5f75cca..0000000000 --- a/changes/3972.misc.md +++ /dev/null @@ -1 +0,0 @@ -Run `mypy` via `uv run mypy` instead of `pre-commit`'s isolated venv. The `dev` dependency group in `pyproject.toml`, locked by `uv.lock`, is now the single source of truth for `mypy`'s dependency set, eliminating the duplicate dependency list previously maintained in `.pre-commit-config.yaml` and giving every contributor and CI an identical, reproducible type-checking environment. diff --git a/changes/3973.removal.md b/changes/3973.removal.md deleted file mode 100644 index c25ac967a2..0000000000 --- a/changes/3973.removal.md +++ /dev/null @@ -1 +0,0 @@ -Removed the NumPy 1.x implementation of the `VariableLengthUTF8` data type because NumPy 1.x is no longer supported under [SPEC0](https://scientific-python.org/specs/spec-0000/). \ No newline at end of file diff --git a/changes/3975.misc.md b/changes/3975.misc.md deleted file mode 100644 index b90147988a..0000000000 --- a/changes/3975.misc.md +++ /dev/null @@ -1 +0,0 @@ -Store `chunks_per_shard` explicitly as a field on `_ShardIndex` instead of inferring it from `offsets_and_lengths.shape[:-1]`. The previous derivation collapsed to rank-1 for 0-D arrays, requiring a numpy-compat cast workaround that is now removed. Also removes the unused `_ShardIndex.is_dense` method, which was ported from an earlier prototype and never had any call sites or tests. diff --git a/changes/3977.bugfix.md b/changes/3977.bugfix.md deleted file mode 100644 index 6a8d9b4244..0000000000 --- a/changes/3977.bugfix.md +++ /dev/null @@ -1 +0,0 @@ -Fix flaky stateful test bookkeeping when `delete_dir` matches string prefixes instead of true directory descendants. Previously a path such as `6/faNT…` could be incorrectly removed when deleting `6/f`. (See [issue #3977](https://github.com/zarr-developers/zarr-python/issues/3977).) diff --git a/changes/3979.misc.md b/changes/3979.misc.md deleted file mode 100644 index 3d49547148..0000000000 --- a/changes/3979.misc.md +++ /dev/null @@ -1 +0,0 @@ -Remove the `UnstableSpecificationWarning` emitted when serializing a `FixedLengthUTF32` data type instance, as a spec for this data type has been published to zarr-extensions. \ No newline at end of file diff --git a/changes/3984.doc.md b/changes/3984.doc.md deleted file mode 100644 index 1b92fb8e74..0000000000 --- a/changes/3984.doc.md +++ /dev/null @@ -1,4 +0,0 @@ -Clarify the difference between `zarr.load` and `zarr.open` in their docstrings. -`load` eagerly reads data into an in-memory array, while `open` returns a -lazy `Array` or `Group` backed by the store, with `See Also` cross-references -linking the two. diff --git a/changes/3987.feature.md b/changes/3987.feature.md deleted file mode 100644 index 2492b4d7dd..0000000000 --- a/changes/3987.feature.md +++ /dev/null @@ -1 +0,0 @@ -Two new fields on `ArrayConfig` control how the sharding codec coalesces partial-shard reads: `sharding_coalesce_max_gap_bytes` (default 1 MiB) and `sharding_coalesce_max_bytes` (default 16 MiB). When reading multiple chunks from the same shard, nearby byte ranges are merged into a single request to the store if separated by no more than `sharding_coalesce_max_gap_bytes` and the merged read stays within `sharding_coalesce_max_bytes`. Defaults are seeded from the matching `array.sharding_coalesce_max_gap_bytes` / `array.sharding_coalesce_max_bytes` keys in [`zarr.config`][] at array-creation time, and can be overridden per array by passing `config={...}` to [`zarr.create_array`][]. diff --git a/changes/3990.misc.md b/changes/3990.misc.md deleted file mode 100644 index ff3fcf4cf2..0000000000 --- a/changes/3990.misc.md +++ /dev/null @@ -1,5 +0,0 @@ -Widen `ChunksLike` type alias to use `Iterable` instead of `Sequence`, and also -remove `None` from the type union. This supports a broader range of types, -removing the necessity to "materialize" iterable values simply to satisfy type -annotations. It also allows use of `ChunksLike` in cases where `None` should -not be permitted. diff --git a/changes/3998.misc.md b/changes/3998.misc.md deleted file mode 100644 index bacfa93a8b..0000000000 --- a/changes/3998.misc.md +++ /dev/null @@ -1 +0,0 @@ -Centralized JSON document I/O behind free functions in `zarr.core._json` and removed the unused private `Store._get_bytes`/`_get_json` methods and their per-store overrides. diff --git a/changes/4000.misc.md b/changes/4000.misc.md deleted file mode 100644 index 17d48d3016..0000000000 --- a/changes/4000.misc.md +++ /dev/null @@ -1 +0,0 @@ -Run all doctests via pytest and fix all broken doctests. diff --git a/changes/4001.misc.md b/changes/4001.misc.md deleted file mode 100644 index adbba988d9..0000000000 --- a/changes/4001.misc.md +++ /dev/null @@ -1,6 +0,0 @@ -Restore sharding write performance for shards with many inner chunks. The -`subchunk_write_order` feature inadvertently rebuilt the per-shard chunk -coordinate grid (up to tens of thousands of coordinate tuples) on every shard -write. These coordinates are now computed once per shard shape and cached, so -repeated writes to same-shaped shards reuse them, restoring write throughput to -its previous level. diff --git a/changes/4003.bugfix.md b/changes/4003.bugfix.md deleted file mode 100644 index 36327b55df..0000000000 --- a/changes/4003.bugfix.md +++ /dev/null @@ -1,19 +0,0 @@ -`FsspecStore.from_url()` and `from_mapper()` now close the async filesystem -they create when `store.close()` is called. Previously the underlying aiohttp -`ClientSession` was left open until garbage collection, producing -`"Unclosed client session"` `ResourceWarning`s from aiohttp. - -The fix introduces `FsspecStore._owns_fs`, a boolean that is ``True`` only when -`FsspecStore` itself created the filesystem (via `from_url` or `from_mapper` -when a sync→async conversion was performed). When `_owns_fs` is ``True``, -`store.close()` calls the new `_close_fs()` helper, which invokes -`fs.set_session()` and closes the returned client. Callers who supply their own -filesystem instance to `FsspecStore()` directly remain responsible for its -lifecycle; `_owns_fs` is ``False`` for those stores. - -**Scope note**: This fix closes the S3 client session that is active at the time -`store.close()` is called. Some S3-backed filesystem implementations (e.g. -s3fs with ``cache_regions=True``) may internally refresh and replace their -client during I/O operations, abandoning prior sessions before ``store.close()`` -is invoked. Those intermediate sessions are outside the scope of this fix and -are an issue in the upstream filesystem library. diff --git a/changes/4016.bugfix.md b/changes/4016.bugfix.md deleted file mode 100644 index 01984110f7..0000000000 --- a/changes/4016.bugfix.md +++ /dev/null @@ -1 +0,0 @@ -Fixed an invalid `zarr.create_array` example in the quick-start documentation (it passed an unsupported `mode` argument) and made the cloud-storage example execute against a mock S3 backend in CI. Added a test ensuring every Python code block in the documentation is either executed or explicitly opted out with a documented reason, so an invalid example can no longer go untested. diff --git a/changes/4032.bugfix.md b/changes/4032.bugfix.md deleted file mode 100644 index 4c9476f583..0000000000 --- a/changes/4032.bugfix.md +++ /dev/null @@ -1 +0,0 @@ -Fixed `ObjectStore.list_dir` for object-store listings that include a directory-marker object matching the requested non-root prefix. diff --git a/changes/4046.misc.md b/changes/4046.misc.md deleted file mode 100644 index 96c0c7d78a..0000000000 --- a/changes/4046.misc.md +++ /dev/null @@ -1 +0,0 @@ -Replace the internal `DefaultFillValue` sentinel class with a `typing_extensions.Sentinel`, and raise the minimum `typing_extensions` version to 4.14. diff --git a/changes/4052.doc.md b/changes/4052.doc.md deleted file mode 100644 index 90b4679db5..0000000000 --- a/changes/4052.doc.md +++ /dev/null @@ -1,21 +0,0 @@ -Updated the custom dtype example in `examples/custom_dtype/custom_dtype.py` to -use only the public API, eliminating all non-public imports, illustrating what -users should do. - -To better support this, the following types and functions were made available -from public modules: - -| Type/Function | Non-public module | Public module | -| ------------------------- | ------------------------ | ------------- | -| `DataTypeValidationError` | `zarr.core.dtype.common` | `zarr.errors` | -| `JSON` | `zarr.core.common` | `zarr.types` | -| `ZarrFormat` | `zarr.core.common` | `zarr.types` | -| `DTypeConfig_V2` | `zarr.core.dtype.common` | `zarr.types` | -| `DTypeJSON` | `zarr.core.dtype.common` | `zarr.types` | -| `DTypeSpec_V2` | `zarr.core.dtype.common` | `zarr.dtype` | -| `check_dtype_spec_v2` | `zarr.core.dtype.common` | `zarr.dtype` | - -`DataTypeValidationError` was *moved* to `zarr.errors`. Importing it from -`zarr.core.dtype.common` (its original location), `zarr.core.dtype`, or -`zarr.dtype` still works but now raises a `ZarrDeprecationWarning`. The remaining -types and functions are simply re-exported from the listed public module. diff --git a/changes/4053.doc.md b/changes/4053.doc.md deleted file mode 100644 index 3220b2b4ec..0000000000 --- a/changes/4053.doc.md +++ /dev/null @@ -1 +0,0 @@ -Document a self-merge policy in the contributor guide, describing when a core developer may merge their own pull request without a second reviewer and which changes warrant more caution. diff --git a/changes/4054.misc.md b/changes/4054.misc.md deleted file mode 100644 index 4aa9435b84..0000000000 --- a/changes/4054.misc.md +++ /dev/null @@ -1 +0,0 @@ -Add Hypothesis property tests for block and mask indexing (`test_block_indexing`, `test_mask_indexing`), along with a `block_indices` strategy in `zarr.testing.strategies`. These extend the existing randomized indexing coverage (basic, orthogonal, and vectorized) to the block and mask selection methods. diff --git a/changes/4059.bugfix.md b/changes/4059.bugfix.md deleted file mode 100644 index 16fb582f65..0000000000 --- a/changes/4059.bugfix.md +++ /dev/null @@ -1 +0,0 @@ -Prevents mutation of the attributes dict provided by the user by copying them instead of keeping the reference diff --git a/changes/4073.misc.md b/changes/4073.misc.md deleted file mode 100644 index bcc6b8281e..0000000000 --- a/changes/4073.misc.md +++ /dev/null @@ -1 +0,0 @@ -Extend the `test_block_indexing` Hypothesis property test to cover rectilinear chunk grids and sharded regular grids, and generalize the `block_indices` strategy in `zarr.testing.strategies` to build its array-space oracle from cumulative chunk offsets (`chunk_sizes` parameter) instead of a uniform chunk size. diff --git a/changes/4074.bugfix.md b/changes/4074.bugfix.md deleted file mode 100644 index d55a52b887..0000000000 --- a/changes/4074.bugfix.md +++ /dev/null @@ -1,7 +0,0 @@ -Fixed several storage and codec bugs: - -- Reading a value with a `SuffixByteRequest` larger than the value now correctly returns the whole value (matching HTTP `bytes=-N` suffix-range semantics), instead of silently returning incorrect data for `MemoryStore`. -- `LoggingStore.get_partial_values` and `FsspecStore.get_partial_values` no longer return empty results when `key_ranges` is passed as a one-shot iterable (e.g. a generator). -- `Store.getsize_prefix` no longer over-counts sibling keys that merely share a string prefix (e.g. `getsize_prefix("foo")` no longer includes keys under `foobar/`). -- `ZipStore.close()` no longer raises `AttributeError` when the store was created but never opened (including when used as a context manager without any I/O). -- `codecs_from_list` now raises a descriptive `TypeError` when a `BytesBytesCodec` immediately follows an `ArrayArrayCodec`, instead of a misleading "Required ArrayBytesCodec was not found" `ValueError`. diff --git a/changes/4086.misc.md b/changes/4086.misc.md deleted file mode 100644 index 2564cf9ea1..0000000000 --- a/changes/4086.misc.md +++ /dev/null @@ -1 +0,0 @@ -Fix `test_multiprocessing[fork]` failing on Python 3.15, where `os.fork()` in a multi-threaded process emits a `DeprecationWarning` that the test suite promotes to an error. diff --git a/changes/4116.bugfix.md b/changes/4116.bugfix.md deleted file mode 100644 index e99eac8c95..0000000000 --- a/changes/4116.bugfix.md +++ /dev/null @@ -1 +0,0 @@ -Fixed writing Fortran-ordered (F-contiguous) arrays through the variable-length string and bytes codecs and through numcodecs array-array filters such as `Delta`, `FixedScaleOffset` and `PackBits`. Chunks are now passed to numcodecs as C-contiguous arrays, so elements are no longer stored in transposed order. diff --git a/changes/4132.doc.md b/changes/4132.doc.md deleted file mode 100644 index 3325c82f43..0000000000 --- a/changes/4132.doc.md +++ /dev/null @@ -1,6 +0,0 @@ -Fixed many documentation errors found in a full review of the user guide, including -prose contradicted by rendered example output on the performance page, invisible -code blocks, an incorrect S3 example, stale "not yet implemented" claims in the -v3 migration guide, and undocumented optional dependency groups. Also improved -navigation order, cross-linking between pages, and coverage of group member -enumeration, bulk attribute updates, and the `use_consolidated` keyword. diff --git a/changes/4133.doc.md b/changes/4133.doc.md deleted file mode 100644 index 9690318b46..0000000000 --- a/changes/4133.doc.md +++ /dev/null @@ -1 +0,0 @@ -Fixed the documented default of ``max_age_seconds`` in the ``CacheStore`` docstring: the default is ``"infinity"`` (no expiration), not ``None``, which is rejected. Also noted that ``cache_store`` must support deletes. diff --git a/changes/4138.misc.md b/changes/4138.misc.md deleted file mode 100644 index 2cf0365bb9..0000000000 --- a/changes/4138.misc.md +++ /dev/null @@ -1 +0,0 @@ -Pass a list rather than a generator to `pytest.mark.parametrize` in `tests/test_docs.py`, so the test suite collects cleanly under pytest 9.1 (which deprecates non-Collection iterables and, under `filterwarnings = error`, turns that into a collection error). diff --git a/changes/4141.bugfix.md b/changes/4141.bugfix.md deleted file mode 100644 index 6a132da3f5..0000000000 --- a/changes/4141.bugfix.md +++ /dev/null @@ -1 +0,0 @@ -Fix silent byte-order corruption for structured dtypes with the `bytes` codec: multi-byte fields are now byte-swapped to the codec's configured `endian` on write and decoded honoring it on read, so non-native-endian structured data (e.g. big-endian fields, as produced by virtual references to external data) round-trips correctly. diff --git a/docs/release-notes.md b/docs/release-notes.md index 3ba870ac92..e85137f2b6 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -2,6 +2,162 @@ +## 3.3.0 (2026-07-15) + +### Features + +- Optimizes reading multiple chunks from a shard. Serial calls to `Store.get()` + in the sharding codec have been replaced with a single call to + `Store.get_ranges()`, which coalesces nearby byte ranges and fetches them + concurrently. ([#3004](https://github.com/zarr-developers/zarr-python/issues/3004)) +- Added a `subchunk_write_order` option to `ShardingCodec` to control the physical order of subchunks within a shard. Supported values are `morton`, `unordered`, `lexicographic`, and `colexicographic`. `unordered` makes no guarantee about subchunk layout. This setting affects only on-disk layout, not the data read back, and is not persisted in array metadata: it applies per codec instance and is not recovered when reopening a sharded array. ([#3826](https://github.com/zarr-developers/zarr-python/issues/3826)) +- Added `SyncByteGetter` and `SyncByteSetter` runtime-checkable protocols and a `get_ranges_sync` method on the `Store` ABC. These let custom byte getters/setters opt into the synchronous codec pipeline's fast path for in-memory IO, which the sharding codec uses for its inner chunks. ([#3885](https://github.com/zarr-developers/zarr-python/issues/3885)) +- Added `FusedCodecPipeline`, an opt-in codec pipeline that runs codec compute synchronously and in bulk (avoiding the per-chunk async scheduling overhead of the default `BatchedCodecPipeline`), giving large speedups for sharded arrays (up to ~24x writes / ~14x reads on many-chunks-per-shard layouts, more with compression) and no regressions on compute-bound workloads. The default `BatchedCodecPipeline` is unchanged for standard configurations, so existing code keeps working unless you opt in; enable the new pipeline with `zarr.config.set({"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"})`. ([#3885](https://github.com/zarr-developers/zarr-python/issues/3885)) +- Add `zarr.abc.store.Store.get_ranges` for concurrent, coalesced multi-range reads from a single key. The method is defined on the `Store` ABC with a default implementation built on `Store.get`, so every store inherits a working version; stores with native multi-range backends (e.g. `FsspecStore`) can override for efficiency. Coalescing knobs (`max_concurrency`, `max_gap_bytes`, `max_coalesced_bytes`) are passed as keyword arguments to `get_ranges`. Failures from underlying fetches surface as a `BaseExceptionGroup` (PEP 654); callers should use `except*` to filter for specific exception types such as `FileNotFoundError`. ([#3925](https://github.com/zarr-developers/zarr-python/issues/3925)) +- Two new fields on `ArrayConfig` control how the sharding codec coalesces partial-shard reads: `sharding_coalesce_max_gap_bytes` (default 1 MiB) and `sharding_coalesce_max_bytes` (default 16 MiB). When reading multiple chunks from the same shard, nearby byte ranges are merged into a single request to the store if separated by no more than `sharding_coalesce_max_gap_bytes` and the merged read stays within `sharding_coalesce_max_bytes`. Defaults are seeded from the matching `array.sharding_coalesce_max_gap_bytes` / `array.sharding_coalesce_max_bytes` keys in [`zarr.config`][] at array-creation time, and can be overridden per array by passing `config={...}` to [`zarr.create_array`][]. ([#3987](https://github.com/zarr-developers/zarr-python/issues/3987)) + +### Bugfixes + +- Stop emitting an `UnstableSpecificationWarning` when serializing the `struct` data type to Zarr V3 metadata. The `struct` data type now has a stable Zarr V3 specification. The legacy `structured` alias and the unspecified `null_terminated_bytes`, `raw_bytes`, and `variable_length_bytes` data types continue to warn. ([#202](https://github.com/zarr-developers/zarr-python/issues/202)) +- Fix equality comparison of `ArrayV2Metadata` and `ArrayV3Metadata` objects with a + `NaN` fill value. Such objects are now compared by their JSON-serialized form, so two + otherwise-identical metadata objects with a `NaN` (or infinite) fill value compare equal. ([#2929](https://github.com/zarr-developers/zarr-python/issues/2929)) +- Fixed `BytesCodec.from_dict` so that `BytesCodec` instances roundtrip to / from + their dict representation. `BytesCodec.from_dict` now interprets a missing + `endian` configuration as `endian=None` (matching what `BytesCodec.to_dict` + emits), instead of falling back to the system's native byte order. ([#3417](https://github.com/zarr-developers/zarr-python/issues/3417)) +- Fixed `save_array`, `Group.__setitem__`, and `load` for 0-dimensional arrays. ([#3469](https://github.com/zarr-developers/zarr-python/issues/3469)) +- Fixed inner-codec spec evolution for sharded arrays. The sharding codec now threads the array spec through its inner codec chain when evolving codecs, so a codec that changes the dtype upstream of `BytesCodec` no longer leaves the inner chain evolved against the wrong spec (which previously failed at decode time). This runs on the default `BatchedCodecPipeline` as well. Standard inner chains (`[BytesCodec]`, `[BytesCodec, ZstdCodec]`, transpose + bytes) are byte-identical to before. Restores the behavior of #2179. ([#3885](https://github.com/zarr-developers/zarr-python/issues/3885)) +- Make chunk normalization properly handle `-1` as a compact representation of the + length of an entire axis. Reject several previously-accepted but ill-defined + chunk specifications: `chunks=True` (previously silently produced size-1 chunks), + chunk tuples shorter than the array's number of dimensions (previously padded to + the array's shape), and `None` as a per-dimension chunk size. These all now + raise informative errors. Also fix chunk handling for 0-length array dimensions, + and add explicit rejection of 0-length chunks. ([#3899](https://github.com/zarr-developers/zarr-python/issues/3899)) +- Handle missing consolidated metadata in leaf Group nodes. ([#3954](https://github.com/zarr-developers/zarr-python/issues/3954)) +- Corrected the JSON type definitions for the `numpy.datetime64` and + `numpy.timedelta64` data types in Zarr V3 metadata: the `configuration` object + (holding `unit` and `scale_factor`) is now required, matching the published + specifications for these data types. Also updated the specification links in + the docstrings to point to the zarr-extensions repository. ([#3955](https://github.com/zarr-developers/zarr-python/issues/3955)) +- Fixed writing to 0-dimensional arrays that use the sharding codec. Previously + assigning to a 0-dimensional sharded array raised an error. ([#3966](https://github.com/zarr-developers/zarr-python/issues/3966)) +- Fix flaky stateful test bookkeeping when `delete_dir` matches string prefixes instead of true directory descendants. Previously a path such as `6/faNT…` could be incorrectly removed when deleting `6/f`. (See [issue #3977](https://github.com/zarr-developers/zarr-python/issues/3977).) ([#3977](https://github.com/zarr-developers/zarr-python/issues/3977)) +- `FsspecStore.from_url()` and `from_mapper()` now close the async filesystem + they create when `store.close()` is called. Previously the underlying aiohttp + `ClientSession` was left open until garbage collection, producing + `"Unclosed client session"` `ResourceWarning`s from aiohttp. + + The fix introduces `FsspecStore._owns_fs`, a boolean that is ``True`` only when + `FsspecStore` itself created the filesystem (via `from_url` or `from_mapper` + when a sync→async conversion was performed). When `_owns_fs` is ``True``, + `store.close()` calls the new `_close_fs()` helper, which invokes + `fs.set_session()` and closes the returned client. Callers who supply their own + filesystem instance to `FsspecStore()` directly remain responsible for its + lifecycle; `_owns_fs` is ``False`` for those stores. + + **Scope note**: This fix closes the S3 client session that is active at the time + `store.close()` is called. Some S3-backed filesystem implementations (e.g. + s3fs with ``cache_regions=True``) may internally refresh and replace their + client during I/O operations, abandoning prior sessions before ``store.close()`` + is invoked. Those intermediate sessions are outside the scope of this fix and + are an issue in the upstream filesystem library. ([#4003](https://github.com/zarr-developers/zarr-python/issues/4003)) +- Fixed an invalid `zarr.create_array` example in the quick-start documentation (it passed an unsupported `mode` argument) and made the cloud-storage example execute against a mock S3 backend in CI. Added a test ensuring every Python code block in the documentation is either executed or explicitly opted out with a documented reason, so an invalid example can no longer go untested. ([#4016](https://github.com/zarr-developers/zarr-python/issues/4016)) +- Fixed `ObjectStore.list_dir` for object-store listings that include a directory-marker object matching the requested non-root prefix. ([#4032](https://github.com/zarr-developers/zarr-python/issues/4032)) +- Prevents mutation of the attributes dict provided by the user by copying them instead of keeping the reference ([#4059](https://github.com/zarr-developers/zarr-python/issues/4059)) +- Fixed several storage and codec bugs: + + - Reading a value with a `SuffixByteRequest` larger than the value now correctly returns the whole value (matching HTTP `bytes=-N` suffix-range semantics), instead of silently returning incorrect data for `MemoryStore`. + - `LoggingStore.get_partial_values` and `FsspecStore.get_partial_values` no longer return empty results when `key_ranges` is passed as a one-shot iterable (e.g. a generator). + - `Store.getsize_prefix` no longer over-counts sibling keys that merely share a string prefix (e.g. `getsize_prefix("foo")` no longer includes keys under `foobar/`). + - `ZipStore.close()` no longer raises `AttributeError` when the store was created but never opened (including when used as a context manager without any I/O). + - `codecs_from_list` now raises a descriptive `TypeError` when a `BytesBytesCodec` immediately follows an `ArrayArrayCodec`, instead of a misleading "Required ArrayBytesCodec was not found" `ValueError`. + + ([#4074](https://github.com/zarr-developers/zarr-python/issues/4074)) +- Fixed writing Fortran-ordered (F-contiguous) arrays through the variable-length string and bytes codecs and through numcodecs array-array filters such as `Delta`, `FixedScaleOffset` and `PackBits`. Chunks are now passed to numcodecs as C-contiguous arrays, so elements are no longer stored in transposed order. ([#4116](https://github.com/zarr-developers/zarr-python/issues/4116)) +- Fix silent byte-order corruption for structured dtypes with the `bytes` codec: multi-byte fields are now byte-swapped to the codec's configured `endian` on write and decoded honoring it on read, so non-native-endian structured data (e.g. big-endian fields, as produced by virtual references to external data) round-trips correctly. ([#4141](https://github.com/zarr-developers/zarr-python/issues/4141)) + +### Improved Documentation + +- Document the changes to `zarr.errors` in the 3.0 migration guide, including the removal of v2 exception classes and the introduction of `NodeNotFoundError`. ([#3009](https://github.com/zarr-developers/zarr-python/issues/3009)) +- Clarify the difference between `zarr.load` and `zarr.open` in their docstrings. + `load` eagerly reads data into an in-memory array, while `open` returns a + lazy `Array` or `Group` backed by the store, with `See Also` cross-references + linking the two. ([#3984](https://github.com/zarr-developers/zarr-python/issues/3984)) +- Updated the custom dtype example in `examples/custom_dtype/custom_dtype.py` to + use only the public API, eliminating all non-public imports, illustrating what + users should do. + + To better support this, the following types and functions were made available + from public modules: + + | Type/Function | Non-public module | Public module | + | ------------------------- | ------------------------ | ------------- | + | `DataTypeValidationError` | `zarr.core.dtype.common` | `zarr.errors` | + | `JSON` | `zarr.core.common` | `zarr.types` | + | `ZarrFormat` | `zarr.core.common` | `zarr.types` | + | `DTypeConfig_V2` | `zarr.core.dtype.common` | `zarr.types` | + | `DTypeJSON` | `zarr.core.dtype.common` | `zarr.types` | + | `DTypeSpec_V2` | `zarr.core.dtype.common` | `zarr.dtype` | + | `check_dtype_spec_v2` | `zarr.core.dtype.common` | `zarr.dtype` | + + `DataTypeValidationError` was *moved* to `zarr.errors`. Importing it from + `zarr.core.dtype.common` (its original location), `zarr.core.dtype`, or + `zarr.dtype` still works but now raises a `ZarrDeprecationWarning`. The remaining + types and functions are simply re-exported from the listed public module. ([#4052](https://github.com/zarr-developers/zarr-python/issues/4052)) +- Document a self-merge policy in the contributor guide, describing when a core developer may merge their own pull request without a second reviewer and which changes warrant more caution. ([#4053](https://github.com/zarr-developers/zarr-python/issues/4053)) +- Fixed many documentation errors found in a full review of the user guide, including + prose contradicted by rendered example output on the performance page, invisible + code blocks, an incorrect S3 example, stale "not yet implemented" claims in the + v3 migration guide, and undocumented optional dependency groups. Also improved + navigation order, cross-linking between pages, and coverage of group member + enumeration, bulk attribute updates, and the `use_consolidated` keyword. ([#4132](https://github.com/zarr-developers/zarr-python/issues/4132)) +- Fixed the documented default of ``max_age_seconds`` in the ``CacheStore`` docstring: the default is ``"infinity"`` (no expiration), not ``None``, which is rejected. Also noted that ``cache_store`` must support deletes. ([#4133](https://github.com/zarr-developers/zarr-python/issues/4133)) + +### Deprecations and Removals + +- The ``BloscShuffle`` and ``BloscCname`` enums (``zarr.codecs.BloscShuffle``, + ``zarr.codecs.BloscCname``) are now deprecated. Pass the equivalent literal + string (e.g. ``"zstd"``, ``"bitshuffle"``) when constructing a ``BloscCodec``. + The enum classes remain importable but emit ``DeprecationWarning`` on member + access, and will be removed in a future release. ``BloscCodec.cname`` and + ``BloscCodec.shuffle`` are now plain strings rather than enum members. + + Additional renames in ``zarr.codecs.blosc`` from the same change: the type + aliases ``Shuffle`` and ``CName`` are now ``BloscShuffleLiteral`` and + ``BloscCnameLiteral``, the constant ``SHUFFLE`` is now ``BLOSC_SHUFFLE`` + (with a new ``BLOSC_CNAME`` alongside it), and ``BloscShuffle.from_int`` + now returns a literal string rather than an enum member. ([#3963](https://github.com/zarr-developers/zarr-python/issues/3963)) +- The ``Endian`` (``zarr.codecs.bytes.Endian``) and ``ShardingCodecIndexLocation`` + (``zarr.codecs.ShardingCodecIndexLocation``) enums are now deprecated. Pass the + equivalent literal string instead (e.g. ``"little"`` / ``"big"``, ``"start"`` / + ``"end"``). The enum classes remain importable but emit ``DeprecationWarning`` + on member access, and will be removed in a future release. ``BytesCodec.endian`` + and ``ShardingCodec.index_location`` are now plain strings rather than enum + members. + + Two follow-on changes from this deprecation: + + - ``NDBuffer.byteorder`` now returns a literal string (``"little"`` or + ``"big"``) rather than an ``Endian`` member. Subclasses overriding this + property should update their return type. + - The module-level binding ``zarr.codecs.bytes.default_system_endian`` was + removed. ``BytesCodec()`` continues to default to ``sys.byteorder``; + external callers that imported ``default_system_endian`` should use + ``sys.byteorder`` directly. + + Additionally, the module-level function ``zarr.codecs.sharding.parse_index_location`` + was made private as part of this change. + + ([#3968](https://github.com/zarr-developers/zarr-python/issues/3968)) +- Removed the NumPy 1.x implementation of the `VariableLengthUTF8` data type because NumPy 1.x is no longer supported under [SPEC0](https://scientific-python.org/specs/spec-0000/). ([#3973](https://github.com/zarr-developers/zarr-python/issues/3973)) + +### Misc + +- [#214](https://github.com/zarr-developers/zarr-python/issues/214), [#215](https://github.com/zarr-developers/zarr-python/issues/215), [#3908](https://github.com/zarr-developers/zarr-python/issues/3908), [#3972](https://github.com/zarr-developers/zarr-python/issues/3972), [#3975](https://github.com/zarr-developers/zarr-python/issues/3975), [#3979](https://github.com/zarr-developers/zarr-python/issues/3979), [#3990](https://github.com/zarr-developers/zarr-python/issues/3990), [#3998](https://github.com/zarr-developers/zarr-python/issues/3998), [#4000](https://github.com/zarr-developers/zarr-python/issues/4000), [#4001](https://github.com/zarr-developers/zarr-python/issues/4001), [#4046](https://github.com/zarr-developers/zarr-python/issues/4046), [#4054](https://github.com/zarr-developers/zarr-python/issues/4054), [#4073](https://github.com/zarr-developers/zarr-python/issues/4073), [#4086](https://github.com/zarr-developers/zarr-python/issues/4086), [#4138](https://github.com/zarr-developers/zarr-python/issues/4138) + + ## 3.2.1 (2026-05-05) ### Bugfixes From 23d96d38ae1cb1b722681f16eec7af171890b367 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:30:12 -0400 Subject: [PATCH 41/70] chore: CI worfklows for linting docs (#4076) * chore: CI worfklows for missing docs and RST docstrings * Add linter for docs * Add link checking workflow * fix links * fix: apply suggestions from code review * refactor: readability improvements * chore: switch lychee action to opening issue on failure --------- Co-authored-by: Davis Bennett --- .github/workflows/docs.yml | 5 + .github/workflows/links.yml | 32 +++ .markdownlint-cli2.jsonc | 54 +++++ .pre-commit-config.yaml | 10 + ci/check_documented_exports.py | 161 +++++++++++++ ci/lint_docs.py | 284 +++++++++++++++++++++++ docs/contributing.md | 23 +- docs/index.md | 14 +- docs/quick-start.md | 2 - docs/release-notes.md | 2 + docs/user-guide/attributes.md | 1 + docs/user-guide/consolidated_metadata.md | 1 - docs/user-guide/data_types.md | 7 + docs/user-guide/experimental.md | 1 - docs/user-guide/extending.md | 1 + docs/user-guide/groups.md | 1 - docs/user-guide/performance.md | 6 +- docs/user-guide/storage.md | 8 +- lychee.toml | 20 ++ src/zarr/core/metadata/v2.py | 2 +- src/zarr/testing/store.py | 6 +- 21 files changed, 608 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/links.yml create mode 100644 .markdownlint-cli2.jsonc create mode 100644 ci/check_documented_exports.py create mode 100644 ci/lint_docs.py create mode 100644 lychee.toml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 59eca8d9b9..c72b493b12 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -24,6 +24,11 @@ jobs: persist-credentials: false - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - run: uv sync --group docs + # Fast source-level guards that need no built site, so they run before the (slower) + # build for a quick failure: every public export is in the API reference, and no + # docstring/Markdown carries reStructuredText markup that MkDocs won't render. + - run: uv run python ci/check_documented_exports.py docs/api + - run: uv run python ci/lint_docs.py # --strict turns warnings into errors, so a docs code block that fails to execute # at build time (e.g. a non-exec python fence disrupting a later exec="true" block) # fails CI instead of merging as a silent warning. diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml new file mode 100644 index 0000000000..a53de8da83 --- /dev/null +++ b/.github/workflows/links.yml @@ -0,0 +1,32 @@ +name: Check links + +on: + repository_dispatch: + workflow_dispatch: + # pull_request: + schedule: + - cron: "00 18 * * *" + +jobs: + linkChecker: + runs-on: ubuntu-latest + permissions: + issues: write # required for peter-evans/create-issue-from-file + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Link Checker + id: lychee + uses: lycheeverse/lychee-action@8646ba30535128ac92d33dfc9133794bfdd9b411 # v2.8.0 + with: + fail: false + + - name: Create Issue From File + if: steps.lychee.outputs.exit_code != 0 + uses: peter-evans/create-issue-from-file@fca9117c27cdc29c6c4db3b86c48e4115a786710 # v6.0.0 + with: + title: Link Checker Report + content-filepath: ./lychee/out.md + labels: report, automated issue diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc new file mode 100644 index 0000000000..3dfdf96856 --- /dev/null +++ b/.markdownlint-cli2.jsonc @@ -0,0 +1,54 @@ +// markdownlint-cli2 configuration for zarr-python docs. +// +// We keep the rules that catch real rendering/structure problems and disable those that +// are pure style, conflict with house conventions, or fire false positives against our +// MkDocs/mkdocstrings + pymdownx toolchain. Complementary, not overlapping, with +// ci/lint_docs.py (RST residue + list-breaking fences) and `mkdocs build --strict`. +{ + "config": { + "default": true, + + // House style: Markdown paragraphs are single unwrapped lines, so line length is not + // a meaningful constraint. + "MD013": false, + + // Purely stylistic marker/emphasis choices -- not worth the churn across existing docs. + "MD004": false, // ul bullet style (-, *, +) + "MD007": false, // ul indentation width + "MD050": false, // strong (bold) style + "MD035": false, // hr style + + // False positives from our toolchain: + // mkdocstrings cross-refs `[`X`][zarr.X]` read as undefined reference links (MD052); + // pymdownx.magiclink auto-links bare URLs (MD034); + // md_in_html lets us embed intentional raw HTML (MD033); + // generated/included files (api stubs, snippets) need not open with an H1 (MD041). + "MD052": false, + "MD034": false, + "MD033": false, + "MD041": false, + + // Duplicate headings are legitimate under different sections (e.g. repeated + // "Documentation"); only flag true sibling duplicates. + "MD024": { "siblings_only": true }, + + // Opinionated table/link/command rules with low value for these docs. + "MD055": false, // table pipe style + "MD060": false, // table column style + "MD059": false, // "descriptive" link text (no "click here") + "MD014": false, // $ before commands without shown output + + // markdownlint does not understand MkDocs `!!!` admonitions, so it reads their + // 4-space-indented bodies as indented code blocks and flags them (and, via inferred + // file style, flags real fenced blocks too). Cannot coexist with our admonitions. + "MD046": false // code block style (fenced vs indented) + // Kept on (structural / real rendering bugs): MD012 (multiple blanks), MD022/MD031/MD032 + // (blanks around headings/fences/lists), MD025 (single H1), MD029 (ordered-list prefix), + // MD040 (fenced code language), MD042 (empty links), + // MD047 (trailing newline), MD056 (table column count), among others. + }, + "globs": ["docs/**/*.md"], + "ignores": [ + "docs/api/**" // mkdocstrings stubs (`::: zarr.X`) + ] +} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e08c98abca..57a1d0d4f7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,6 +31,16 @@ repos: - id: check-yaml exclude: mkdocs.yml - id: trailing-whitespace + - repo: https://github.com/DavidAnson/markdownlint-cli2 + rev: v0.22.1 + hooks: + # Markdown structure/hygiene. Rule selection and ignores are in + # .markdownlint-cli2.jsonc; complements ci/lint_docs.py (RST residue, + # list-breaking fences) and `mkdocs build --strict`. Scoped to docs/ to + # match the config's globs (pre-commit passes filenames, which would + # otherwise override that scoping and lint all repo Markdown). + - id: markdownlint-cli2 + files: ^docs/ - repo: local hooks: - id: mypy diff --git a/ci/check_documented_exports.py b/ci/check_documented_exports.py new file mode 100644 index 0000000000..0772954399 --- /dev/null +++ b/ci/check_documented_exports.py @@ -0,0 +1,161 @@ +"""Check that every public top-level export is in the API reference. + +The API reference is authored as explicit mkdocstrings directives (``::: target``) +under ``docs/api/`` -- one per documented symbol -- rather than autodoc, so a newly +added ``zarr.__all__`` entry will not appear in the docs until someone writes a page +for it (or it becomes a rendered member of an already-documented module). This script +catches that gap: it resolves every ``:::`` target, expands module directives into the +members they render (honoring ``members: false``), and asserts each name in +``zarr.__all__`` resolves to a documented object. + +Usage: + python ci/check_documented_exports.py [API_DOCS_DIR] + +API_DOCS_DIR defaults to the repo-root ``docs/api``. Exits non-zero (and prints the +undocumented exports to stderr) if any public export is missing from the reference. +""" + +from __future__ import annotations + +import importlib +import re +import sys +from pathlib import Path +from types import ModuleType +from typing import TYPE_CHECKING, Any + +import zarr + +if TYPE_CHECKING: + from collections.abc import Iterator + +REPO_ROOT = Path(__file__).parent.parent.resolve() +DEFAULT_API_DOCS_ROOT = REPO_ROOT / "docs" / "api" + +# Names in zarr.__all__ that are intentionally absent from the API reference. +# Keep this list short and justified -- it is the only escape hatch from the guard. +EXEMPT_EXPORTS = { + "__version__", # version string, not an API symbol + "print_debug_info", # debugging helper, deliberately not in the reference +} + +# A mkdocstrings autodoc directive: `::: some.dotted.target` at the start of a line. +DIRECTIVE_RE = re.compile(r"^:::[ \t]+(?P\S+)") +# `members: false` (or `members: []`) within a directive's option block disables +# rendering of a module's members. +MEMBERS_DISABLED_RE = re.compile(r"^\s+members:\s*(false|\[\s*\])\s*$") + + +def resolve(target: str) -> Any: + """Resolve a `:::` target (a dotted path) to the Python object it documents.""" + try: + return importlib.import_module(target) + except ImportError: + pass + module_path, _, attr = target.rpartition(".") + try: + return getattr(importlib.import_module(module_path), attr) + except (ImportError, AttributeError): + return None + + +def iter_directives(text: str) -> Iterator[tuple[str, bool]]: + """Yield ``(target, members_enabled)`` for each ``:::`` directive in ``text``. + + The file is split into lines once; for each directive we scan its indented option + block -- stopping at the first non-indented line, which ends the block -- so options + belonging to a later directive are never consulted. ``members_enabled`` is False when + that block sets ``members: false`` (or ``members: []``).""" + lines = text.splitlines() + i = 0 + while i < len(lines): + match = DIRECTIVE_RE.match(lines[i]) + if match is None: + i += 1 + continue + members_enabled = True + i += 1 + while i < len(lines): + line = lines[i] + if line.strip() == "": + i += 1 + continue + if not line.startswith((" ", "\t")): + break # non-indented line: end of this directive's option block + if MEMBERS_DISABLED_RE.match(line): + members_enabled = False + i += 1 + yield match.group("target"), members_enabled + + +def module_member_ids(module: ModuleType) -> Iterator[int]: + """Yield the id() of each public member a module directive renders. + + The rendered members are the module's ``__all__`` if defined, else its public + (non-underscore) attributes.""" + member_names = getattr(module, "__all__", None) or [ + name for name in dir(module) if not name.startswith("_") + ] + for name in member_names: + member = getattr(module, name, None) + if member is not None: + yield id(member) + + +def documented_object_ids(api_docs_root: Path) -> set[int]: + """Collect the id()s of every object rendered by a `:::` directive under api_docs_root. + + A directive pointing at an object documents that object. A directive pointing at a + module documents the module's public members unless the directive sets + ``members: false``.""" + documented: set[int] = set() + for md_file in sorted(api_docs_root.rglob("*.md")): + for target, members_enabled in iter_directives(md_file.read_text(encoding="utf-8")): + obj = resolve(target) + if obj is None: + continue + documented.add(id(obj)) + if isinstance(obj, ModuleType) and members_enabled: + documented.update(module_member_ids(obj)) + return documented + + +def find_undocumented_exports(api_docs_root: Path) -> list[str]: + documented = documented_object_ids(api_docs_root) + return sorted( + name + for name in zarr.__all__ + if name not in EXEMPT_EXPORTS and id(getattr(zarr, name)) not in documented + ) + + +def main() -> int: + args = sys.argv[1:] + api_docs_root = Path(args[0]).resolve() if args else DEFAULT_API_DOCS_ROOT + if not api_docs_root.exists(): + print(f"{api_docs_root} does not exist.", file=sys.stderr) + return 1 + + missing = find_undocumented_exports(api_docs_root) + if not missing: + print(f"All {len(zarr.__all__)} public exports are documented.") + return 0 + + print( + f"Found {len(missing)} public export(s) in zarr.__all__ missing from the API " + "reference (docs/api/):\n", + file=sys.stderr, + ) + for name in missing: + print(f" - zarr.{name}", file=sys.stderr) + print( + "\nAdd a `::: zarr.` page under docs/api/zarr/ (and register it in " + "mkdocs.yml and docs/api/zarr/index.md), or -- if the export is intentionally " + "undocumented -- add it to EXEMPT_EXPORTS in this script with a reason.", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/lint_docs.py b/ci/lint_docs.py new file mode 100644 index 0000000000..ff7b56649b --- /dev/null +++ b/ci/lint_docs.py @@ -0,0 +1,284 @@ +"""Lint docstrings and Markdown for reStructuredText markup that won't render. + +This project renders API docs with mkdocstrings (``docstring_style: numpy``) and prose +with MkDocs + Markdown -- not Sphinx/reStructuredText. RST constructs that survive from +older docstrings (or muscle memory) are not interpreted: a Sphinx role passes through as +literal text instead of becoming a link, an ``.. note::`` directive renders as a stray +line, and a ``:param:`` field list never becomes a documented parameter. + +Crucially, none of this is caught by the rest of the docs CI. ``mkdocs build --strict`` +sees the residue as ordinary prose (no warning), and ``ci/check_unlinked_types.py`` only +finds cross-references mkdocstrings *attempted* to resolve -- a raw ``:class:`` role is +never attempted, so it leaves no unlinked-type span. This linter fills that gap with a +fast, source-level check that needs no docs build. + +Checks fall into two groups -- RST markup that silently fails under MkDocs/mkdocstrings, +and a Markdown structural problem that renders as valid-but-wrong HTML (so `mkdocs build` +emits no warning): + + sphinx-role :class:`X`, :func:`X`, :py:meth:`X` -> [`X`][zarr.X] + rst-directive .. note:: / .. code-block:: python -> MkDocs admonition / fenced code + rst-field :param x:, :returns:, :rtype: -> numpydoc Parameters/Returns/Raises + rst-link `text `_ -> [text](https://example) + list-break unindented code fence between list items -> indent the fence under its item + +The ``list-break`` check catches a fenced code block at column 0 placed *between* two list +items: because the fence is not indented into the preceding item, Markdown ends the list at +the fence and the following item starts a fresh list -- renumbering an ordered list (1, 1, 2 +instead of 1, 2, 3) or breaking the grouping/spacing of any list. markdownlint's MD029 only +notices this for sequentially-numbered ordered lists; lazily-numbered (1., 1.) and unordered +lists slip past it, so this structural check covers the gap. + +Usage: + python ci/lint_docs.py [PATH ...] + +PATH defaults to the repo-root ``src/zarr`` and ``docs``. Each PATH may be a file or a +directory (directories are searched for ``*.py`` and ``*.md``). Exits non-zero if any +issues are found. +""" + +from __future__ import annotations + +import ast +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import NamedTuple + +REPO_ROOT = Path(__file__).parent.parent.resolve() +DEFAULT_PATHS = (REPO_ROOT / "src" / "zarr", REPO_ROOT / "docs") + +# A Sphinx interpreted-text role: an optional domain, a role name, then a backtick +# target -- e.g. :class:`Foo` or :py:meth:`Foo.bar`. Requires the trailing backtick so +# plain "::" (RST literal markers, time strings, mkdocs-material :icon: shortcodes) and +# URLs ("https://") never match. +SPHINX_ROLE = re.compile(r":[a-zA-Z_]\w*(?::[a-zA-Z_]\w*)?:`[^`\n]+`") + +# An RST directive line: ".. name::" (with or without an argument after it). RST hyperlink +# targets (".. _label:") and comments (".. text") lack the "::" and are not flagged. +RST_DIRECTIVE = re.compile(r"^\s*\.\.[ \t]+[\w-]+::") + +# An RST field-list entry used for docstring fields. The role names above (class, func, +# ...) are deliberately excluded so a role is reported as a role, not a field. +RST_FIELD = re.compile( + r"^\s*:(param|parameter|arg|argument|key|keyword|kwarg|type|returns?|rtype" + r"|raises?|except|exception|yields?|ytype|var|cvar|ivar)\b[^:]*:" +) + +# An RST external hyperlink: `text `_ +RST_LINK = re.compile(r"`[^`\n]+\n]+>`_") + +# A list item at column 0: an ordered marker (1. / 1)) or a bullet (-, *, +) followed by +# whitespace and content. Leading-whitespace (nested/continuation) lines are intentionally +# not matched -- the list-break check only fires on top-level items. +LIST_ITEM = re.compile(r"^(?:\d+[.)]|[-*+])\s+\S") + + +class Check(NamedTuple): + """One docs-residue check: its category, the line pattern that flags it (None for a + structural check matched outside ``_scan_line``), and the user-facing remediation + shown by ``main()``. Keeping ``example``/``fix`` here makes this the single source for + the help text, so adding a check can't leave the help out of date.""" + + category: str + pattern: re.Pattern[str] | None + example: str + fix: str + + +# ``list-break`` carries no pattern -- it is detected structurally in find_list_breaking_fences, +# not by scanning a single line -- but it appears here so it shares the remediation help. +CHECKS = ( + Check("sphinx-role", SPHINX_ROLE, ":class:`X`", "[`X`][zarr.X]"), + Check("rst-directive", RST_DIRECTIVE, ".. note::", "MkDocs admonition (!!! note)"), + Check("rst-field", RST_FIELD, ":param x:", "numpydoc Parameters/Returns/Raises section"), + Check("rst-link", RST_LINK, "`text `_", "[text](url)"), + Check("list-break", None, "fence between items", "indent the fence 4 spaces to nest it"), +) + + +@dataclass(frozen=True) +class Finding: + path: Path + line: int + category: str + snippet: str + + def format(self) -> str: + try: + location: Path | str = self.path.relative_to(REPO_ROOT) + except ValueError: + location = self.path + return f" {location}:{self.line}: [{self.category}] {self.snippet.strip()}" + + +def _scan_line(text: str) -> list[str]: + """Return every RST-residue category found in a single line (a line can carry more + than one, e.g. a role and an external link).""" + return [c.category for c in CHECKS if c.pattern is not None and c.pattern.search(text)] + + +def lint_python(path: Path) -> list[Finding]: + """Scan the docstrings (module, classes, functions) of a Python file. + + Only docstrings are checked -- they are what mkdocstrings renders -- so RST-looking + text inside ordinary code or string literals is never misreported.""" + source = path.read_text(encoding="utf-8") + try: + tree = ast.parse(source) + except SyntaxError as exc: # pragma: no cover - surfaced, not silently skipped + return [Finding(path, exc.lineno or 0, "syntax-error", str(exc.msg))] + + doc_nodes = (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef) + # node.body[0].value is the docstring literal; its lineno is the line the string opens + # on, so content line i maps to source line (start + i). + docstrings = [ + (docstring, node.body[0].value.lineno) # type: ignore[attr-defined] + for node in ast.walk(tree) + if isinstance(node, doc_nodes) + if (docstring := ast.get_docstring(node, clean=False)) + ] + return [ + Finding(path, start + offset, category, line) + for docstring, start in docstrings + for offset, line in enumerate(docstring.splitlines()) + for category in _scan_line(line) + ] + + +class Fence(NamedTuple): + """A fenced code block, by 0-based line index. ``terminated`` is False when the fence + has no closing delimiter before EOF, in which case ``close`` is the last line.""" + + open: int + close: int + terminated: bool + + +def fenced_blocks(lines: list[str]) -> list[Fence]: + """Index every fenced code block in ``lines``. + + An unterminated fence is malformed Markdown that `mkdocs build` surfaces anyway; it is + still returned (with ``terminated=False``, ``close`` at the last line) so callers that + skip code can skip to EOF.""" + blocks: list[Fence] = [] + fence: str | None = None + open_idx = -1 + for i, line in enumerate(lines): + stripped = line.lstrip() + if fence is None: + if stripped.startswith(("```", "~~~")): + fence, open_idx = stripped[:3], i + elif stripped.startswith(fence): + blocks.append(Fence(open_idx, i, terminated=True)) + fence = None + if fence is not None: + blocks.append(Fence(open_idx, len(lines) - 1, terminated=False)) + return blocks + + +def find_list_breaking_fences(lines: list[str], blocks: list[Fence]) -> list[tuple[int, str]]: + """Return ``(lineno, snippet)`` for each fenced code block at column 0 that splits a + list -- i.e. one whose nearest non-blank neighbours on both sides are top-level list + items. Such a fence is not indented into the preceding item, so Markdown closes the + list at the fence and the following item starts a new one. The fix is to indent the + fence (4 spaces) so it nests inside its list item. See the module docstring. + + Conservative on purpose: it requires a list item *directly* before and after (a + continuation line or paragraph in between is not matched), keeping false positives low + for a check that fails CI. Unterminated fences are ignored.""" + + def neighbour(start: int, step: int) -> str | None: + j = start + step + while 0 <= j < len(lines): + if lines[j].strip(): + return lines[j] + j += step + return None + + def splits_a_list(open_i: int, close_i: int) -> bool: + if lines[open_i][:1].isspace(): + return False # indented fence: already nested in the list item, not a break + before = neighbour(open_i, -1) + after = neighbour(close_i, +1) + return bool(before and after and LIST_ITEM.match(before) and LIST_ITEM.match(after)) + + return [ + (fence.open + 1, lines[fence.open]) + for fence in blocks + if fence.terminated and splits_a_list(fence.open, fence.close) + ] + + +def lint_markdown(path: Path) -> list[Finding]: + """Scan a Markdown file: RST residue in prose (skipping fenced code blocks), plus + fenced code blocks that break a list (see find_list_breaking_fences).""" + lines = path.read_text(encoding="utf-8").splitlines() + blocks = fenced_blocks(lines) + in_code = {i for fence in blocks for i in range(fence.open, fence.close + 1)} + + prose = [ + Finding(path, lineno, category, line) + for lineno, line in enumerate(lines, start=1) + if lineno - 1 not in in_code + for category in _scan_line(line) + ] + breaks = [ + Finding(path, lineno, "list-break", snippet) + for lineno, snippet in find_list_breaking_fences(lines, blocks) + ] + return prose + breaks + + +def iter_files(paths: tuple[Path, ...]) -> list[Path]: + files: list[Path] = [] + for path in paths: + if path.is_file(): + files.append(path) + elif path.is_dir(): + files.extend(sorted(path.rglob("*.py"))) + files.extend(sorted(path.rglob("*.md"))) + else: + raise FileNotFoundError(f"{path} does not exist") + return files + + +LINTERS = {".py": lint_python, ".md": lint_markdown} + + +def lint(paths: tuple[Path, ...]) -> list[Finding]: + return [ + finding + for file in iter_files(paths) + if file.suffix in LINTERS + for finding in LINTERS[file.suffix](file) + ] + + +def main() -> int: + args = sys.argv[1:] + paths = tuple(Path(a).resolve() for a in args) if args else DEFAULT_PATHS + findings = lint(paths) + + if not findings: + print("No reStructuredText residue or list-breaking fences found in docs.") + return 0 + + print( + f"Found {len(findings)} docs issue(s) -- RST markup that will not render under " + "MkDocs/mkdocstrings, or Markdown that renders as valid-but-wrong HTML:\n", + file=sys.stderr, + ) + for finding in findings: + print(finding.format(), file=sys.stderr) + remediation = "\n".join(f" {c.category:<13} {c.example:<19} -> {c.fix}" for c in CHECKS) + print( + f"\nFix each issue (see ci/lint_docs.py header):\n{remediation}", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/contributing.md b/docs/contributing.md index eaab26fbc7..df46f381de 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -12,23 +12,23 @@ If you find a bug, please raise a [GitHub issue](https://github.com/zarr-develop 1. A minimal, self-contained snippet of Python code reproducing the problem. You can format the code nicely using markdown, e.g.: -```python exec="false" reason="illustrative pseudocode with a '# etc.' placeholder, not runnable" -import zarr -g = zarr.group() -# etc. -``` + ```python exec="false" reason="illustrative pseudocode with a '# etc.' placeholder, not runnable" + import zarr + g = zarr.group() + # etc. + ``` 2. An explanation of why the current behavior is wrong/not desired, and what you expect instead. 3. Information about the version of Zarr, along with versions of dependencies and the Python interpreter, and installation information. The version of Zarr can be obtained from the `zarr.__version__` attribute. Please also state how Zarr was installed, e.g., "installed via pip into a virtual environment", or "installed using conda". Information about other packages installed can be obtained by executing `pip freeze` (if using pip to install packages) or `conda env export` (if using conda to install packages) from the operating system command prompt. The version of the Python interpreter can be obtained by running a Python interactive session, e.g.: -```console -python -``` + ```console + python + ``` -```ansi -Python 3.12.7 | packaged by conda-forge | (main, Oct 4 2024, 15:57:01) [Clang 17.0.6 ] on darwin -``` + ```ansi + Python 3.12.7 | packaged by conda-forge | (main, Oct 4 2024, 15:57:01) [Clang 17.0.6 ] on darwin + ``` ## Enhancement proposals @@ -401,7 +401,6 @@ The Zarr library is an implementation of a file format standard defined external If an existing Zarr format version changes, or a new version of the Zarr format is released, then the Zarr library will generally require changes. It is very likely that a new Zarr format will require extensive breaking changes to the Zarr library, and so support for a new Zarr format in the Zarr library will almost certainly come in a new `major` release. When the Zarr library adds support for a new Zarr format, there may be a period of accelerated changes as developers refine newly added APIs and deprecate old APIs. In such a transitional phase breaking changes may be more frequent than usual. - ## Experimental API policy The `zarr.experimental` namespace contains features that are under active development and may change without notice. When contributing to or depending on experimental features, please keep the following in mind: diff --git a/docs/index.md b/docs/index.md index 1ec5e93c75..ee4098a8ea 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,7 +6,6 @@ [Developer Chat](https://ossci.zulipchat.com/) | [Zarr specifications](https://zarr-specs.readthedocs.io) - Zarr is a powerful library for storage of n-dimensional arrays, supporting chunking, compression, and various backends, making it a versatile choice for scientific and large-scale data. @@ -37,22 +36,20 @@ conda install -c conda-forge zarr
-- [:material-clock-fast:{ .lg .middle } __Quick start__](quick-start.md) +- [:material-clock-fast:{ .lg .middle } __Quick start__](quick-start.md) --- New to Zarr? Check out the quick start guide. It contains a brief introduction to Zarr's main concepts and links to additional tutorials. - -- [:material-book-open:{ .lg .middle } __User guide__](user-guide/index.md) +- [:material-book-open:{ .lg .middle } __User guide__](user-guide/index.md) --- A detailed guide for how to use Zarr-Python. - -- [:material-api:{ .lg .middle } __API Reference__](api/zarr/index.md) +- [:material-api:{ .lg .middle } __API Reference__](api/zarr/index.md) --- @@ -61,8 +58,7 @@ conda install -c conda-forge zarr which parameters can be used. It assumes that you have an understanding of the key concepts. - -- [:material-account-group:{ .lg .middle } __Contributor's Guide__](contributing.md) +- [:material-account-group:{ .lg .middle } __Contributor's Guide__](contributing.md) --- @@ -72,7 +68,6 @@ conda install -c conda-forge zarr
- ## Project Status More information about the Zarr format can be found on the [main website](https://zarr.dev). @@ -80,6 +75,7 @@ More information about the Zarr format can be found on the [main website](https: If you are using Zarr-Python, we would [love to hear about it](https://github.com/zarr-developers/community/issues/19). ### Funding and Support + The project is fiscally sponsored by [NumFOCUS](https://numfocus.org/), a US 501(c)(3) public charity, and development has been supported by the [MRC Centre for Genomics and Global Health](https://github.com/cggh/) diff --git a/docs/quick-start.md b/docs/quick-start.md index 29232d2a04..17cb1c599a 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -50,7 +50,6 @@ written to a `LocalStore` in the `data/example-1.zarr` directory. Zarr supports data compression and filters. For example, to use Blosc compression: - ```python exec="true" session="quickstart" source="above" result="ansi" # Create a 2D Zarr array with Blosc compression @@ -73,7 +72,6 @@ print(z.info) This compresses the data using the Blosc codec with shuffle enabled for better compression. - ## Hierarchical Groups Zarr allows you to create hierarchical groups, similar to directories: diff --git a/docs/release-notes.md b/docs/release-notes.md index e85137f2b6..7a5b12f59f 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -1,5 +1,7 @@ # Release notes + + ## 3.3.0 (2026-07-15) diff --git a/docs/user-guide/attributes.md b/docs/user-guide/attributes.md index 96a5d67584..8c11c853f1 100644 --- a/docs/user-guide/attributes.md +++ b/docs/user-guide/attributes.md @@ -20,6 +20,7 @@ print('foo' in root.attrs) ```python exec="true" session="attributes" source="above" result="ansi" print(root.attrs['foo']) ``` + ```python exec="true" session="attributes" source="above" result="ansi" print(sorted(z.attrs)) ``` diff --git a/docs/user-guide/consolidated_metadata.md b/docs/user-guide/consolidated_metadata.md index 7030ed6cbc..9cb4d87c89 100644 --- a/docs/user-guide/consolidated_metadata.md +++ b/docs/user-guide/consolidated_metadata.md @@ -119,7 +119,6 @@ removed, or modified, consolidated metadata may not be desirable. `use_consolidated=False` to [`zarr.open_group`][] to always read child metadata directly from the store. - ## Stores Without Support for Consolidated Metadata Some stores may want to opt out of the consolidated metadata mechanism. This diff --git a/docs/user-guide/data_types.md b/docs/user-guide/data_types.md index 642a09d714..ac966f885f 100644 --- a/docs/user-guide/data_types.md +++ b/docs/user-guide/data_types.md @@ -194,9 +194,11 @@ Python supports nearly all of the data types in NumPy. If you need a data type t here, it's possible to create it yourself: see [Adding new data types](#adding-new-data-types). #### Boolean + - [Boolean][zarr.dtype.Bool] #### Integral + - [Signed 8-bit integer][zarr.dtype.Int8] - [Signed 16-bit integer][zarr.dtype.Int16] - [Signed 32-bit integer][zarr.dtype.Int32] @@ -207,6 +209,7 @@ here, it's possible to create it yourself: see [Adding new data types](#adding-n - [Unsigned 64-bit integer][zarr.dtype.UInt64] #### Floating-point + - [16-bit floating-point][zarr.dtype.Float16] - [32-bit floating-point][zarr.dtype.Float32] - [64-bit floating-point][zarr.dtype.Float64] @@ -214,19 +217,23 @@ here, it's possible to create it yourself: see [Adding new data types](#adding-n - [128-bit complex floating-point][zarr.dtype.Complex128] #### String + - [Fixed-length UTF-32 string][zarr.dtype.FixedLengthUTF32] - [Variable-length UTF-8 string][zarr.dtype.VariableLengthUTF8] #### Bytes + - [Fixed-length null-terminated bytes][zarr.dtype.NullTerminatedBytes] - [Fixed-length raw bytes][zarr.dtype.RawBytes] - [Variable-length bytes][zarr.dtype.VariableLengthBytes] #### Temporal + - [DateTime64][zarr.dtype.DateTime64] - [TimeDelta64][zarr.dtype.TimeDelta64] #### Struct-like + - [Structured][zarr.dtype.Structured] !!! note "Zarr V3 Structured Data Types" diff --git a/docs/user-guide/experimental.md b/docs/user-guide/experimental.md index a70ee923fe..e14146610c 100644 --- a/docs/user-guide/experimental.md +++ b/docs/user-guide/experimental.md @@ -168,7 +168,6 @@ print(f"Speedup is {speedup}") Cache effectiveness is particularly pronounced with repeated access to the same data chunks. - ### Cache Configuration The CacheStore can be configured with several parameters: diff --git a/docs/user-guide/extending.md b/docs/user-guide/extending.md index 8602ec8c3b..f852f9105e 100644 --- a/docs/user-guide/extending.md +++ b/docs/user-guide/extending.md @@ -14,6 +14,7 @@ in the following ways: [numcodecs.registry.register_codec](https://numcodecs.readthedocs.io/en/stable/registry.html#numcodecs.registry.register_codec). There are three types of codecs in Zarr: + - array-to-array - array-to-bytes - bytes-to-bytes diff --git a/docs/user-guide/groups.md b/docs/user-guide/groups.md index a2b40dfe8a..337ad39554 100644 --- a/docs/user-guide/groups.md +++ b/docs/user-guide/groups.md @@ -174,4 +174,3 @@ Groups also have the [`zarr.Group.tree`][] method, e.g.: ```python exec="true" session="groups" source="above" result="ansi" print(root.tree()) ``` - diff --git a/docs/user-guide/performance.md b/docs/user-guide/performance.md index 818386578c..52c1cf0d71 100644 --- a/docs/user-guide/performance.md +++ b/docs/user-guide/performance.md @@ -120,7 +120,6 @@ The order of chunks **within each shard** can be changed via the `subchunk_write By default [`morton`](https://en.wikipedia.org/wiki/Z-order_curve) order provides good spatial locality. [`lexicographic` (i.e., row-major)](https://en.wikipedia.org/wiki/Row-_and_column-major_order), for example, may be better suited to "batched" workflows where some form of sequential reading through a fixed number of outer dimensions is desired, and `colexicographic` is its reverse. `unordered` makes no guarantee about the order in which subchunks are laid out within a shard. - ### Empty chunks It is possible to configure how Zarr handles the storage of chunks that are "empty" @@ -221,11 +220,13 @@ zarr.config.set({'async.concurrency': 128}) ``` Higher concurrency values can improve throughput when: + - Working with remote storage (e.g., S3, GCS) where network latency is high - Reading/writing many small chunks in parallel - The storage backend can handle many concurrent requests Lower concurrency values may be beneficial when: + - Working with local storage with limited I/O bandwidth - Memory is constrained (each concurrent operation requires buffer space) - Using Zarr within a parallel computing framework (see below) @@ -258,7 +259,7 @@ concurrently. **Important**: When using many Dask threads, you may need to reduce both Zarr's `async.concurrency` and `threading.max_workers` settings to avoid creating too many concurrent operations. The total number of concurrent I/O operations can be roughly estimated as: -``` +```text total_concurrency ≈ dask_threads × zarr_async_concurrency ``` @@ -298,6 +299,7 @@ You may need to experiment with different values to find the optimal balance for Zarr arrays are designed to be thread-safe for concurrent reads and writes from multiple threads within the same process. However, proper synchronization is required when writing to overlapping regions from multiple threads. For multi-process parallelism, Zarr provides safe concurrent writes as long as: + - Different processes write to different chunks - The storage backend supports atomic writes (most do) diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index d32fe217c9..7e0154b2a0 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -41,17 +41,21 @@ group = zarr.create_group(store=data) print(group) ``` + [](){#user-guide-store-like} + ### StoreLike `StoreLike` values can be: - a `Path` or string indicating a location on the local file system. This will create a [local store](#local-store): + ```python exec="true" session="storage" source="above" result="ansi" group = zarr.open_group(store='data/foo/bar') print(group) ``` + ```python exec="true" session="storage" source="above" result="ansi" from pathlib import Path group = zarr.open_group(store=Path('data/foo/bar')) @@ -59,6 +63,7 @@ print(group) ``` - an FSSpec URI string, indicating a [remote store](#remote-store) location: + ```python exec="true" session="storage" source="above" result="ansi" # Note: requires s3fs to be installed group = zarr.open_group( @@ -70,10 +75,12 @@ print(group) ``` - an empty dictionary or None, which will create a new [memory store](#memory-store): + ```python exec="true" session="storage" source="above" result="ansi" group = zarr.create_group(store={}) print(group) ``` + ```python exec="true" session="storage" source="above" result="ansi" group = zarr.create_group(store=None) print(group) @@ -152,7 +159,6 @@ print(store) When using an S3-compatible service other than AWS, pass the service endpoint to the filesystem via `client_kwargs={'endpoint_url': 'https://...'}`. - ### Memory Store The [`zarr.storage.MemoryStore`][] stores Zarr data (metadata and chunks) in an diff --git a/lychee.toml b/lychee.toml new file mode 100644 index 0000000000..38b2b8ab7a --- /dev/null +++ b/lychee.toml @@ -0,0 +1,20 @@ +# Configuration for the lychee link checker (https://lychee.cli.rs/). +# Auto-discovered as ./lychee.toml by the lychee GitHub Action. + +# Files lychee should not scan for links. +exclude_path = [ + # mkdocs-material theme overrides: hrefs are Jinja expressions like + # `{{ '../' ~ base_url }}`, not real URLs, so lychee cannot resolve them. + "docs/overrides", + # Design notes: working records that point at transient artifacts (commits, + # fork branches, compare URLs) which are expected to disappear over time. + "design", +] + +# URL patterns to ignore (regex, matched against the full URL). +exclude = [ + # Local docs preview server shown in the contributing guide ("hatch run serve"), + # documentation of a command rather than a reachable link. + '^https?://0\.0\.0\.0', + '^https?://(localhost|127\.0\.0\.1)(:\d+)?', +] diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index ac32521239..91515d87b9 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -120,7 +120,7 @@ def ndim(self) -> int: def chunk_grid(self) -> ChunkGrid: """Backwards-compatible chunk grid property. - .. deprecated:: + !!! warning "Deprecated" Access the chunk grid via the array layer instead. This property will be removed in a future release. """ diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index 80a0996ca6..46287ccffb 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -40,21 +40,21 @@ class StoreTests[S: Store, B: Buffer]: @staticmethod def _require_get_sync(store: S) -> SupportsGetSync: - """Skip unless *store* implements :class:`SupportsGetSync`.""" + """Skip unless *store* implements [`SupportsGetSync`][zarr.abc.store.SupportsGetSync].""" if not isinstance(store, SupportsGetSync): pytest.skip("store does not implement SupportsGetSync") return store # type: ignore[unreachable] @staticmethod def _require_set_sync(store: S) -> SupportsSetSync: - """Skip unless *store* implements :class:`SupportsSetSync`.""" + """Skip unless *store* implements [`SupportsSetSync`][zarr.abc.store.SupportsSetSync].""" if not isinstance(store, SupportsSetSync): pytest.skip("store does not implement SupportsSetSync") return store # type: ignore[unreachable] @staticmethod def _require_delete_sync(store: S) -> SupportsDeleteSync: - """Skip unless *store* implements :class:`SupportsDeleteSync`.""" + """Skip unless *store* implements [`SupportsDeleteSync`][zarr.abc.store.SupportsDeleteSync].""" if not isinstance(store, SupportsDeleteSync): pytest.skip("store does not implement SupportsDeleteSync") return store # type: ignore[unreachable] From 7f8834ab14d607d1ec03209d9509a153fe81b173 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:42:21 -0400 Subject: [PATCH 42/70] chore: cleanup indentation and guidance for release (#4153) * chore: cleanup indentation and guidance for release * Make explanation a comment --- .github/ISSUE_TEMPLATE/release-checklist.md | 7 +- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/workflows/prepare_release.yml | 74 ------ ci/lint_docs.py | 67 ++++- docs/contributing.md | 21 +- docs/release-notes.md | 258 ++++++++++---------- docs/user-guide/data_types.md | 20 +- docs/user-guide/v3_migration.md | 73 +++--- 8 files changed, 251 insertions(+), 271 deletions(-) delete mode 100644 .github/workflows/prepare_release.yml diff --git a/.github/ISSUE_TEMPLATE/release-checklist.md b/.github/ISSUE_TEMPLATE/release-checklist.md index 5e073cb59f..309c76b4dc 100644 --- a/.github/ISSUE_TEMPLATE/release-checklist.md +++ b/.github/ISSUE_TEMPLATE/release-checklist.md @@ -18,8 +18,8 @@ assignees: '' - [ ] Check [SPEC 0](https://scientific-python.org/specs/spec-0000/#support-window) to see if the minimum supported version of Python or NumPy needs bumping. - [ ] Verify that the latest CI workflows on `main` are passing: [Tests](https://github.com/zarr-developers/zarr-python/actions/workflows/test.yml), [GPU Tests](https://github.com/zarr-developers/zarr-python/actions/workflows/gpu_test.yml), [Hypothesis](https://github.com/zarr-developers/zarr-python/actions/workflows/hypothesis.yaml), [Docs](https://github.com/zarr-developers/zarr-python/actions/workflows/docs.yml), [Lint](https://github.com/zarr-developers/zarr-python/actions/workflows/lint.yml), [Wheels](https://github.com/zarr-developers/zarr-python/actions/workflows/releases.yml). -- [ ] Run the ["Prepare release" workflow](https://github.com/zarr-developers/zarr-python/actions/workflows/prepare_release.yml) with the target version. This will build the changelog and open a release PR with the `run-downstream` label. -- [ ] Verify that the [downstream tests](https://github.com/zarr-developers/zarr-python/actions/workflows/downstream.yml) (triggered automatically by the `run-downstream` label) pass on the release PR. +- [ ] Run the [downstream tests](https://github.com/zarr-developers/zarr-python/actions/workflows/downstream.yml) against `main`: go to the workflow page, click "Run workflow", and select the `main` branch. Verify that the Xarray and numcodecs integration tests pass. +- [ ] Open a release PR with the changelog entries for the upcoming release, generated with `uv run --only-group release towncrier build --version x.y.z`. - [ ] Review the release PR and verify the changelog in `docs/release-notes.md` looks correct. - [ ] Merge the release PR. @@ -45,7 +45,8 @@ In rare cases (e.g. patch releases for an older minor version), you may need to - Create the release branch from the appropriate tag if it doesn't already exist. - Cherry-pick or backport the necessary commits onto the branch. -- Run `towncrier build --version x.y.z` and commit the result to the release branch instead of `main`. +- Run `towncrier build --version x.y.z` and open the release PR against the release branch instead of `main`. +- Run the downstream tests against the release branch instead of `main`. - When drafting the GitHub Release, set the target to the release branch instead of `main`. - After the release, ensure any relevant changelog updates are also reflected on `main`. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1d12aa02eb..47adb4b19e 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -15,7 +15,7 @@ - [ ] I am a human, these are my changes, and I have reviewed and understood every change and can explain why each is correct. -AI coding assistance is welcome, but a human must be the author and is responsible for the contents of the PR. The description and any review responses must be in your own words. Please read [AI-assisted contributions](https://zarr.readthedocs.io/en/stable/contributing/#ai-assisted-contributions) before opening. + ## TODO diff --git a/.github/workflows/prepare_release.yml b/.github/workflows/prepare_release.yml deleted file mode 100644 index d558779bf2..0000000000 --- a/.github/workflows/prepare_release.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Prepare release notes - -on: - workflow_dispatch: - inputs: - version: - description: 'Release version notes (e.g. 3.2.0)' - required: true - type: string - target_branch: - description: 'Branch to target' - required: false - default: 'main' - type: string - -permissions: - contents: write - pull-requests: write - -jobs: - prepare: - name: Build changelog and open PR - runs-on: ubuntu-latest - steps: - - name: Validate inputs - run: | - if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+([-\.][a-zA-Z0-9]+)*$ ]]; then - echo "::error::Invalid version format: '$VERSION'" - exit 1 - fi - if [[ ! "$TARGET_BRANCH" =~ ^[a-zA-Z0-9._/-]+$ ]]; then - echo "::error::Invalid branch name: '$TARGET_BRANCH'" - exit 1 - fi - env: - VERSION: ${{ inputs.version }} - TARGET_BRANCH: ${{ inputs.target_branch }} - - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - ref: ${{ inputs.target_branch }} - fetch-depth: 0 - persist-credentials: false - - - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - - - name: Build changelog - # Use the pinned towncrier from the `release` dependency group (single - # source of truth) rather than an unpinned standalone install. - run: uv run --only-group release towncrier build --version "$VERSION" --yes - env: - VERSION: ${{ inputs.version }} - - - name: Create pull request - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 - with: - branch: release/v${{ inputs.version }} - base: ${{ inputs.target_branch }} - title: "Release v${{ inputs.version }}" - body: | - Automated release preparation for v${{ inputs.version }}. - - This PR was generated by the "Prepare release" workflow. It includes: - - Rendered changelog via `towncrier build --version ${{ inputs.version }}` - - Removal of consumed changelog fragments from `changes/` - - ## Checklist - - - [ ] Review the rendered changelog in `docs/release-notes.md` - - [ ] Downstream tests pass (see [downstream workflow](https://github.com/zarr-developers/zarr-python/actions/workflows/downstream.yml)) - - [ ] Merge this PR, then [draft a GitHub Release](https://github.com/zarr-developers/zarr-python/releases/new) targeting `${{ inputs.target_branch }}` with tag `v${{ inputs.version }}` - commit-message: "chore: build changelog for v${{ inputs.version }}" - labels: run-downstream - delete-branch: true diff --git a/ci/lint_docs.py b/ci/lint_docs.py index ff7b56649b..a847e8aa9d 100644 --- a/ci/lint_docs.py +++ b/ci/lint_docs.py @@ -13,7 +13,7 @@ fast, source-level check that needs no docs build. Checks fall into two groups -- RST markup that silently fails under MkDocs/mkdocstrings, -and a Markdown structural problem that renders as valid-but-wrong HTML (so `mkdocs build` +and Markdown structural problems that render as valid-but-wrong HTML (so `mkdocs build` emits no warning): sphinx-role :class:`X`, :func:`X`, :py:meth:`X` -> [`X`][zarr.X] @@ -21,6 +21,8 @@ rst-field :param x:, :returns:, :rtype: -> numpydoc Parameters/Returns/Raises rst-link `text `_ -> [text](https://example) list-break unindented code fence between list items -> indent the fence under its item + list-indent continuation block indented < 4 spaces -> indent it 4 spaces + list-blank list item directly after indented block -> blank line before the item The ``list-break`` check catches a fenced code block at column 0 placed *between* two list items: because the fence is not indented into the preceding item, Markdown ends the list at @@ -29,6 +31,15 @@ notices this for sequentially-numbered ordered lists; lazily-numbered (1., 1.) and unordered lists slip past it, so this structural check covers the gap. +The ``list-indent`` and ``list-blank`` checks catch the two halves of Python-Markdown's +strict list-continuation rules, which differ from CommonMark. A blank-line-separated +block (paragraph, nested list, table) belongs to a list item only when indented at least +4 spaces; at the 2-space indent other renderers accept, Python-Markdown ends the list and +the block escapes to the top level (``list-indent``). And a new list item can not start +directly after an indented continuation block: without a blank line first, the ``- `` line +is lazily absorbed into the preceding paragraph as literal text (``list-blank``). Both +produced silently-broken changelog rendering in ``docs/release-notes.md``. + Usage: python ci/lint_docs.py [PATH ...] @@ -87,14 +98,16 @@ class Check(NamedTuple): fix: str -# ``list-break`` carries no pattern -- it is detected structurally in find_list_breaking_fences, -# not by scanning a single line -- but it appears here so it shares the remediation help. +# The ``list-*`` checks carry no pattern -- they are detected structurally, not by scanning +# a single line -- but they appear here so they share the remediation help. CHECKS = ( Check("sphinx-role", SPHINX_ROLE, ":class:`X`", "[`X`][zarr.X]"), Check("rst-directive", RST_DIRECTIVE, ".. note::", "MkDocs admonition (!!! note)"), Check("rst-field", RST_FIELD, ":param x:", "numpydoc Parameters/Returns/Raises section"), Check("rst-link", RST_LINK, "`text `_", "[text](url)"), Check("list-break", None, "fence between items", "indent the fence 4 spaces to nest it"), + Check("list-indent", None, "2-space continuation", "indent the block 4 spaces under its item"), + Check("list-blank", None, "item after indented block", "add a blank line before the item"), ) @@ -211,9 +224,49 @@ def splits_a_list(open_i: int, close_i: int) -> bool: ] +def find_list_continuation_issues( + lines: list[str], in_code: set[int] +) -> list[tuple[int, str, str]]: + """Return ``(lineno, category, snippet)`` for list continuations Python-Markdown will + mis-render (see the module docstring): + + - ``list-indent``: a blank-line-separated block inside a list item indented 1-3 + spaces. Python-Markdown requires 4; at less, the block escapes the list. + - ``list-blank``: a top-level list item directly after a line indented 4+ spaces. + Without a blank line in between, the item is absorbed into the preceding paragraph + as literal ``- `` text. + + Lazy continuations (an indented line with no blank line before it) are valid at any + indent and are not flagged. Fenced-code lines are opaque: never flagged themselves, + but they keep the item scope open and their indent feeds the ``list-blank`` check so + an item directly after an indented fence is still caught.""" + findings: list[tuple[int, str, str]] = [] + in_item = False # inside a top-level list item's scope + prev_blank = True + prev_indent = 0 + for i, line in enumerate(lines): + stripped = line.strip() + if not stripped: + prev_blank = True + continue + indent = len(line) - len(line.lstrip(" ")) + if i not in in_code: + if indent == 0: + is_item = bool(LIST_ITEM.match(line)) + if is_item and in_item and not prev_blank and prev_indent >= 4: + findings.append((i + 1, "list-blank", line)) + in_item = is_item + elif in_item and prev_blank and indent < 4: + findings.append((i + 1, "list-indent", line)) + prev_blank = False + prev_indent = indent + return findings + + def lint_markdown(path: Path) -> list[Finding]: """Scan a Markdown file: RST residue in prose (skipping fenced code blocks), plus - fenced code blocks that break a list (see find_list_breaking_fences).""" + list-structure problems (see find_list_breaking_fences and + find_list_continuation_issues).""" lines = path.read_text(encoding="utf-8").splitlines() blocks = fenced_blocks(lines) in_code = {i for fence in blocks for i in range(fence.open, fence.close + 1)} @@ -228,7 +281,11 @@ def lint_markdown(path: Path) -> list[Finding]: Finding(path, lineno, "list-break", snippet) for lineno, snippet in find_list_breaking_fences(lines, blocks) ] - return prose + breaks + continuations = [ + Finding(path, lineno, category, snippet) + for lineno, category, snippet in find_list_continuation_issues(lines, in_code) + ] + return prose + breaks + continuations def iter_files(paths: tuple[Path, ...]) -> list[Path]: diff --git a/docs/contributing.md b/docs/contributing.md index df46f381de..aeb88e6ce1 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -354,18 +354,7 @@ This policy exists to lower the cost of routine work and to help newer core deve ### Release procedure -Open an issue on GitHub announcing the release using the release checklist template: -[https://github.com/zarr-developers/zarr-python/issues/new?template=release-checklist.md](https://github.com/zarr-developers/zarr-python/issues/new?template=release-checklist.md). The release checklist includes all steps necessary for the release. - -#### Preparing a release - -Releases are prepared using the ["Prepare release notes"](https://github.com/zarr-developers/zarr-python/actions/workflows/prepare_release.yml) workflow. To run it: - -1. Go to the [workflow page](https://github.com/zarr-developers/zarr-python/actions/workflows/prepare_release.yml) and click "Run workflow". -2. Enter the release version (e.g. `3.2.0`) and the target branch (defaults to `main`). -3. The workflow will run `towncrier build` to render the changelog, remove consumed fragments from `changes/`, and open a pull request on the `release/v` branch. -4. The release PR is automatically labeled `run-downstream`, which triggers the [downstream test workflow](https://github.com/zarr-developers/zarr-python/actions/workflows/downstream.yml) to run Xarray and numcodecs integration tests against the release branch. -5. Review the rendered changelog in `docs/release-notes.md` and verify downstream tests pass before merging. +To give the release visibility and a single place to track progress, open an issue on GitHub announcing the release using the [release checklist template](https://github.com/zarr-developers/zarr-python/issues/new?template=release-checklist.md). The release checklist includes all steps necessary for the release. ## Compatibility and versioning policies @@ -377,17 +366,17 @@ Releases are classified by the library changes contained in that release. This c * **major** releases (for example, `2.18.0` -> `3.0.0`) are for changes that will require extensive adaptation efforts from many users and downstream projects. For example, breaking changes to widely-used user-facing APIs should only be applied in a major release. - Users and downstream projects should carefully consider the impact of a major release before adopting it. In advance of a major release, developers should communicate the scope of the upcoming changes, and help users prepare for them. + Users and downstream projects should carefully consider the impact of a major release before adopting it. In advance of a major release, developers should communicate the scope of the upcoming changes, and help users prepare for them. * **minor** releases (for example, `3.0.0` -> `3.1.0`) are for changes that do not require significant effort from most users or downstream projects to respond to. API changes are possible in minor releases if the burden on users imposed by those changes is sufficiently small. - For example, a recently released API may need fixes or refinements that are breaking, but low impact due to the recency of the feature. Such API changes are permitted in a minor release. + For example, a recently released API may need fixes or refinements that are breaking, but low impact due to the recency of the feature. Such API changes are permitted in a minor release. - Minor releases are safe for most users and downstream projects to adopt. + Minor releases are safe for most users and downstream projects to adopt. * **patch** releases (for example, `3.1.0` -> `3.1.1`) are for changes that contain no breaking or behavior changes for downstream projects or users. Examples of changes suitable for a patch release are bugfixes and documentation improvements. - Users should always feel safe upgrading to the latest patch release. + Users should always feel safe upgrading to the latest patch release. Note that this versioning scheme is not consistent with [Semantic Versioning](https://semver.org/). Contrary to SemVer, the Zarr library may release breaking changes in `minor` releases, or even `patch` releases under exceptional circumstances. But we should strive to avoid doing so. diff --git a/docs/release-notes.md b/docs/release-notes.md index 7a5b12f59f..3fd8a5f360 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -51,32 +51,34 @@ `ClientSession` was left open until garbage collection, producing `"Unclosed client session"` `ResourceWarning`s from aiohttp. - The fix introduces `FsspecStore._owns_fs`, a boolean that is ``True`` only when - `FsspecStore` itself created the filesystem (via `from_url` or `from_mapper` - when a sync→async conversion was performed). When `_owns_fs` is ``True``, - `store.close()` calls the new `_close_fs()` helper, which invokes - `fs.set_session()` and closes the returned client. Callers who supply their own - filesystem instance to `FsspecStore()` directly remain responsible for its - lifecycle; `_owns_fs` is ``False`` for those stores. - - **Scope note**: This fix closes the S3 client session that is active at the time - `store.close()` is called. Some S3-backed filesystem implementations (e.g. - s3fs with ``cache_regions=True``) may internally refresh and replace their - client during I/O operations, abandoning prior sessions before ``store.close()`` - is invoked. Those intermediate sessions are outside the scope of this fix and - are an issue in the upstream filesystem library. ([#4003](https://github.com/zarr-developers/zarr-python/issues/4003)) + The fix introduces `FsspecStore._owns_fs`, a boolean that is ``True`` only when + `FsspecStore` itself created the filesystem (via `from_url` or `from_mapper` + when a sync→async conversion was performed). When `_owns_fs` is ``True``, + `store.close()` calls the new `_close_fs()` helper, which invokes + `fs.set_session()` and closes the returned client. Callers who supply their own + filesystem instance to `FsspecStore()` directly remain responsible for its + lifecycle; `_owns_fs` is ``False`` for those stores. + + **Scope note**: This fix closes the S3 client session that is active at the time + `store.close()` is called. Some S3-backed filesystem implementations (e.g. + s3fs with ``cache_regions=True``) may internally refresh and replace their + client during I/O operations, abandoning prior sessions before ``store.close()`` + is invoked. Those intermediate sessions are outside the scope of this fix and + are an issue in the upstream filesystem library. ([#4003](https://github.com/zarr-developers/zarr-python/issues/4003)) + - Fixed an invalid `zarr.create_array` example in the quick-start documentation (it passed an unsupported `mode` argument) and made the cloud-storage example execute against a mock S3 backend in CI. Added a test ensuring every Python code block in the documentation is either executed or explicitly opted out with a documented reason, so an invalid example can no longer go untested. ([#4016](https://github.com/zarr-developers/zarr-python/issues/4016)) - Fixed `ObjectStore.list_dir` for object-store listings that include a directory-marker object matching the requested non-root prefix. ([#4032](https://github.com/zarr-developers/zarr-python/issues/4032)) - Prevents mutation of the attributes dict provided by the user by copying them instead of keeping the reference ([#4059](https://github.com/zarr-developers/zarr-python/issues/4059)) - Fixed several storage and codec bugs: - - Reading a value with a `SuffixByteRequest` larger than the value now correctly returns the whole value (matching HTTP `bytes=-N` suffix-range semantics), instead of silently returning incorrect data for `MemoryStore`. - - `LoggingStore.get_partial_values` and `FsspecStore.get_partial_values` no longer return empty results when `key_ranges` is passed as a one-shot iterable (e.g. a generator). - - `Store.getsize_prefix` no longer over-counts sibling keys that merely share a string prefix (e.g. `getsize_prefix("foo")` no longer includes keys under `foobar/`). - - `ZipStore.close()` no longer raises `AttributeError` when the store was created but never opened (including when used as a context manager without any I/O). - - `codecs_from_list` now raises a descriptive `TypeError` when a `BytesBytesCodec` immediately follows an `ArrayArrayCodec`, instead of a misleading "Required ArrayBytesCodec was not found" `ValueError`. + - Reading a value with a `SuffixByteRequest` larger than the value now correctly returns the whole value (matching HTTP `bytes=-N` suffix-range semantics), instead of silently returning incorrect data for `MemoryStore`. + - `LoggingStore.get_partial_values` and `FsspecStore.get_partial_values` no longer return empty results when `key_ranges` is passed as a one-shot iterable (e.g. a generator). + - `Store.getsize_prefix` no longer over-counts sibling keys that merely share a string prefix (e.g. `getsize_prefix("foo")` no longer includes keys under `foobar/`). + - `ZipStore.close()` no longer raises `AttributeError` when the store was created but never opened (including when used as a context manager without any I/O). + - `codecs_from_list` now raises a descriptive `TypeError` when a `BytesBytesCodec` immediately follows an `ArrayArrayCodec`, instead of a misleading "Required ArrayBytesCodec was not found" `ValueError`. + + ([#4074](https://github.com/zarr-developers/zarr-python/issues/4074)) - ([#4074](https://github.com/zarr-developers/zarr-python/issues/4074)) - Fixed writing Fortran-ordered (F-contiguous) arrays through the variable-length string and bytes codecs and through numcodecs array-array filters such as `Delta`, `FixedScaleOffset` and `PackBits`. Chunks are now passed to numcodecs as C-contiguous arrays, so elements are no longer stored in transposed order. ([#4116](https://github.com/zarr-developers/zarr-python/issues/4116)) - Fix silent byte-order corruption for structured dtypes with the `bytes` codec: multi-byte fields are now byte-swapped to the codec's configured `endian` on write and decoded honoring it on read, so non-native-endian structured data (e.g. big-endian fields, as produced by virtual references to external data) round-trips correctly. ([#4141](https://github.com/zarr-developers/zarr-python/issues/4141)) @@ -91,23 +93,24 @@ use only the public API, eliminating all non-public imports, illustrating what users should do. - To better support this, the following types and functions were made available - from public modules: - - | Type/Function | Non-public module | Public module | - | ------------------------- | ------------------------ | ------------- | - | `DataTypeValidationError` | `zarr.core.dtype.common` | `zarr.errors` | - | `JSON` | `zarr.core.common` | `zarr.types` | - | `ZarrFormat` | `zarr.core.common` | `zarr.types` | - | `DTypeConfig_V2` | `zarr.core.dtype.common` | `zarr.types` | - | `DTypeJSON` | `zarr.core.dtype.common` | `zarr.types` | - | `DTypeSpec_V2` | `zarr.core.dtype.common` | `zarr.dtype` | - | `check_dtype_spec_v2` | `zarr.core.dtype.common` | `zarr.dtype` | - - `DataTypeValidationError` was *moved* to `zarr.errors`. Importing it from - `zarr.core.dtype.common` (its original location), `zarr.core.dtype`, or - `zarr.dtype` still works but now raises a `ZarrDeprecationWarning`. The remaining - types and functions are simply re-exported from the listed public module. ([#4052](https://github.com/zarr-developers/zarr-python/issues/4052)) + To better support this, the following types and functions were made available + from public modules: + + | Type/Function | Non-public module | Public module | + | ------------------------- | ------------------------ | ------------- | + | `DataTypeValidationError` | `zarr.core.dtype.common` | `zarr.errors` | + | `JSON` | `zarr.core.common` | `zarr.types` | + | `ZarrFormat` | `zarr.core.common` | `zarr.types` | + | `DTypeConfig_V2` | `zarr.core.dtype.common` | `zarr.types` | + | `DTypeJSON` | `zarr.core.dtype.common` | `zarr.types` | + | `DTypeSpec_V2` | `zarr.core.dtype.common` | `zarr.dtype` | + | `check_dtype_spec_v2` | `zarr.core.dtype.common` | `zarr.dtype` | + + `DataTypeValidationError` was *moved* to `zarr.errors`. Importing it from + `zarr.core.dtype.common` (its original location), `zarr.core.dtype`, or + `zarr.dtype` still works but now raises a `ZarrDeprecationWarning`. The remaining + types and functions are simply re-exported from the listed public module. ([#4052](https://github.com/zarr-developers/zarr-python/issues/4052)) + - Document a self-merge policy in the contributor guide, describing when a core developer may merge their own pull request without a second reviewer and which changes warrant more caution. ([#4053](https://github.com/zarr-developers/zarr-python/issues/4053)) - Fixed many documentation errors found in a full review of the user guide, including prose contradicted by rendered example output on the performance page, invisible @@ -126,11 +129,12 @@ access, and will be removed in a future release. ``BloscCodec.cname`` and ``BloscCodec.shuffle`` are now plain strings rather than enum members. - Additional renames in ``zarr.codecs.blosc`` from the same change: the type - aliases ``Shuffle`` and ``CName`` are now ``BloscShuffleLiteral`` and - ``BloscCnameLiteral``, the constant ``SHUFFLE`` is now ``BLOSC_SHUFFLE`` - (with a new ``BLOSC_CNAME`` alongside it), and ``BloscShuffle.from_int`` - now returns a literal string rather than an enum member. ([#3963](https://github.com/zarr-developers/zarr-python/issues/3963)) + Additional renames in ``zarr.codecs.blosc`` from the same change: the type + aliases ``Shuffle`` and ``CName`` are now ``BloscShuffleLiteral`` and + ``BloscCnameLiteral``, the constant ``SHUFFLE`` is now ``BLOSC_SHUFFLE`` + (with a new ``BLOSC_CNAME`` alongside it), and ``BloscShuffle.from_int`` + now returns a literal string rather than an enum member. ([#3963](https://github.com/zarr-developers/zarr-python/issues/3963)) + - The ``Endian`` (``zarr.codecs.bytes.Endian``) and ``ShardingCodecIndexLocation`` (``zarr.codecs.ShardingCodecIndexLocation``) enums are now deprecated. Pass the equivalent literal string instead (e.g. ``"little"`` / ``"big"``, ``"start"`` / @@ -139,20 +143,21 @@ and ``ShardingCodec.index_location`` are now plain strings rather than enum members. - Two follow-on changes from this deprecation: + Two follow-on changes from this deprecation: - - ``NDBuffer.byteorder`` now returns a literal string (``"little"`` or - ``"big"``) rather than an ``Endian`` member. Subclasses overriding this - property should update their return type. - - The module-level binding ``zarr.codecs.bytes.default_system_endian`` was - removed. ``BytesCodec()`` continues to default to ``sys.byteorder``; - external callers that imported ``default_system_endian`` should use - ``sys.byteorder`` directly. + - ``NDBuffer.byteorder`` now returns a literal string (``"little"`` or + ``"big"``) rather than an ``Endian`` member. Subclasses overriding this + property should update their return type. + - The module-level binding ``zarr.codecs.bytes.default_system_endian`` was + removed. ``BytesCodec()`` continues to default to ``sys.byteorder``; + external callers that imported ``default_system_endian`` should use + ``sys.byteorder`` directly. - Additionally, the module-level function ``zarr.codecs.sharding.parse_index_location`` - was made private as part of this change. + Additionally, the module-level function ``zarr.codecs.sharding.parse_index_location`` + was made private as part of this change. + + ([#3968](https://github.com/zarr-developers/zarr-python/issues/3968)) - ([#3968](https://github.com/zarr-developers/zarr-python/issues/3968)) - Removed the NumPy 1.x implementation of the `VariableLengthUTF8` data type because NumPy 1.x is no longer supported under [SPEC0](https://scientific-python.org/specs/spec-0000/). ([#3973](https://github.com/zarr-developers/zarr-python/issues/3973)) ### Misc @@ -191,19 +196,19 @@ a `ManagedMemoryStore` is used. ([#3944](https://github.com/zarr-developers/zarr - Add support for rectilinear (variable-sized) chunk grids. This feature is experimental and must be explicitly enabled via `zarr.config.set({'array.rectilinear_chunks': True})`. - Rectilinear chunks can be used through: + Rectilinear chunks can be used through: - - **Creating arrays**: Pass nested sequences (e.g., `[[10, 20, 30], [50, 50]]`) to `chunks` - in `zarr.create_array`, `zarr.from_array`, `zarr.zeros`, `zarr.ones`, `zarr.full`, - `zarr.open`, and related functions, or to `chunk_shape` in `zarr.create`. - - **Opening existing arrays**: Arrays stored with the `rectilinear` chunk grid are read - transparently via `zarr.open` and `zarr.open_array`. - - **Rectilinear sharding**: Shard boundaries can be rectilinear while inner chunks remain regular. + - **Creating arrays**: Pass nested sequences (e.g., `[[10, 20, 30], [50, 50]]`) to `chunks` + in `zarr.create_array`, `zarr.from_array`, `zarr.zeros`, `zarr.ones`, `zarr.full`, + `zarr.open`, and related functions, or to `chunk_shape` in `zarr.create`. + - **Opening existing arrays**: Arrays stored with the `rectilinear` chunk grid are read + transparently via `zarr.open` and `zarr.open_array`. + - **Rectilinear sharding**: Shard boundaries can be rectilinear while inner chunks remain regular. - **Breaking change**: The `validate` method on `BaseCodec` and `CodecPipeline` now receives - a `ChunkGridMetadata` instance instead of a `ChunkGrid` instance for the `chunk_grid` - parameter. Third-party codecs that override `validate` and inspect the chunk grid will need to - update their type annotations. No known downstream packages were using this parameter. ([#3802](https://github.com/zarr-developers/zarr-python/issues/3802)) + **Breaking change**: The `validate` method on `BaseCodec` and `CodecPipeline` now receives + a `ChunkGridMetadata` instance instead of a `ChunkGrid` instance for the `chunk_grid` + parameter. Third-party codecs that override `validate` and inspect the chunk grid will need to + update their type annotations. No known downstream packages were using this parameter. ([#3802](https://github.com/zarr-developers/zarr-python/issues/3802)) - Add `cast_value` and `scale_offset` codecs. ([#3874](https://github.com/zarr-developers/zarr-python/issues/3874)) @@ -394,52 +399,52 @@ a `ManagedMemoryStore` is used. ([#3944](https://github.com/zarr-developers/zarr - Ensure that invocations of `create_array` use consistent keyword arguments, with consistent defaults. - [`zarr.api.synchronous.create_array`][] now takes a `write_data` keyword argument - The `Group.create_array` method takes `data` and `write_data` keyword arguments. - The functions [`zarr.api.asynchronous.create`][], [`zarr.api.asynchronous.create_array`] - and the methods `Group.create_array`, `Group.array`, had the default - `fill_value` changed from `0` to the `DEFAULT_FILL_VALUE` value, which instructs Zarr to - use the default scalar value associated with the array's data type as the fill value. These are - all functions or methods for array creation that mirror, wrap or are wrapped by, another function - that already has a default `fill_value` set to `DEFAULT_FILL_VALUE`. This change is necessary - to make these functions consistent across the entire codebase, but as this changes default values, - new data might have a different fill value than expected after this change. - - For data types where 0 is meaningful, like integers or floats, the default scalar is 0, so this - change should not be noticeable. For data types where 0 is ambiguous, like fixed-length unicode - strings, the default fill value might be different after this change. Users who were relying on how - Zarr interpreted `0` as a non-numeric scalar value should set their desired fill value explicitly - after this change. + [`zarr.api.synchronous.create_array`][] now takes a `write_data` keyword argument + The `Group.create_array` method takes `data` and `write_data` keyword arguments. + The functions [`zarr.api.asynchronous.create`][], [`zarr.api.asynchronous.create_array`] + and the methods `Group.create_array`, `Group.array`, had the default + `fill_value` changed from `0` to the `DEFAULT_FILL_VALUE` value, which instructs Zarr to + use the default scalar value associated with the array's data type as the fill value. These are + all functions or methods for array creation that mirror, wrap or are wrapped by, another function + that already has a default `fill_value` set to `DEFAULT_FILL_VALUE`. This change is necessary + to make these functions consistent across the entire codebase, but as this changes default values, + new data might have a different fill value than expected after this change. + + For data types where 0 is meaningful, like integers or floats, the default scalar is 0, so this + change should not be noticeable. For data types where 0 is ambiguous, like fixed-length unicode + strings, the default fill value might be different after this change. Users who were relying on how + Zarr interpreted `0` as a non-numeric scalar value should set their desired fill value explicitly + after this change. - Added public API for Buffer ABCs and implementations. - Use `zarr.buffer` to access buffer implementations, and - `zarr.abc.buffer` for the interface to implement new buffer types. + Use `zarr.buffer` to access buffer implementations, and + `zarr.abc.buffer` for the interface to implement new buffer types. - Users previously importing buffer from `zarr.core.buffer` should update their - imports to use `zarr.buffer`. As a reminder, all of `zarr.core` is - considered a private API that's not covered by zarr-python's versioning policy. ([#2871](https://github.com/zarr-developers/zarr-python/issues/2871)) + Users previously importing buffer from `zarr.core.buffer` should update their + imports to use `zarr.buffer`. As a reminder, all of `zarr.core` is + considered a private API that's not covered by zarr-python's versioning policy. ([#2871](https://github.com/zarr-developers/zarr-python/issues/2871)) - Adds zarr-specific data type classes. - This change adds a `ZDType` base class for Zarr V2 and Zarr V3 data types. Child classes are - defined for each NumPy data type. Each child class defines routines for `JSON` serialization. - New data types can be created and registered dynamically. + This change adds a `ZDType` base class for Zarr V2 and Zarr V3 data types. Child classes are + defined for each NumPy data type. Each child class defines routines for `JSON` serialization. + New data types can be created and registered dynamically. - Prior to this change, Zarr Python had two streams for handling data types. For Zarr V2 arrays, - we used NumPy data type identifiers. For Zarr V3 arrays, we used a fixed set of string enums. Both - of these systems proved hard to extend. + Prior to this change, Zarr Python had two streams for handling data types. For Zarr V2 arrays, + we used NumPy data type identifiers. For Zarr V3 arrays, we used a fixed set of string enums. Both + of these systems proved hard to extend. - This change is largely internal, but it does change the type of the `dtype` and `data_type` - fields on the `ArrayV2Metadata` and `ArrayV3Metadata` classes. Previously, `ArrayV2Metadata.dtype` - was a NumPy `dtype` object, and `ArrayV3Metadata.data_type` was an internally-defined `enum`. - After this change, both `ArrayV2Metadata.dtype` and `ArrayV3Metadata.data_type` are instances of - `ZDType`. A NumPy data type can be generated from a `ZDType` via the `ZDType.to_native_dtype()` - method. The internally-defined Zarr V3 `enum` class is gone entirely, but the `ZDType.to_json(zarr_format=3)` - method can be used to generate either a string, or dictionary that has a string `name` field, that - represents the string value previously associated with that `enum`. + This change is largely internal, but it does change the type of the `dtype` and `data_type` + fields on the `ArrayV2Metadata` and `ArrayV3Metadata` classes. Previously, `ArrayV2Metadata.dtype` + was a NumPy `dtype` object, and `ArrayV3Metadata.data_type` was an internally-defined `enum`. + After this change, both `ArrayV2Metadata.dtype` and `ArrayV3Metadata.data_type` are instances of + `ZDType`. A NumPy data type can be generated from a `ZDType` via the `ZDType.to_native_dtype()` + method. The internally-defined Zarr V3 `enum` class is gone entirely, but the `ZDType.to_json(zarr_format=3)` + method can be used to generate either a string, or dictionary that has a string `name` field, that + represents the string value previously associated with that `enum`. - For more on this new feature, see the [documentation](user-guide/data_types.md) ([#2874](https://github.com/zarr-developers/zarr-python/issues/2874)) + For more on this new feature, see the [documentation](user-guide/data_types.md) ([#2874](https://github.com/zarr-developers/zarr-python/issues/2874)) - Added `NDBuffer.empty` method for faster ndbuffer initialization. ([#3191](https://github.com/zarr-developers/zarr-python/issues/3191)) @@ -451,10 +456,10 @@ a `ManagedMemoryStore` is used. ([#3944](https://github.com/zarr-developers/zarr - Fixes a variety of issues related to string data types. - - Brings the `VariableLengthUTF8` data type Zarr V3 identifier in alignment with Zarr Python 3.0.8 - - Disallows creation of 0-length fixed-length data types - - Adds a regression test for the `VariableLengthUTF8` data type that checks against version 3.0.8 - - Allows users to request the `VariableLengthUTF8` data type with `str`, `"str"`, or `"string"`. ([#3170](https://github.com/zarr-developers/zarr-python/issues/3170)) + - Brings the `VariableLengthUTF8` data type Zarr V3 identifier in alignment with Zarr Python 3.0.8 + - Disallows creation of 0-length fixed-length data types + - Adds a regression test for the `VariableLengthUTF8` data type that checks against version 3.0.8 + - Allows users to request the `VariableLengthUTF8` data type with `str`, `"str"`, or `"string"`. ([#3170](https://github.com/zarr-developers/zarr-python/issues/3170)) - Add human readable size for No. bytes stored to `info_complete` ([#3190](https://github.com/zarr-developers/zarr-python/issues/3190)) @@ -475,26 +480,26 @@ a `ManagedMemoryStore` is used. ([#3944](https://github.com/zarr-developers/zarr - Add a description on how to create a RemoteStore of a specific filesystem to the `Remote Store` section in `docs/user-guide/storage.md`. State in the docstring of `FsspecStore.from_url` that the filesystem type is inferred from the URL scheme. - It should help a user handling the case when the type of FsspecStore doesn't match the URL scheme. ([#3212](https://github.com/zarr-developers/zarr-python/issues/3212)) + It should help a user handling the case when the type of FsspecStore doesn't match the URL scheme. ([#3212](https://github.com/zarr-developers/zarr-python/issues/3212)) ### Deprecations and Removals - Removes default chunk encoding settings (filters, serializer, compressors) from the global configuration object. - This removal is justified on the basis that storing chunk encoding settings in the config required - a brittle, confusing, and inaccurate categorization of array data types, which was particularly - unsuitable after the recent addition of new data types that didn't fit naturally into the - pre-existing categories. + This removal is justified on the basis that storing chunk encoding settings in the config required + a brittle, confusing, and inaccurate categorization of array data types, which was particularly + unsuitable after the recent addition of new data types that didn't fit naturally into the + pre-existing categories. - The default chunk encoding is the same (Zstandard compression, and the required object codecs for - variable length data types), but the chunk encoding is now generated by functions that cannot be - reconfigured at runtime. Users who relied on setting the default chunk encoding via the global configuration object should - instead specify the desired chunk encoding explicitly when creating an array. + The default chunk encoding is the same (Zstandard compression, and the required object codecs for + variable length data types), but the chunk encoding is now generated by functions that cannot be + reconfigured at runtime. Users who relied on setting the default chunk encoding via the global configuration object should + instead specify the desired chunk encoding explicitly when creating an array. - This change also adds an extra validation step to the creation of Zarr V2 arrays, which ensures that - arrays with a `VariableLengthUTF8` or `VariableLengthBytes` data type cannot be created without the - correct "object codec". ([#3228](https://github.com/zarr-developers/zarr-python/issues/3228)) + This change also adds an extra validation step to the creation of Zarr V2 arrays, which ensures that + arrays with a `VariableLengthUTF8` or `VariableLengthBytes` data type cannot be created without the + correct "object codec". ([#3228](https://github.com/zarr-developers/zarr-python/issues/3228)) - Removes support for passing keyword-only arguments positionally to the following functions and methods: `save_array`, `open`, `group`, `open_group`, `create`, `get_basic_selection`, `set_basic_selection`, @@ -546,8 +551,8 @@ a `ManagedMemoryStore` is used. ([#3944](https://github.com/zarr-developers/zarr For large arrays this will result in smaller default chunk sizes. To retain previous behaviour, explicitly set the chunk shape to the data shape. - This fix matches the existing chunking behaviour of - `zarr.save_array` and `zarr.api.asynchronous.AsyncArray.create`. ([#3103](https://github.com/zarr-developers/zarr-python/issues/3103)) + This fix matches the existing chunking behaviour of + `zarr.save_array` and `zarr.api.asynchronous.AsyncArray.create`. ([#3103](https://github.com/zarr-developers/zarr-python/issues/3103)) - When `zarr.save` has an argument `path=some/path/` and multiple arrays in `args`, the path resulted in `some/path/some/path` due to using the `path` argument twice while building the array path. This is now fixed. ([#3127](https://github.com/zarr-developers/zarr-python/issues/3127)) @@ -556,12 +561,12 @@ a `ManagedMemoryStore` is used. ([#3944](https://github.com/zarr-developers/zarr - Suppress `FileNotFoundError` when deleting non-existent keys in the `obstore` adapter. - When writing empty chunks (i.e. chunks where all values are equal to the array's fill value) to a zarr array, zarr - will delete those chunks from the underlying store. For zarr arrays backed by the `obstore` adapter, this will potentially - raise a `FileNotFoundError` if the chunk doesn't already exist. - Since whether or not a delete of a non-existing object raises an error depends on the behavior of the underlying store, - suppressing the error in all cases results in consistent behavior across stores, and is also what `zarr` seems to expect - from the store. ([#3140](https://github.com/zarr-developers/zarr-python/issues/3140)) + When writing empty chunks (i.e. chunks where all values are equal to the array's fill value) to a zarr array, zarr + will delete those chunks from the underlying store. For zarr arrays backed by the `obstore` adapter, this will potentially + raise a `FileNotFoundError` if the chunk doesn't already exist. + Since whether or not a delete of a non-existing object raises an error depends on the behavior of the underlying store, + suppressing the error in all cases results in consistent behavior across stores, and is also what `zarr` seems to expect + from the store. ([#3140](https://github.com/zarr-developers/zarr-python/issues/3140)) - Trying to open a StorePath/Array with `mode='r'` when the store is not read-only creates a read-only copy of the store. ([#3156](https://github.com/zarr-developers/zarr-python/issues/3156)) @@ -583,8 +588,9 @@ a `ManagedMemoryStore` is used. ([#3944](https://github.com/zarr-developers/zarr - It is now possible to specify no compressor when creating a zarr format 2 array. This can be done by passing `compressor=None` to the various array creation routines. - The default behaviour of automatically choosing a suitable default compressor remains if the compressor argument is not given. - To reproduce the behaviour in previous zarr-python versions when `compressor=None` was passed, pass `compressor='auto'` instead. ([#3039](https://github.com/zarr-developers/zarr-python/issues/3039)) + The default behaviour of automatically choosing a suitable default compressor remains if the compressor argument is not given. + To reproduce the behaviour in previous zarr-python versions when `compressor=None` was passed, pass `compressor='auto'` instead. ([#3039](https://github.com/zarr-developers/zarr-python/issues/3039)) + - Fixed the typing of `dimension_names` arguments throughout so that it now accepts iterables that contain `None` alongside `str`. ([#3045](https://github.com/zarr-developers/zarr-python/issues/3045)) - Using various functions to open data with `mode='a'` no longer deletes existing data in the store. ([#3062](https://github.com/zarr-developers/zarr-python/issues/3062)) - Internally use `typesize` constructor parameter for `numcodecs.blosc.Blosc` to improve compression ratios back to the v2-package levels. ([#2962](https://github.com/zarr-developers/zarr-python/issues/2962)) diff --git a/docs/user-guide/data_types.md b/docs/user-guide/data_types.md index ac966f885f..91f828a738 100644 --- a/docs/user-guide/data_types.md +++ b/docs/user-guide/data_types.md @@ -139,17 +139,17 @@ Zarr V3 introduced several key changes to how data types are represented: The basic data types are identified by strings like `"int8"`, `"int16"`, etc., and data types that require a configuration can be identified by a JSON object. - For example, this JSON object declares a datetime data type: - - ```json - { - "name": "numpy.datetime64", - "configuration": { - "unit": "s", - "scale_factor": 10 + For example, this JSON object declares a datetime data type: + + ```json + { + "name": "numpy.datetime64", + "configuration": { + "unit": "s", + "scale_factor": 10 + } } - } - ``` + ``` - Zarr V3 data types do not have endianness. This is a departure from Zarr V2, where multi-byte data types are defined with endianness information. Instead, Zarr V3 requires that the endianness diff --git a/docs/user-guide/v3_migration.md b/docs/user-guide/v3_migration.md index 6d9d516ebe..4d97963be2 100644 --- a/docs/user-guide/v3_migration.md +++ b/docs/user-guide/v3_migration.md @@ -42,40 +42,40 @@ the following actions in order: will be compatible in Zarr-Python 3. However, the following breaking API changes are planned: - - `numcodecs.*` will no longer be available in `zarr.*`. To migrate, import codecs - directly from `numcodecs`: - - ```python exec="false" reason="intentionally shows the old/incorrect import for contrast" - from numcodecs import Blosc - # instead of: - # from zarr import Blosc - ``` - - - The `zarr.v3_api_available` feature flag is being removed. In Zarr-Python 3 - the v3 API is always available, so you shouldn't need to use this flag. - - `zarr.errors` has been consolidated. Several exception classes from - Zarr-Python 2 (such as `zarr.errors.PathNotFoundError`) have been removed - or replaced. For example, missing nodes now raise `zarr.errors.NodeNotFoundError` - (which subclasses both `BaseZarrError` and `FileNotFoundError`) instead of - `zarr.errors.PathNotFoundError`. Review any code that catches exceptions - from `zarr.errors` after migrating. - - The following internal modules are being removed or significantly changed. If - your application relies on imports from any of the below modules, you will need - to either a) modify your application to no longer rely on these imports or b) - vendor the parts of the specific modules that you need. - - * `zarr.attrs` has gone, with no replacement - * `zarr.codecs` has changed, see "Codecs" section below for more information - * `zarr.context` has gone, with no replacement - * `zarr.core` remains but should be considered private API - * `zarr.hierarchy` has gone, with no replacement (use `zarr.Group` in place of `zarr.hierarchy.Group`) - * `zarr.indexing` has gone, with no replacement - * `zarr.meta` has gone, with no replacement - * `zarr.meta_v1` has gone, with no replacement - * `zarr.sync` has gone, with no replacement - * `zarr.types` has gone, with no replacement - * `zarr.util` has gone, with no replacement - * `zarr.n5` has gone, see below for an alternative N5 option + - `numcodecs.*` will no longer be available in `zarr.*`. To migrate, import codecs + directly from `numcodecs`: + + ```python exec="false" reason="intentionally shows the old/incorrect import for contrast" + from numcodecs import Blosc + # instead of: + # from zarr import Blosc + ``` + + - The `zarr.v3_api_available` feature flag is being removed. In Zarr-Python 3 + the v3 API is always available, so you shouldn't need to use this flag. + - `zarr.errors` has been consolidated. Several exception classes from + Zarr-Python 2 (such as `zarr.errors.PathNotFoundError`) have been removed + or replaced. For example, missing nodes now raise `zarr.errors.NodeNotFoundError` + (which subclasses both `BaseZarrError` and `FileNotFoundError`) instead of + `zarr.errors.PathNotFoundError`. Review any code that catches exceptions + from `zarr.errors` after migrating. + - The following internal modules are being removed or significantly changed. If + your application relies on imports from any of the below modules, you will need + to either a) modify your application to no longer rely on these imports or b) + vendor the parts of the specific modules that you need. + + * `zarr.attrs` has gone, with no replacement + * `zarr.codecs` has changed, see "Codecs" section below for more information + * `zarr.context` has gone, with no replacement + * `zarr.core` remains but should be considered private API + * `zarr.hierarchy` has gone, with no replacement (use `zarr.Group` in place of `zarr.hierarchy.Group`) + * `zarr.indexing` has gone, with no replacement + * `zarr.meta` has gone, with no replacement + * `zarr.meta_v1` has gone, with no replacement + * `zarr.sync` has gone, with no replacement + * `zarr.types` has gone, with no replacement + * `zarr.util` has gone, with no replacement + * `zarr.n5` has gone, see below for an alternative N5 option 3. Test that your package works with version 3. 4. Update the pin to include `zarr>=3,<4`. @@ -123,8 +123,9 @@ The following sections provide details on breaking changes in Zarr-Python 3. 2. The h5py compatibility methods `create_dataset` and `require_dataset` have been removed. Use the following replacements: - - [`zarr.Group.create_array`][] in place of `Group.create_dataset` - - [`zarr.Group.require_array`][] in place of `Group.require_dataset` + - [`zarr.Group.create_array`][] in place of `Group.create_dataset` + - [`zarr.Group.require_array`][] in place of `Group.require_dataset` + 3. Disallow "." syntax for getting group members. To get a member of a group named `foo`, use `group["foo"]` in place of `group.foo`. 4. The `zarr.storage.init_group` low-level helper function has been removed. Use From 326545217ffe60112b2da1175a7423ff4acb7219 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:18:56 +0200 Subject: [PATCH 43/70] chore(deps): bump the actions group across 1 directory with 6 updates (#4152) Bumps the actions group with 6 updates in the / directory: | Package | From | To | | --- | --- | --- | | [actions/checkout](https://github.com/actions/checkout) | `6.0.3` | `7.0.0` | | [astral-sh/setup-uv](https://github.com/astral-sh/setup-uv) | `8.2.0` | `8.3.2` | | [CodSpeedHQ/action](https://github.com/codspeedhq/action) | `4.18.1` | `4.18.5` | | [github-community-projects/issue-metrics](https://github.com/github-community-projects/issue-metrics) | `4.2.8` | `5.0.0` | | [lycheeverse/lychee-action](https://github.com/lycheeverse/lychee-action) | `2.8.0` | `2.9.0` | | [actions/labeler](https://github.com/actions/labeler) | `6.1.0` | `6.2.0` | Updates `actions/checkout` from 6.0.3 to 7.0.0 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v6.0.3...9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0) Updates `astral-sh/setup-uv` from 8.2.0 to 8.3.2 - [Release notes](https://github.com/astral-sh/setup-uv/releases) - [Commits](https://github.com/astral-sh/setup-uv/compare/fac544c07dec837d0ccb6301d7b5580bf5edae39...11f9893b081a58869d3b5fccaea48c9e9e46f990) Updates `CodSpeedHQ/action` from 4.18.1 to 4.18.5 - [Release notes](https://github.com/codspeedhq/action/releases) - [Changelog](https://github.com/CodSpeedHQ/action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codspeedhq/action/compare/a4a36bb07c0638b0b4ca52bf1f3dad1b4289e52f...f99becdce5e5d51fd556489ebef684f4ecfd6286) Updates `github-community-projects/issue-metrics` from 4.2.8 to 5.0.0 - [Release notes](https://github.com/github-community-projects/issue-metrics/releases) - [Commits](https://github.com/github-community-projects/issue-metrics/compare/44173f9e0a3b2144a777a10a340e4c09a25ac9f8...df8c49d20958f9345281fa2124858bd0ad227e1f) Updates `lycheeverse/lychee-action` from 2.8.0 to 2.9.0 - [Release notes](https://github.com/lycheeverse/lychee-action/releases) - [Commits](https://github.com/lycheeverse/lychee-action/compare/8646ba30535128ac92d33dfc9133794bfdd9b411...e7477775783ea5526144ba13e8db5eec57747ce8) Updates `actions/labeler` from 6.1.0 to 6.2.0 - [Release notes](https://github.com/actions/labeler/releases) - [Commits](https://github.com/actions/labeler/compare/f27b608878404679385c85cfa523b85ccb86e213...b8dd2d9be0f68b860e7dae5dae7d772984eacd6d) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: actions/labeler dependency-version: 6.2.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: astral-sh/setup-uv dependency-version: 8.3.2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: CodSpeedHQ/action dependency-version: 4.18.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: github-community-projects/issue-metrics dependency-version: 5.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: lycheeverse/lychee-action dependency-version: 2.9.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/check_changelogs.yml | 2 +- .github/workflows/codspeed.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/downstream.yml | 4 ++-- .github/workflows/gpu_test.yml | 2 +- .github/workflows/hypothesis.yaml | 2 +- .github/workflows/issue-metrics.yml | 2 +- .github/workflows/links.yml | 4 ++-- .github/workflows/lint.yml | 2 +- .github/workflows/needs_release_notes.yml | 2 +- .github/workflows/test.yml | 8 ++++---- .github/workflows/zarr-metadata-release.yml | 2 +- .github/workflows/zarr-metadata.yml | 6 +++--- 13 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/check_changelogs.yml b/.github/workflows/check_changelogs.yml index 25034b868d..0033b43db2 100644 --- a/.github/workflows/check_changelogs.yml +++ b/.github/workflows/check_changelogs.yml @@ -22,7 +22,7 @@ jobs: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 - name: Check zarr-python changelog entries run: uv run --no-sync python ci/check_changelog_entries.py diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 1d8e847ec5..39cd8eb261 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -32,7 +32,7 @@ jobs: with: version: '1.16.5' - name: Run the benchmarks - uses: CodSpeedHQ/action@a4a36bb07c0638b0b4ca52bf1f3dad1b4289e52f # v4.18.1 + uses: CodSpeedHQ/action@f99becdce5e5d51fd556489ebef684f4ecfd6286 # v4.18.5 with: mode: walltime run: hatch run test.py3.12-minimal:pytest tests/benchmarks --codspeed diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c72b493b12..baf9233fc7 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 - run: uv sync --group docs # Fast source-level guards that need no built site, so they run before the (slower) # build for a quick failure: every public export is in the API reference, and no diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index d8aa2907be..f65f8d47e3 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -45,7 +45,7 @@ jobs: python-version: '3.13' - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 - name: Install xarray and test dependencies working-directory: xarray @@ -102,7 +102,7 @@ jobs: python-version: '3.13' - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 - name: Install numcodecs with test-zarr-main group working-directory: numcodecs diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index f2bf4907fa..bbbb3e5133 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -62,7 +62,7 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 - name: Install Hatch uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc with: diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index 01212dfb56..e836f30a5b 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -57,7 +57,7 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 - name: Install Hatch uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc with: diff --git a/.github/workflows/issue-metrics.yml b/.github/workflows/issue-metrics.yml index 53860d21c8..adbd2748a0 100644 --- a/.github/workflows/issue-metrics.yml +++ b/.github/workflows/issue-metrics.yml @@ -33,7 +33,7 @@ jobs: echo "last_month=$first_day..$last_day" >> "$GITHUB_ENV" - name: Run issue-metrics tool - uses: github-community-projects/issue-metrics@44173f9e0a3b2144a777a10a340e4c09a25ac9f8 # v4.2.8 + uses: github-community-projects/issue-metrics@df8c49d20958f9345281fa2124858bd0ad227e1f # v5.0.0 env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} SEARCH_QUERY: 'repo:zarr-developers/zarr-python is:issue created:${{ env.last_month }} -reason:"not planned"' diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml index a53de8da83..f606f12a3e 100644 --- a/.github/workflows/links.yml +++ b/.github/workflows/links.yml @@ -13,13 +13,13 @@ jobs: permissions: issues: write # required for peter-evans/create-issue-from-file steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Link Checker id: lychee - uses: lycheeverse/lychee-action@8646ba30535128ac92d33dfc9133794bfdd9b411 # v2.8.0 + uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0 with: fail: false diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8190b1e061..dacba6648f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -27,7 +27,7 @@ jobs: with: python-version: "3.12" - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: enable-cache: true - uses: j178/prek-action@e98a699c41eb69ab013a45817a0406469a748f8d # v2.0.5 diff --git a/.github/workflows/needs_release_notes.yml b/.github/workflows/needs_release_notes.yml index 1f79725b86..fa555d1478 100644 --- a/.github/workflows/needs_release_notes.yml +++ b/.github/workflows/needs_release_notes.yml @@ -21,7 +21,7 @@ jobs: pull-requests: write # Required to add labels to PRs runs-on: ubuntu-latest steps: - - uses: actions/labeler@f27b608878404679385c85cfa523b85ccb86e213 # v6.1.0 + - uses: actions/labeler@b8dd2d9be0f68b860e7dae5dae7d772984eacd6d # v6.2.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} sync-labels: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7bc43512b5..ab78cfbe2b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,7 +66,7 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 - name: Install Hatch run: python -m pip install hatch==1.16.5 - name: Set Up Hatch Env @@ -115,7 +115,7 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 - name: Install Hatch run: python -m pip install hatch==1.16.5 - name: Set Up Hatch Env @@ -150,7 +150,7 @@ jobs: python-version: '3.13' cache: 'pip' - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 - name: Install Hatch run: python -m pip install hatch==1.16.5 - name: Set Up Hatch Env @@ -174,7 +174,7 @@ jobs: python-version: '3.13' cache: 'pip' - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 - name: Install Hatch run: python -m pip install hatch==1.16.5 - name: Run Benchmarks diff --git a/.github/workflows/zarr-metadata-release.yml b/.github/workflows/zarr-metadata-release.yml index db05489798..5021f79d2e 100644 --- a/.github/workflows/zarr-metadata-release.yml +++ b/.github/workflows/zarr-metadata-release.yml @@ -51,7 +51,7 @@ jobs: path: dist - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: enable-cache: false diff --git a/.github/workflows/zarr-metadata.yml b/.github/workflows/zarr-metadata.yml index 95e8251227..4e5bb0fb1a 100644 --- a/.github/workflows/zarr-metadata.yml +++ b/.github/workflows/zarr-metadata.yml @@ -36,7 +36,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: enable-cache: true - name: Set up Python ${{ matrix.python-version }} @@ -58,7 +58,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 - name: Run ruff run: uvx ruff check . @@ -74,7 +74,7 @@ jobs: with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: enable-cache: true - name: Set up Python From 50b7e016590d76011382771fd52843949782c9c0 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Fri, 17 Jul 2026 15:34:21 +0200 Subject: [PATCH 44/70] Fix/memorystore buffer aliasing (#4157) * fix: byte-order handling for structured dtypes in the bytes codec (#220) * fix: byte-order handling for structured dtypes in the bytes codec The bytes codec neither byte-swapped structured-dtype fields to its configured endian on encode (numpy reports byteorder '|' for void dtypes, so the top-level byteorder comparison never detected a mismatch) nor honored its endian when decoding, silently corrupting any structured data whose field byte order differed from the stored one (e.g. virtual references to external big-endian data). Encode now detects byte-order mismatches by comparing full dtypes via newbyteorder, and decode reinterprets raw bytes in the stored byte order before converting to the data type's declared byte order, so the stored layout (codec state) and the in-memory layout (array data type) are independent. Closes #4141 Assisted-by: ClaudeCode:claude-fable-5 * test: fold structured byte-order cases into existing bytes codec tests Extend test_endian's parametrization with structured dtypes and test_bytes_codec_sync_roundtrip with endian/dtype parametrization plus stored-layout and decoded-dtype assertions, instead of adding parallel test functions for the same properties. Assisted-by: ClaudeCode:claude-fable-5 * refactor: rename stored_dtype to view_dtype in BytesCodec decode The variable is the dtype used to view the raw chunk bytes (byte order from the codec's endian configuration), not a property of the stored data or of the returned buffer, which always carries the array's declared dtype. Assisted-by: ClaudeCode:claude-fable-5 * docs: note that the decode-side byte-order conversion copies the chunk Assisted-by: ClaudeCode:claude-fable-5 * fix(store): copy buffers on write in MemoryStore Encoding an uncompressed chunk hands the store a zero-copy view of the caller's array, and MemoryStore keeps whatever it is given alive in a dict rather than serializing it. Mutating the source array after a write therefore rewrote chunks already committed to the store, silently. Stores that serialize on write (LocalStore, ZipStore, remote stores) are unaffected, so they keep the full benefit of #3885. Only MemoryStore pays the copy, and only where it was aliasing to begin with: an uncompressed 34 MB write goes from ~23 ms to ~39 ms, while compressed writes are unchanged. Copying at the store boundary rather than narrowing the fast path in _merge_chunk_array also fixes the single-chunk case, which aliased in v3.2.1 too. Assisted-by: ClaudeCode:claude-opus-4.8 * docs: add changelog entry for MemoryStore buffer copy Assisted-by: ClaudeCode:claude-opus-4.8 * docs: correct changelog --- changes/4157.bugfix.md | 11 +++++++ src/zarr/core/buffer/core.py | 2 ++ src/zarr/storage/_memory.py | 24 ++++++++++++++-- tests/test_store/test_memory.py | 51 ++++++++++++++++++++++++++++++++- 4 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 changes/4157.bugfix.md diff --git a/changes/4157.bugfix.md b/changes/4157.bugfix.md new file mode 100644 index 0000000000..6b0d0fcc67 --- /dev/null +++ b/changes/4157.bugfix.md @@ -0,0 +1,11 @@ +`MemoryStore` now copies buffers as they are written, so it never retains the +caller's memory. Previously an uncompressed write handed the store a zero-copy +view of the user's array, and mutating that array afterwards would silently +rewrite chunks already committed to the store. + +Only `MemoryStore` is affected: stores that serialize on write, such as +`LocalStore` and `ZipStore`, never aliased the caller's memory. Uncompressed +writes to a `MemoryStore` are correspondingly slower, since the copy that makes +the stored data independent is now actually performed; compressed writes are +unchanged. Buffers supplied through the `store_dict` argument remain the +caller's responsibility and are stored as-is. diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py index 497543f88f..b8f7f11cd4 100644 --- a/src/zarr/core/buffer/core.py +++ b/src/zarr/core/buffer/core.py @@ -45,6 +45,8 @@ def __getitem__(self, key: slice) -> Self: ... def __setitem__(self, key: slice, value: Any) -> None: ... + def copy(self) -> Self: ... + @runtime_checkable class NDArrayLike(Protocol): diff --git a/src/zarr/storage/_memory.py b/src/zarr/storage/_memory.py index 5f5b632d4a..97dd355515 100644 --- a/src/zarr/storage/_memory.py +++ b/src/zarr/storage/_memory.py @@ -26,6 +26,18 @@ logger = getLogger(__name__) +def _copy_buffer(value: Buffer) -> Buffer: + """Copy `value` so the store does not retain the caller's memory. + + Encoding a chunk can hand the store a zero-copy view of the user's array + (an uncompressed write is the common case), and unlike stores that + serialize on write, this one keeps whatever it is given alive in a dict. + Without this copy a later mutation of the user's array would rewrite + chunks already committed to the store. + """ + return type(value).from_array_like(value.as_array_like().copy()) + + class MemoryStore(Store): """ Store for local memory. @@ -42,6 +54,12 @@ class MemoryStore(Store): supports_writes supports_deletes supports_listing + + Notes + ----- + Writes copy the buffer they are given, so the store never aliases the + caller's memory. Buffers passed via `store_dict` are the caller's + responsibility and are stored as-is. """ supports_writes: bool = True @@ -117,7 +135,7 @@ def set_sync(self, key: str, value: Buffer) -> None: raise TypeError( f"MemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." ) - self._store_dict[key] = value + self._store_dict[key] = _copy_buffer(value) def delete_sync(self, key: str) -> None: self._check_writable() @@ -178,13 +196,13 @@ async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None buf[byte_range[0] : byte_range[1]] = value self._store_dict[key] = buf else: - self._store_dict[key] = value + self._store_dict[key] = _copy_buffer(value) async def set_if_not_exists(self, key: str, value: Buffer) -> None: # docstring inherited self._check_writable() await self._ensure_open() - self._store_dict.setdefault(key, value) + self._store_dict.setdefault(key, _copy_buffer(value)) async def delete(self, key: str) -> None: # docstring inherited diff --git a/tests/test_store/test_memory.py b/tests/test_store/test_memory.py index 35504718a7..36265423e6 100644 --- a/tests/test_store/test_memory.py +++ b/tests/test_store/test_memory.py @@ -8,7 +8,7 @@ import pytest import zarr -from zarr.core.buffer import Buffer, cpu, gpu +from zarr.core.buffer import Buffer, cpu, default_buffer_prototype, gpu from zarr.errors import ZarrUserWarning from zarr.storage import GpuMemoryStore, ManagedMemoryStore, MemoryStore from zarr.testing.store import StoreTests @@ -76,6 +76,55 @@ async def test_deterministic_size( np.testing.assert_array_equal(a[:3], 1) np.testing.assert_array_equal(a[3:], 0) + @pytest.mark.parametrize("method", ["set", "set_sync", "set_if_not_exists"]) + async def test_set_does_not_retain_caller_buffer(self, store: MemoryStore, method: str) -> None: + """Writing a buffer must not alias the caller's memory. + + MemoryStore keeps whatever it is handed alive in a dict, so retaining + the caller's buffer lets a later mutation of that buffer rewrite data + already committed to the store. + """ + source = np.frombuffer(bytearray(b"\x01\x02\x03\x04"), dtype="B") + value = cpu.Buffer.from_array_like(source) + + if method == "set_sync": + store.set_sync("k", value) + else: + await getattr(store, method)("k", value) + + source[:] = 0xF # mutate the caller's memory after the write + stored = await store.get("k", prototype=default_buffer_prototype()) + assert stored is not None + assert stored.to_bytes() == b"\x01\x02\x03\x04" + + @pytest.mark.parametrize( + "pipeline", + [ + "zarr.core.codec_pipeline.BatchedCodecPipeline", + "zarr.core.codec_pipeline.FusedCodecPipeline", + ], + ) + @pytest.mark.parametrize(("shape", "chunks"), [((30,), (10,)), ((8,), (4,)), ((4,), (4,))]) + def test_write_does_not_alias_source_array( + self, pipeline: str, shape: tuple[int], chunks: tuple[int] + ) -> None: + """Mutating the source array after a write must not corrupt stored chunks. + + Without compression the encoded buffer is a zero-copy view of the + caller's array all the way down to the store, so this covers both the + single-chunk and multi-chunk write paths. + """ + with zarr.config.set({"codec_pipeline.path": pipeline}): + array = zarr.create_array( + store=MemoryStore(), shape=shape, chunks=chunks, dtype="i4", compressors=None + ) + source = np.arange(shape[0], dtype="i4") + expected = source.copy() + array[:] = source + source[:] = -1 + + np.testing.assert_array_equal(array[:], expected) + # --- byte-range-write tests: disabled --- # Byte-range-write support (set_range / set_range_sync / SupportsSetRange) # was removed from this PR pending a decision on the store interface. These From ab769985d7aa0279e9f628ab4148660bbf4921a5 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Sun, 19 Jul 2026 14:19:48 -0400 Subject: [PATCH 45/70] chore: refine link checker configuration (#4158) --- lychee.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lychee.toml b/lychee.toml index 38b2b8ab7a..54a5b49b8d 100644 --- a/lychee.toml +++ b/lychee.toml @@ -1,6 +1,9 @@ # Configuration for the lychee link checker (https://lychee.cli.rs/). # Auto-discovered as ./lychee.toml by the lychee GitHub Action. +# Treat redirect status codes as success rather than failures. +accept = ["200..=299"] + # Files lychee should not scan for links. exclude_path = [ # mkdocs-material theme overrides: hrefs are Jinja expressions like @@ -17,4 +20,6 @@ exclude = [ # documentation of a command rather than a reachable link. '^https?://0\.0\.0\.0', '^https?://(localhost|127\.0\.0\.1)(:\d+)?', + # SPEC 0 page times out but is valid. + '^https://scientific-python\.org/specs/spec-0000', ] From ecc2d77718dbc4343da052347c4377945246d4fa Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Mon, 20 Jul 2026 16:39:17 +0200 Subject: [PATCH 46/70] fix(store): don't close a shared filesystem in FsspecStore.close() (#4165) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: byte-order handling for structured dtypes in the bytes codec (#220) * fix: byte-order handling for structured dtypes in the bytes codec The bytes codec neither byte-swapped structured-dtype fields to its configured endian on encode (numpy reports byteorder '|' for void dtypes, so the top-level byteorder comparison never detected a mismatch) nor honored its endian when decoding, silently corrupting any structured data whose field byte order differed from the stored one (e.g. virtual references to external big-endian data). Encode now detects byte-order mismatches by comparing full dtypes via newbyteorder, and decode reinterprets raw bytes in the stored byte order before converting to the data type's declared byte order, so the stored layout (codec state) and the in-memory layout (array data type) are independent. Closes #4141 Assisted-by: ClaudeCode:claude-fable-5 * test: fold structured byte-order cases into existing bytes codec tests Extend test_endian's parametrization with structured dtypes and test_bytes_codec_sync_roundtrip with endian/dtype parametrization plus stored-layout and decoded-dtype assertions, instead of adding parallel test functions for the same properties. Assisted-by: ClaudeCode:claude-fable-5 * refactor: rename stored_dtype to view_dtype in BytesCodec decode The variable is the dtype used to view the raw chunk bytes (byte order from the codec's endian configuration), not a property of the stored data or of the returned buffer, which always carries the array's declared dtype. Assisted-by: ClaudeCode:claude-fable-5 * docs: note that the decode-side byte-order conversion copies the chunk Assisted-by: ClaudeCode:claude-fable-5 * fix(store): FsspecStore.close() no longer closes the filesystem FsspecStore.close() closed the underlying filesystem's session, on the premise that a store built by from_url "owns" the filesystem it created. That premise does not hold: fsspec caches and shares filesystem instances across callers (its instance cache keys on storage options, not path), and users can hand one filesystem to many stores directly. Closing one store therefore killed the session that sibling stores were still using, and left the dead filesystem in fsspec's cache for later callers. Determining whether a filesystem is actually shared requires reaching into fsspec's private instance cache (_cache, _fs_token, cachable) and walking wrapper chains for caching/proxy filesystems — an implementation detail that leaks upward and that we would have to keep in sync with fsspec forever, getting it subtly wrong in between. The wrapper case alone (simplecache::/dir://) already slipped through a cache-membership check. The filesystem's lifecycle is simply not the store's to manage. This removes the ownership model added in the unreleased gh-4003: no _owns_fs, no _close_fs, no ownership transfer in with_read_only, and close() just marks the store not-open. The only thing given up is suppressing an "Unclosed client session" ResourceWarning, which was true anyway — the session belongs to a cached filesystem that outlives the store. Since gh-4003 never shipped (latest release is v3.2.1), its changelog fragment is removed rather than superseded. Assisted-by: ClaudeCode:claude-opus-4.8 * test: skip with_read_only fs test when AsyncFileSystemWrapper is absent test_with_read_only_shares_filesystem replaced an ownership test that carried a guard for fsspec < 2024.12.0, and the guard was dropped in the rewrite. The test still opens a file:// URL, which needs AsyncFileSystemWrapper, so it failed the min_deps job. Assisted-by: ClaudeCode:claude-opus-4.8 * docs: correct changelog claim about gh-4003 release status The fragment said gh-4003 was unreleased with no net change for released versions. Its text is already in the staged 3.3.0 release notes, so the revert is a real behavior change for anyone relying on close() releasing the session. Assisted-by: ClaudeCode:claude-opus-4.8 * docs: remove changelog entry for unreleased versions --- src/zarr/storage/_fsspec.py | 70 ++++------------- tests/test_store/test_fsspec.py | 134 ++++++++------------------------ 2 files changed, 47 insertions(+), 157 deletions(-) diff --git a/src/zarr/storage/_fsspec.py b/src/zarr/storage/_fsspec.py index 617980ac19..37d134dd95 100644 --- a/src/zarr/storage/_fsspec.py +++ b/src/zarr/storage/_fsspec.py @@ -3,7 +3,6 @@ import json import warnings from contextlib import suppress -from logging import getLogger from typing import TYPE_CHECKING, Any from packaging.version import parse as parse_version @@ -19,8 +18,6 @@ from zarr.errors import ZarrUserWarning from zarr.storage._utils import _dereference_path -logger = getLogger(__name__) - if TYPE_CHECKING: from collections.abc import AsyncIterator, Iterable @@ -38,26 +35,6 @@ ) -async def _close_fs(fs: AsyncFileSystem) -> None: - """ - Best-effort async close of an fsspec async filesystem owned by FsspecStore. - - For filesystems that expose `set_session()` (e.g. s3fs) the underlying - aiohttp `ClientSession` is closed explicitly, which prevents - "Unclosed client session" `ResourceWarning`s from aiohttp. For all - other filesystem types the call is a no-op (not every implementation - manages an HTTP session directly). - - Note that `set_session()` lazily creates a session if none exists yet, so - closing a store that never performed any I/O may instantiate a session - purely to close it. This is accepted best-effort behavior; fsspec does not - expose a stable, cross-implementation way to test for an existing session. - """ - if hasattr(fs, "set_session"): - session = await fs.set_session() - await session.close() - - def _make_async(fs: AbstractFileSystem) -> AsyncFileSystem: """Convert a sync FSSpec filesystem to an async FFSpec filesystem @@ -126,6 +103,15 @@ class FsspecStore(Store): ZarrUserWarning If the file system (fs) was not created with `asynchronous=True`. + Notes + ----- + Closing the store does not close the underlying filesystem or its network + session. fsspec caches and shares filesystem instances across callers, so + the store cannot know whether it is the only user, and closing a shared + session would break other stores. The filesystem's lifecycle belongs to + whoever created it; use fsspec's own tools (e.g. `clear_instance_cache`) + to release it. + See Also -------- FsspecStore.from_upath @@ -152,9 +138,6 @@ def __init__( self.fs = fs self.path = path self.allowed_exceptions = allowed_exceptions - # True only when this store created fs itself (from_url / from_mapper with new instance). - # Callers who supply their own fs remain responsible for its lifecycle. - self._owns_fs: bool = False if not self.fs.async_impl: raise TypeError("Filesystem needs to support async operations.") @@ -220,17 +203,13 @@ def from_mapper( ------- FsspecStore """ - original_fs = fs_map.fs - fs = _make_async(original_fs) - store = cls( + fs = _make_async(fs_map.fs) + return cls( fs=fs, path=fs_map.root, read_only=read_only, allowed_exceptions=allowed_exceptions, ) - # _make_async returns a new instance when converting sync→async; own it. - store._owns_fs = fs is not original_fs - return store @classmethod def from_url( @@ -272,39 +251,16 @@ def from_url( if not fs.async_impl: fs = _make_async(fs) - store = cls(fs=fs, path=path, read_only=read_only, allowed_exceptions=allowed_exceptions) - store._owns_fs = True - return store + return cls(fs=fs, path=path, read_only=read_only, allowed_exceptions=allowed_exceptions) def with_read_only(self, read_only: bool = False) -> FsspecStore: # docstring inherited - new_store = type(self)( + return type(self)( fs=self.fs, path=self.path, allowed_exceptions=self.allowed_exceptions, read_only=read_only, ) - # The derived store shares the same fs. Transfer ownership so the - # surviving store closes it, and clear ours to avoid a double-close. - # Otherwise the common `from_url(...).with_read_only()` chain would - # drop the only owner (the unreferenced source) and leak the session. - new_store._owns_fs = self._owns_fs - self._owns_fs = False - return new_store - - def close(self) -> None: - # docstring inherited - if self._owns_fs: - from zarr.core.sync import sync as zarr_sync - - # Best-effort: a failure to release the session must not block close(), - # but log it so a genuine regression in the close path stays observable - # rather than silently reverting to the leaking behavior. - try: - zarr_sync(_close_fs(self.fs)) - except Exception: - logger.debug("Failed to close owned filesystem %r", self.fs, exc_info=True) - super().close() async def clear(self) -> None: # docstring inherited diff --git a/tests/test_store/test_fsspec.py b/tests/test_store/test_fsspec.py index 898d49ec08..515e1526b6 100644 --- a/tests/test_store/test_fsspec.py +++ b/tests/test_store/test_fsspec.py @@ -276,75 +276,20 @@ async def test_delete_dir_unsupported_deletes(self, store: FsspecStore) -> None: ): await store.delete_dir("test_prefix") - # ── Filesystem lifecycle (ownership) ────────────────────────────────────── + # ── Filesystem lifecycle ────────────────────────────────────────────────── - def test_from_url_owns_filesystem(self, endpoint_url: str) -> None: - """FsspecStore.from_url() creates the async fs; it must own it.""" + async def test_close_marks_store_closed(self, endpoint_url: str) -> None: + """close() must succeed and mark the store not-open.""" store = FsspecStore.from_url( f"s3://{test_bucket_name}/lifecycle/", storage_options={"endpoint_url": endpoint_url, "anon": False}, ) - assert store._owns_fs - store.close() - - async def test_from_url_close_releases_store(self, endpoint_url: str) -> None: - """ - close() on a from_url() store must succeed without error and mark the - store as closed. For the owned filesystem, _close_fs() is invoked to - release the underlying S3 client / aiohttp connection pool. - """ - store = FsspecStore.from_url( - f"s3://{test_bucket_name}/lifecycle/", - storage_options={"endpoint_url": endpoint_url, "anon": False}, - ) - # Materialise the S3 client and connection pool. await store.set("probe", cpu.Buffer.from_bytes(b"x")) store.close() assert not store._is_open - def test_direct_construction_does_not_own_filesystem(self, endpoint_url: str) -> None: - """Direct FsspecStore() must not claim ownership — the caller owns the fs.""" - try: - from fsspec import url_to_fs - except ImportError: - from fsspec.core import url_to_fs - fs, path = url_to_fs( - f"s3://{test_bucket_name}", endpoint_url=endpoint_url, anon=False, asynchronous=True - ) - store = FsspecStore(fs=fs, path=path) - assert not store._owns_fs - - @pytest.mark.skipif( - parse_version(fsspec.__version__) < parse_version("2024.03.01"), - reason="Prior bug in from_upath", - ) - def test_from_upath_does_not_own_filesystem(self, endpoint_url: str) -> None: - """from_upath() uses the UPath's existing fs; the store must not own it.""" - upath = pytest.importorskip("upath") - path = upath.UPath( - f"s3://{test_bucket_name}/foo/bar/", - endpoint_url=endpoint_url, - anon=False, - asynchronous=True, - ) - store = FsspecStore.from_upath(path) - assert not store._owns_fs - - def test_from_mapper_does_not_own_already_async_filesystem(self, endpoint_url: str) -> None: - """from_mapper() with an already-async fs must not claim ownership.""" - s3_filesystem = s3fs.S3FileSystem( - asynchronous=True, - endpoint_url=endpoint_url, - anon=False, - skip_instance_cache=True, - ) - mapper = s3_filesystem.get_mapper(f"s3://{test_bucket_name}/") - store = FsspecStore.from_mapper(mapper) - # _make_async returns the same instance for an already-async fs. - assert not store._owns_fs - def array_roundtrip(store: FsspecStore) -> None: """ @@ -574,47 +519,47 @@ def test_open_s3map_raises(endpoint_url: str) -> None: zarr.open(store=mapper, storage_options={"anon": True}, mode="w", shape=(3, 3)) -async def test_close_fs_closes_s3_client() -> None: - """ - _close_fs() must call set_session() and then close() on the returned - S3 client. This is verified with mocks to avoid a real S3 connection. - """ - from unittest.mock import AsyncMock +async def test_close_does_not_close_filesystem_session() -> None: + """close() must not touch the filesystem's session. - from zarr.storage._fsspec import _close_fs + fsspec caches and shares filesystem instances across callers, so the + session is not the store's to close. HTTP is used because its aiohttp + session is observably closed for good; s3fs transparently reconnects, which + would hide a regression. No request is issued — set_session() only + constructs the session. + """ + pytest.importorskip("aiohttp") + store = FsspecStore.from_url("http://example.com/a") + session = await store.fs.set_session() - mock_client = AsyncMock() - mock_fs = AsyncMock() - mock_fs.set_session = AsyncMock(return_value=mock_client) + store.close() - await _close_fs(mock_fs) + assert not session.closed - mock_fs.set_session.assert_called_once() - mock_client.close.assert_called_once() +async def test_close_does_not_break_a_sibling_store() -> None: + """Closing one store must not close a session another store is using. -async def test_close_fs_no_op_for_fs_without_set_session() -> None: - """_close_fs() must be a no-op for filesystems that don't expose set_session().""" - from unittest.mock import AsyncMock + Two stores from different URLs on one host are handed the same cached + filesystem; a store that closed it on close() would take the sibling's + session down too. This is the regression guard for that bug. + """ + pytest.importorskip("aiohttp") + s1 = FsspecStore.from_url("http://example.com/a") + s2 = FsspecStore.from_url("http://example.com/b") + session = await s2.fs.set_session() - from zarr.storage._fsspec import _close_fs + s1.close() - mock_fs = AsyncMock(spec=[]) # empty spec — no set_session attribute - await _close_fs(mock_fs) # must not raise + assert not session.closed @pytest.mark.skipif( parse_version(fsspec.__version__) < parse_version("2024.12.0"), reason="No AsyncFileSystemWrapper", ) -def test_from_mapper_owns_wrapped_sync_filesystem(tmp_path: pathlib.Path) -> None: - """ - from_mapper() with a sync fs must wrap it in AsyncFileSystemWrapper and - claim ownership so that close() cleans it up. - - The local filesystem is synchronous; _make_async() produces a new - AsyncFileSystemWrapper instance — a different object from the original fs. - """ +def test_from_mapper_wraps_sync_filesystem(tmp_path: pathlib.Path) -> None: + """from_mapper() with a sync fs wraps it in an AsyncFileSystemWrapper.""" import fsspec as _fsspec from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper @@ -622,32 +567,21 @@ def test_from_mapper_owns_wrapped_sync_filesystem(tmp_path: pathlib.Path) -> Non mapper = fs.get_mapper(str(tmp_path)) store = FsspecStore.from_mapper(mapper) assert isinstance(store.fs, AsyncFileSystemWrapper) - assert store._owns_fs @pytest.mark.skipif( parse_version(fsspec.__version__) < parse_version("2024.12.0"), reason="No AsyncFileSystemWrapper", ) -def test_with_read_only_transfers_filesystem_ownership(tmp_path: pathlib.Path) -> None: - """ - with_read_only() must transfer fs ownership to the derived store and clear - it on the source, so the surviving store closes the shared fs exactly once. - - In the common ``from_url(...).with_read_only()`` chain the source store is - immediately unreferenced; if ownership were not transferred, the only owner - would be garbage-collected without close() and the session would leak. - """ +def test_with_read_only_shares_filesystem(tmp_path: pathlib.Path) -> None: + """with_read_only() returns a store sharing the source's filesystem.""" source = FsspecStore.from_url(f"file://{tmp_path}", storage_options={"auto_mkdir": False}) - assert source._owns_fs derived = source.with_read_only(read_only=True) - # Ownership moved to the survivor; the source no longer owns it (no double-close). - assert derived._owns_fs - assert not source._owns_fs - # The derived store shares the same underlying fs. assert derived.fs is source.fs + assert derived.read_only + assert not source.read_only @pytest.mark.parametrize("asynchronous", [True, False]) From ee2c3d2140ababd8feb8a392bec5ae794f0a30c1 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 12:52:15 +0200 Subject: [PATCH 47/70] fix(zarr-metadata): harden model validation Assisted-by: Codex:gpt-5 --- .../src/zarr_metadata/model/_array.py | 9 +- .../src/zarr_metadata/model/_group.py | 30 +++- .../src/zarr_metadata/model/_validation.py | 155 ++++++++++++++++-- .../zarr-metadata/tests/model/test_array.py | 86 ++++++++++ .../zarr-metadata/tests/model/test_group.py | 41 +++++ 5 files changed, 297 insertions(+), 24 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index f68c1b0f5e..aee4f80fa8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -447,11 +447,14 @@ def from_json(cls, data: object) -> ArrayMetadataModelV2: @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV2: - zarray = load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V2) + zarray_raw = cast("object", load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V2)) + if not isinstance(zarray_raw, Mapping): + return cls.from_json(zarray_raw) + zarray = cast("Mapping[str, object]", zarray_raw) if ATTRIBUTES_STORE_KEY_V2 in mapping: - zattrs = load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) + zattrs = cast("object", load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2)) return cls.from_json({**zarray, "attributes": zattrs}) - return cls.from_json(dict(zarray)) + return cls.from_json(zarray) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: # Attributes live only in the sibling `.zattrs` file; the `.zarray` diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 222b0b3bf9..81bbbad270 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -25,6 +25,7 @@ parse_group_metadata_v2, parse_group_metadata_v3, validate_consolidated_metadata_v3, + validate_json, ) if TYPE_CHECKING: @@ -323,11 +324,14 @@ def from_json(cls, data: object) -> GroupMetadataModelV2: @classmethod def from_key_value(cls, mapping: Mapping[str, bytes]) -> GroupMetadataModelV2: - zgroup = load_store_json(mapping, GROUP_METADATA_STORE_KEY_V2) + zgroup_raw = cast("object", load_store_json(mapping, GROUP_METADATA_STORE_KEY_V2)) + if not isinstance(zgroup_raw, Mapping): + return cls.from_json(zgroup_raw) + zgroup = cast("Mapping[str, object]", zgroup_raw) if ATTRIBUTES_STORE_KEY_V2 in mapping: - zattrs = load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2) + zattrs = cast("object", load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2)) return cls.from_json({**zgroup, "attributes": zattrs}) - return cls.from_json(dict(zgroup)) + return cls.from_json(zgroup) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: # Attributes live only in the sibling `.zattrs` file; the `.zgroup` @@ -374,6 +378,18 @@ def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: for key in ("zarr_consolidated_format", "metadata") if key not in doc ] + if "zarr_consolidated_format" in doc and ( + not isinstance(doc["zarr_consolidated_format"], int) + or isinstance(doc["zarr_consolidated_format"], bool) + or doc["zarr_consolidated_format"] != 1 + ): + problems.append( + ValidationProblem( + ("zarr_consolidated_format",), + f"expected 1, got {doc['zarr_consolidated_format']!r}", + "invalid_value", + ) + ) if "metadata" in doc: entries = doc["metadata"] if not isinstance(entries, Mapping) or not all( @@ -384,6 +400,14 @@ def from_json(cls, data: object) -> ConsolidatedMetadataModelV2: ("metadata",), "expected a mapping with string keys", "invalid_type" ) ) + else: + for key, value in cast("Mapping[str, object]", entries).items(): + problems.extend( + ValidationProblem( + ("metadata", key, *problem.loc), problem.message, problem.kind + ) + for problem in validate_json(value) + ) if problems: raise MetadataValidationError(problems) entries_tupled = cast( diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index fae09187a4..a05fe7a430 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -161,6 +161,27 @@ def _check_literal( return [] +def _validate_extension_fields_v3( + doc: Mapping[object, object], + standard_keys: frozenset[str], + *, + additional_reserved_keys: frozenset[str] = frozenset(), +) -> list[ValidationProblem]: + """Validate v3 top-level key types and unknown-field JSON payloads.""" + problems: list[ValidationProblem] = [] + reserved_keys = standard_keys | additional_reserved_keys + for key, value in doc.items(): + if not isinstance(key, str): + problems.append( + ValidationProblem((), f"non-string top-level key {key!r}", "invalid_type") + ) + continue + if key in reserved_keys: + continue + problems.extend(_prefix(key, validate_json(value))) + return problems + + def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: """Return every reason `value` is not a v3 metadata field. @@ -216,7 +237,7 @@ def _is_int_sequence(value: object) -> bool: is not an integer in a metadata document, so booleans are excluded. """ return ( - not isinstance(value, str) + not isinstance(value, (str, bytes, bytearray)) and isinstance(value, Sequence) and all( isinstance(item, int) and not isinstance(item, bool) @@ -267,6 +288,78 @@ def _is_dtype_v2(value: object) -> bool: return True +def _is_canonical_dtype_v2(value: object) -> bool: + """Whether a validated v2 dtype uses the tuple-backed public representation.""" + if isinstance(value, str): + return True + if not isinstance(value, tuple): + return False + for record in cast("tuple[object, ...]", value): + if not isinstance(record, tuple): + return False + fields = cast("tuple[object, ...]", record) + if not _is_canonical_dtype_v2(fields[1]): + return False + if len(fields) == 3 and not isinstance(fields[2], tuple): + return False + return True + + +def _is_canonical_metadata_field_v3(value: object) -> bool: + """Whether a validated v3 metadata field has its declared runtime container type.""" + return isinstance(value, (str, dict)) + + +def _is_canonical_array_metadata_v3(value: object) -> bool: + """Whether a validated v3 array document matches `ArrayMetadataV3` at runtime.""" + if not isinstance(value, dict): + return False + doc = cast("dict[str, object]", value) + if not isinstance(doc["shape"], tuple) or not isinstance(doc["codecs"], tuple): + return False + if "storage_transformers" in doc and not isinstance(doc["storage_transformers"], tuple): + return False + if "dimension_names" in doc and not isinstance(doc["dimension_names"], tuple): + return False + if not all( + _is_canonical_metadata_field_v3(doc[key]) + for key in ("data_type", "chunk_grid", "chunk_key_encoding") + ): + return False + if not all( + _is_canonical_metadata_field_v3(item) for item in cast("tuple[object, ...]", doc["codecs"]) + ): + return False + if "storage_transformers" in doc and not all( + _is_canonical_metadata_field_v3(item) + for item in cast("tuple[object, ...]", doc["storage_transformers"]) + ): + return False + return all( + key in ARRAY_METADATA_STANDARD_KEYS_V3 or isinstance(item, dict) + for key, item in doc.items() + ) + + +def _is_canonical_array_metadata_v2(value: object) -> bool: + """Whether a validated v2 array document matches `ArrayMetadataV2` at runtime.""" + if not isinstance(value, dict): + return False + doc = cast("dict[str, object]", value) + if not isinstance(doc["shape"], tuple) or not isinstance(doc["chunks"], tuple): + return False + if not _is_canonical_dtype_v2(doc["dtype"]): + return False + compressor = doc["compressor"] + if compressor is not None and not isinstance(compressor, dict): + return False + filters = doc["filters"] + return filters is None or ( + isinstance(filters, tuple) + and all(isinstance(item, dict) for item in cast("tuple[object, ...]", filters)) + ) + + def _is_codec_v2(value: object) -> bool: """Whether `value` is shaped like a v2 codec config: a mapping with a string `id`.""" return isinstance(value, Mapping) and isinstance( @@ -274,6 +367,17 @@ def _is_codec_v2(value: object) -> bool: ) +def _validate_codec_v2(value: object) -> list[ValidationProblem]: + """Validate a v2 codec's required shape and JSON-valued configuration.""" + if not _is_codec_v2(value): + return [ + ValidationProblem( + (), "expected a codec configuration with a string 'id'", "invalid_type" + ) + ] + return validate_json(value) + + def _validate_attributes(value: object) -> list[ValidationProblem]: """Validate an `attributes` value: a mapping with string keys. @@ -307,6 +411,11 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V3, doc) + problems.extend( + _validate_extension_fields_v3( + cast("Mapping[object, object]", value), ARRAY_METADATA_STANDARD_KEYS_V3 + ) + ) problems.extend(_check_literal(doc, "zarr_format", 3)) problems.extend(_check_literal(doc, "node_type", "array")) problems.extend(_validate_dim_sequence(doc, "shape")) @@ -357,15 +466,16 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: def is_array_metadata_v3(value: object) -> TypeIs[ArrayMetadataV3]: """Whether `value` is a structurally-valid v3 array metadata document.""" - return not validate_array_metadata_v3(value) + return not validate_array_metadata_v3(value) and _is_canonical_array_metadata_v3(value) def parse_array_metadata_v3(value: object) -> ArrayMetadataV3: """Return `value` narrowed to `ArrayMetadataV3`, or raise `MetadataValidationError`.""" - problems = validate_array_metadata_v3(value) + normalized = arrays_to_tuples(value) + problems = validate_array_metadata_v3(normalized) if problems: raise MetadataValidationError(problems) - return cast(ArrayMetadataV3, value) + return cast(ArrayMetadataV3, normalized) def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: @@ -399,14 +509,8 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: ) if "compressor" in doc: compressor = doc["compressor"] - if compressor is not None and not _is_codec_v2(compressor): - problems.append( - ValidationProblem( - ("compressor",), - "expected null or a codec configuration with a string 'id'", - "invalid_type", - ) - ) + if compressor is not None: + problems.extend(_prefix("compressor", _validate_codec_v2(compressor))) if "filters" in doc: filters = doc["filters"] if filters is not None and ( @@ -421,6 +525,9 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: "invalid_type", ) ) + elif filters is not None: + for index, item in enumerate(cast("Sequence[object]", filters)): + problems.extend(_prefix("filters", _prefix(index, validate_json(item)))) if "dimension_separator" in doc and doc["dimension_separator"] not in (".", "/"): problems.append( ValidationProblem( @@ -438,15 +545,16 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: def is_array_metadata_v2(value: object) -> TypeIs[ArrayMetadataV2]: """Whether `value` is a structurally-valid v2 array metadata document.""" - return not validate_array_metadata_v2(value) + return not validate_array_metadata_v2(value) and _is_canonical_array_metadata_v2(value) def parse_array_metadata_v2(value: object) -> ArrayMetadataV2: """Return `value` narrowed to `ArrayMetadataV2`, or raise `MetadataValidationError`.""" - problems = validate_array_metadata_v2(value) + normalized = arrays_to_tuples(value) + problems = validate_array_metadata_v2(normalized) if problems: raise MetadataValidationError(problems) - return cast(ArrayMetadataV2, value) + return cast(ArrayMetadataV2, normalized) def validate_consolidated_metadata_v3(value: object) -> list[ValidationProblem]: @@ -514,6 +622,13 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) problems: list[ValidationProblem] = _missing_keys(GROUP_METADATA_REQUIRED_KEYS_V3, doc) + problems.extend( + _validate_extension_fields_v3( + cast("Mapping[object, object]", value), + GROUP_METADATA_STANDARD_KEYS_V3, + additional_reserved_keys=frozenset({"consolidated_metadata"}), + ) + ) problems.extend(_check_literal(doc, "zarr_format", 3)) problems.extend(_check_literal(doc, "node_type", "group")) if "attributes" in doc: @@ -587,7 +702,7 @@ def load_store_json(mapping: Mapping[str, bytes], key: str) -> Any: ) try: return json.loads(mapping[key]) - except json.JSONDecodeError as exc: + except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise MetadataValidationError( [ValidationProblem((key,), f"invalid JSON: {exc}", "invalid_json")] ) from exc @@ -598,7 +713,11 @@ def arrays_to_tuples(obj: object) -> object: if isinstance(obj, list): return tuple(arrays_to_tuples(item) for item in cast("list[object]", obj)) if isinstance(obj, dict): - return { - key: arrays_to_tuples(value) for key, value in cast("dict[object, object]", obj).items() + mapping = cast("dict[object, object]", obj) + converted: dict[object, object] = { + key: arrays_to_tuples(value) for key, value in mapping.items() } + if all(converted[key] is value for key, value in mapping.items()): + return cast("object", obj) + return converted return obj diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index ff27923635..a17548634b 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1152,6 +1152,22 @@ def test_from_key_value_invalid_json_raises_metadata_error() -> None: assert [p.kind for p in exc_info.value.problems] == ["invalid_json"] +def test_from_key_value_invalid_utf8_raises_metadata_error() -> None: + """Invalid UTF-8 store bytes use the same invalid_json error channel.""" + with pytest.raises(MetadataValidationError) as exc_info: + ArrayMetadataModelV3.from_key_value({"zarr.json": b"\x80"}) + assert [p.kind for p in exc_info.value.problems] == ["invalid_json"] + + +def test_v2_from_key_value_scalar_root_raises_metadata_error() -> None: + """A scalar .zarray document fails through the unified metadata error channel.""" + with pytest.raises(MetadataValidationError) as exc_info: + ArrayMetadataModelV2.from_key_value({".zarray": b"null"}) + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + ((), "invalid_type") + ] + + def test_from_key_value_missing_key_kind() -> None: """A missing store key surfaces as a missing_key problem at the store-key loc.""" with pytest.raises(MetadataValidationError) as exc_info: @@ -1234,6 +1250,76 @@ def test_configuration_values_must_be_json() -> None: ] +def test_v3_extension_keys_must_be_strings() -> None: + """A non-string top-level key cannot be represented by a v3 document type.""" + doc: dict[object, object] = dict(ArrayMetadataModelV3.create_default().to_json()) + doc[1] = {"must_understand": False} + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + ((), "invalid_type") + ] + + +def test_v3_extension_values_must_be_json() -> None: + """Extension payloads are JSON-checked before a model is constructed.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) + doc["ext"] = {"must_understand": False, "payload": object()} + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + (("ext", "payload"), "invalid_type") + ] + + +def test_v3_json_extension_without_waiver_is_preserved_as_must_understand() -> None: + """A JSON extension without an explicit false waiver remains must-understand.""" + doc = dict(ArrayMetadataModelV3.create_default().to_json()) + doc["ext"] = 1 + model = ArrayMetadataModelV3.from_json(doc) + assert model.extra_fields["ext"] == 1 + assert model.must_understand_fields == {"ext": 1} + + +def test_v2_codec_configuration_values_must_be_json() -> None: + """Non-JSON codec parameters are rejected for compressors and filters.""" + for field, value, expected_loc in ( + ("compressor", {"id": "x", "payload": object()}, ("compressor", "payload")), + ("filters", ({"id": "x", "payload": object()},), ("filters", 0, "payload")), + ): + doc = dict(ArrayMetadataModelV2.create_default().to_json()) + doc[field] = value + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v2(doc)] == [ + (expected_loc, "invalid_type") + ] + + +def test_dimension_sequences_reject_binary_values() -> None: + """Binary buffers are not JSON arrays even though they are integer sequences.""" + v3 = dict(ArrayMetadataModelV3.create_default().to_json()) | {"shape": b"\x02"} + v2 = dict(ArrayMetadataModelV2.create_default().to_json()) | {"chunks": b"\x02"} + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(v3)] == [ + (("shape",), "invalid_type") + ] + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v2(v2)] == [ + (("chunks",), "invalid_type") + ] + + +def test_array_parsers_normalize_json_lists_before_narrowing() -> None: + """Parsers return tuple-backed document types while guards reject raw list forms.""" + v3_raw = json.loads(json.dumps(ArrayMetadataModelV3.create_default(shape=(2,)).to_json())) + v2_raw = json.loads(json.dumps(ArrayMetadataModelV2.create_default(shape=(2,)).to_json())) + + assert validate_array_metadata_v3(v3_raw) == [] + assert validate_array_metadata_v2(v2_raw) == [] + assert not is_array_metadata_v3(v3_raw) + assert not is_array_metadata_v2(v2_raw) + + v3_parsed = parse_array_metadata_v3(v3_raw) + v2_parsed = parse_array_metadata_v2(v2_raw) + assert isinstance(v3_parsed["shape"], tuple) + assert isinstance(v3_parsed["codecs"], tuple) + assert isinstance(v2_parsed["shape"], tuple) + assert isinstance(v2_parsed["chunks"], tuple) + + # --- must_understand partition (spec: MUST fail to open unrecognized fields) -- diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 58b92f5d71..08adb19d40 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -90,6 +90,18 @@ def test_group_v3_bad_attributes() -> None: parse_group_metadata_v3({"zarr_format": 3, "node_type": "group", "attributes": 5}) +def test_group_v3_extension_fields_are_validated() -> None: + """Group extension payloads must be JSON values with a must-understand flag.""" + doc = { + "zarr_format": 3, + "node_type": "group", + "ext": {"must_understand": False, "payload": object()}, + } + assert [(problem.loc, problem.kind) for problem in validate_group_metadata_v3(doc)] == [ + (("ext", "payload"), "invalid_type") + ] + + def test_group_v3_key_value_roundtrip() -> None: """from_key_value(to_key_value()) is the identity for v3 groups.""" model = GroupMetadataModelV3.create_default(attributes={"a": 1}) @@ -279,6 +291,35 @@ def test_consolidated_v2_not_a_mapping() -> None: ConsolidatedMetadataModelV2.from_json([1]) +def test_consolidated_v2_format_literal_enforced() -> None: + """A .zmetadata document must declare consolidated format 1.""" + with pytest.raises(MetadataValidationError) as exc_info: + ConsolidatedMetadataModelV2.from_json({"zarr_consolidated_format": 2, "metadata": {}}) + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + (("zarr_consolidated_format",), "invalid_value") + ] + + +def test_consolidated_v2_metadata_values_must_be_json() -> None: + """Non-JSON values in the flat metadata map are rejected during ingestion.""" + with pytest.raises(MetadataValidationError) as exc_info: + ConsolidatedMetadataModelV2.from_json( + {"zarr_consolidated_format": 1, "metadata": {".zgroup": object()}} + ) + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + (("metadata", ".zgroup"), "invalid_type") + ] + + +def test_group_v2_from_key_value_scalar_root_raises_metadata_error() -> None: + """A scalar .zgroup document fails through the unified metadata error channel.""" + with pytest.raises(MetadataValidationError) as exc_info: + GroupMetadataModelV2.from_key_value({".zgroup": b"null"}) + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + ((), "invalid_type") + ] + + # --- Literal-value enforcement ----------------------------------------------- From a3a4f1961785f667d94a3ede663b05a7061daeb8 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 13:06:30 +0200 Subject: [PATCH 48/70] docs(zarr-metadata): define v3 conformance boundary Assisted-by: Codex:gpt-5 --- ...rr-v3-metadata-model-conformance-design.md | 234 ++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md diff --git a/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md b/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md new file mode 100644 index 0000000000..de352a374d --- /dev/null +++ b/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md @@ -0,0 +1,234 @@ +# Design: Zarr v3 metadata model conformance + +**Status:** Approved design; implementation pending. +**Scope:** `packages/zarr-metadata`, principally the raw v3 metadata types, +the immutable model layer, and their validators and tests. +**Normative authority:** Zarr core protocol v3.1. The behavior of zarrs and +zarr-python is interoperability evidence where the specification leaves an +implementation boundary or where a documented legacy-read exception is +required. + +## Goal + +Make the v3 array and group metadata models conform to the core Zarr v3.1 +document grammar without turning the generic models into interpreters for +individual extensions. + +The generic layer will validate required and optional core fields, JSON value +and container types, fixed literals and core cross-field constraints, the +common extension envelope, the core rules governing `must_understand`, and the +forward-compatibility obligations for unknown top-level fields. + +It will not validate the configuration semantics of a data type, chunk grid, +chunk key encoding, codec, or storage transformer. Those belong to typed +extension models and readers that resolve extension names. + +## Sources and precedence + +The implementation will be checked against: + +1. [Zarr core protocol v3.1](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html), + especially Array metadata, Group metadata, Codecs, and Extensions. +2. [zarrs](https://github.com/zarrs/zarrs), especially its `MetadataV3`, + `AdditionalFieldV3`, array-opening validation, and consolidated-metadata + extension handling. +3. Existing zarr-python metadata emitted in the wild, but only for explicit + tolerant-read paths. Legacy data does not redefine what this package emits + or calls conformant. + +If these disagree, the current v3.1 specification wins. An interoperability +exception must be narrow, documented as non-conformant input, normalized on +read, and never emitted. + +## Chosen approach + +Evolve the existing normalized `NamedConfigModelV3` instead of introducing a +new hierarchy of role-specific extension classes or preserving every source +JSON spelling. + +This keeps the public shape small and minimizes compatibility churn. +Validation remains context-aware: the same extension envelope is used at all +extension points, while the array-document validator supplies the few rules +that vary by field. + +Two alternatives were rejected: + +- A spelling-preserving model would retain shorthand versus object form and + absent versus empty configuration. That fidelity is not needed by this + semantic model and would substantially change its API. +- Separate required/optional extension classes would encode + `must_understand` restrictions in types, but would duplicate behavior and + force callers to use different classes for otherwise identical envelopes. + +## Extension envelope + +### Raw type + +`NamedConfigV3` will describe the v3.1 object form: + +- required `name: str`; +- optional `configuration: Mapping[str, JSONValue]`; and +- optional `must_understand: bool`. + +`MetadataV3` remains the union of a shorthand name string and this object +form. + +The object form accepts only those three members. Extension-owned fields must +be placed inside `configuration`. Rejecting other envelope members matches +zarrs' `deny_unknown_fields` behavior and prevents silently discarding data +during normalization. + +### Normalized model + +`NamedConfigModelV3` will hold: + +- `name: str`; +- `configuration: dict[str, JSONValue]`, normalized to an empty mapping when + absent; and +- `must_understand: bool`, normalized to `True` when absent or when the input + is a shorthand string. + +`to_json()` will continue emitting the canonical object form with a +configuration mapping. It will omit `must_understand` when true and emit +`"must_understand": false` when false. Thus the model is semantically +lossless, but deliberately not spelling-preserving. + +### Name validation + +Names will be checked syntactically without attempting to query or freeze the +external extension registry: + +- registered-name syntax is `^[a-z][a-z0-9-_.]+$`; and +- legacy URI names remain accepted for v3.0 compatibility. + +Registry membership is not a structural property that an extension-agnostic +offline model can determine. Extension resolution remains a reader concern. + +## Context-sensitive `must_understand` rules + +An omitted `must_understand` value and every shorthand name mean `True`. + +The array validator will reject `must_understand=False` for `data_type`, +`chunk_grid`, and `chunk_key_encoding`. It will permit false for individual +codecs and storage transformers. + +This validation is structural only. The model does not decide whether an +implementation recognizes a name and does not remove or skip optional +extensions. A reader that resolves extensions owns that decision. + +## Core array and group constraints + +The generic array model will enforce the core constraints that do not require +interpreting an extension: + +- `zarr_format` is exactly `3` and `node_type` is exactly `"array"`; +- `shape` contains non-negative JSON integers (booleans excluded); +- all mandatory fields are present; +- `fill_value` is JSON, while its data-type-dependent meaning remains opaque; +- `codecs` is a non-empty sequence of valid extension envelopes; +- `storage_transformers`, when present, is a sequence of valid envelopes; +- `attributes`, when present, is a string-keyed JSON object; and +- `dimension_names`, when present, contains strings or null and has the same + length as `shape`. + +The model will not enforce regular-grid rank or positive chunk lengths, +data-type-specific fill values, codec kinds or ordering, or storage-transformer +behavior. These rules require resolving an extension name and therefore live +outside the generic model. + +The generic group model will enforce the corresponding core document rules: +fixed literals, required fields, JSON attributes, string top-level keys, and +unknown-field handling. + +## Unknown top-level fields + +Unknown array and group members will be represented as arbitrary `JSONValue`, +not as a TypedDict that requires an object. + +For reader obligations: + +- an object whose `must_understand` member is the literal JSON boolean `false` + is explicitly waivable; and +- every other value, including scalars, arrays, objects with no such member, + and objects with a non-boolean value, implicitly requires understanding. + +This matches the v3.1 implicit-true rule and zarrs' `AdditionalFieldV3` +behavior. The raw and model annotations, `is_*` guards, parsers, and +`must_understand_fields` property must all agree on this representation. + +The model itself will not fail merely because such a field requires +understanding. It has no extension registry. It exposes the partition so the +reader can fail when a required field is unrecognized. + +## Consolidated metadata + +Inline consolidated metadata is a known interoperability extension rather +than a core v3.1 group field. Its dedicated raw and model types will remain, +and group parsing may continue recognizing it explicitly. + +The standard emitted form is an object containing `kind`, `metadata`, and a +`must_understand` marker. The extension model validates its own payload and +recursively validates embedded v3 nodes. This special handling must not cause +the generic unknown-field type to become object-only. + +Historical zarr-python versions wrote `"consolidated_metadata": null`. +zarrs also accepts this input as a compatibility hotfix. `from_json()` will +continue repairing that exact value to absence, and `to_json()` will never +write it. Documentation and tests will identify this as a tolerant-read +exception, not valid core metadata. + +## Errors and normalization + +All newly enforced constraints use the existing aggregated +`MetadataValidationError`/`ValidationProblem` mechanism. Locations must point +to the envelope member or array entry that failed, and problem kinds remain +machine-readable. + +Parsing still recursively converts JSON arrays to the tuple-backed public +representation. Validation and type guards must agree on canonical runtime +containers after normalization. + +No constructor-wide semantic validation will be added to `update()`; +`from_json()` remains the validated ingestion boundary. Direct construction +and `update()` continue to support building or repairing intermediate models. + +## Public API compatibility + +The existing class names remain. The intended public changes are additive or +corrections to annotations that did not match accepted runtime data: + +- `NamedConfigV3` gains optional `must_understand`; +- `NamedConfigModelV3` gains a defaulted `must_understand` field; and +- v3 additional-field annotations widen to arbitrary JSON. + +Documents currently accepted only because an extension envelope contains +unknown members, an invalid name, a forbidden false `must_understand`, or an +empty codec list will become validation errors. Those documents violate the +agreed core grammar; accepting them is not a compatibility contract to retain. + +## Test strategy + +Implementation will be test-first and cover: + +1. shorthand and object normalization, including implicit true; +2. explicit true/false round trips; +3. invalid `must_understand` types and unknown envelope members; +4. registered and legacy URI name forms plus invalid names; +5. false `must_understand` rejected at mandatory extension points; +6. false accepted for codecs and storage transformers; +7. empty codecs rejected without interpreting non-empty pipelines; +8. arbitrary JSON unknown top-level fields and their obligation partition; +9. raw type, parser, type-guard, model, and serializer agreement; +10. consolidated-metadata parsing and the legacy-null read repair; and +11. representative metadata serialized by zarrs. + +The package test suite, formatting, linting, and static type checking will run +before the implementation commit. + +## Completion criteria + +The work is complete when every normative core-document rule in scope has an +explicit validator or a documented extension-owned boundary; raw annotations +and runtime behavior agree; tests demonstrate `must_understand` at every +extension point; zarrs-compatible metadata round-trips semantically; and the +package tests and quality checks pass. From 55d329769827e453230b8849669b043bb8867b5e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 14:54:22 +0200 Subject: [PATCH 49/70] fix(zarr-metadata): preserve v3 extension obligations Assisted-by: Codex:gpt-5 --- .../src/zarr_metadata/_common.py | 1 + .../src/zarr_metadata/model/_array.py | 28 +++++++--- .../src/zarr_metadata/model/_validation.py | 28 +++++++++- .../src/zarr_metadata/pydantic.py | 2 +- .../zarr-metadata/tests/model/test_array.py | 54 +++++++++++++++---- .../tests/model/test_pydantic_module.py | 11 +++- 6 files changed, 102 insertions(+), 22 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index b335cd0bd6..5c530fcebf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -34,3 +34,4 @@ class NamedConfigV3(TypedDict): name: str configuration: NotRequired[Mapping[str, JSONValue]] + must_understand: NotRequired[bool] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index aee4f80fa8..ed2767c851 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -23,7 +23,7 @@ ) if TYPE_CHECKING: - from zarr_metadata._common import JSONValue + from zarr_metadata._common import JSONValue, NamedConfigV3 from zarr_metadata.v2.array import ( ArrayDimensionSeparatorV2, ArrayMetadataV2, @@ -46,31 +46,43 @@ @dataclass(frozen=True, slots=True, kw_only=True) class NamedConfigModelV3: - """A v3 metadata field in normalized form: a name plus a configuration. + """A normalized v3 metadata field with its reader obligation. - This is the in-memory model of `MetadataV3` (a bare name string or a - `{name, configuration}` mapping): the bare-name and missing-configuration - forms normalize to an empty configuration. + Bare names and missing configurations normalize to an empty configuration. + Bare names and missing `must_understand` members normalize to the spec's + implicit `True` value. """ name: str configuration: dict[str, JSONValue] + must_understand: bool = True def to_json(self) -> MetadataV3: - return {"name": self.name, "configuration": self.configuration} + if not self.configuration and self.must_understand: + return self.name + out: NamedConfigV3 = {"name": self.name} + if self.configuration: + out["configuration"] = self.configuration + if not self.must_understand: + out["must_understand"] = False + return out @classmethod def from_json(cls, data: object) -> NamedConfigModelV3: field = parse_metadata_field_v3(data) if isinstance(field, str): - return cls(name=field, configuration={}) + return cls(name=field, configuration={}, must_understand=True) # Sound cast: parse_metadata_field_v3 checked the configuration is a # string-keyed mapping of JSON values; arrays_to_tuples only converts # lists to tuples within that shape. configuration = cast( "dict[str, JSONValue]", arrays_to_tuples(dict(field.get("configuration", {}))) ) - return cls(name=field["name"], configuration=configuration) + return cls( + name=field["name"], + configuration=configuration, + must_understand=field.get("must_understand", True), + ) MetadataFieldModelV3: TypeAlias = NamedConfigModelV3 diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index a05fe7a430..1c8834d3f2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -182,7 +182,9 @@ def _validate_extension_fields_v3( return problems -def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: +def validate_metadata_field_v3( + value: object, *, allow_must_understand_false: bool = True +) -> list[ValidationProblem]: """Return every reason `value` is not a v3 metadata field. A metadata field is a bare name string or a `{name, configuration}` mapping. @@ -199,6 +201,16 @@ def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: ] field = cast("Mapping[object, object]", value) problems: list[ValidationProblem] = [] + allowed_keys = frozenset({"name", "configuration", "must_understand"}) + for key in field: + if not isinstance(key, str): + problems.append( + ValidationProblem((), f"non-string metadata field key {key!r}", "invalid_type") + ) + elif key not in allowed_keys: + problems.append( + ValidationProblem((key,), "unexpected metadata field member", "invalid_value") + ) if not isinstance(field.get("name"), str): problems.append(ValidationProblem(("name",), "expected a string name", "invalid_type")) if "configuration" in field: @@ -214,6 +226,20 @@ def validate_metadata_field_v3(value: object) -> list[ValidationProblem]: else: for key, item in cast("Mapping[str, object]", configuration).items(): problems.extend(_prefix("configuration", _prefix(key, validate_json(item)))) + if "must_understand" in field: + must_understand = field["must_understand"] + if not isinstance(must_understand, bool): + problems.append( + ValidationProblem(("must_understand",), "expected a boolean", "invalid_type") + ) + elif not allow_must_understand_false and not must_understand: + problems.append( + ValidationProblem( + ("must_understand",), + "false is not supported at this extension point", + "invalid_value", + ) + ) return problems diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index 3eada76316..a19e192eb4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -115,7 +115,7 @@ def coerce(value: object) -> _M: MetadataFieldV3 = Annotated[ InstanceOf[NamedConfigModelV3], BeforeValidator(_coerce_to(NamedConfigModelV3, NamedConfigModelV3.from_json)), - PlainSerializer(NamedConfigModelV3.to_json, return_type=dict), + PlainSerializer(NamedConfigModelV3.to_json, return_type=str | dict), WithJsonSchema(_FIELD_SCHEMA | {"title": "MetadataFieldV3"}), ] """Field type for one v3 metadata field (bare name string or name + configuration).""" diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index a17548634b..dc235fb2dd 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -132,18 +132,24 @@ def test_string_nan_fill_value_roundtrips() -> None: ), Expect( NamedConfigModelV3(name="bytes", configuration={}), - {"name": "bytes", "configuration": {}}, - id="without-configuration", + "bytes", + id="empty-configuration-shorthand", ), ] @pytest.mark.parametrize("case", ZARR_TO_JSON_CASES, ids=lambda c: c.id) -def test_zarr_metadata_v3_to_json(case: Expect[NamedConfigModelV3, dict[str, object]]) -> None: - """NamedConfigModelV3.to_json emits the canonical object form.""" +def test_zarr_metadata_v3_to_json(case: Expect[NamedConfigModelV3, object]) -> None: + """NamedConfigModelV3.to_json emits the canonical extension form.""" assert case.input.to_json() == case.output +def test_zarr_metadata_v3_to_json_preserves_false_obligation() -> None: + """An empty optional extension stays an object so false is not lost.""" + model = NamedConfigModelV3(name="optional", configuration={}, must_understand=False) + assert model.to_json() == {"name": "optional", "must_understand": False} + + # --- NamedConfigModelV3.from_json ----------------------------------------------- ZARR_FROM_JSON_CASES = [ @@ -167,6 +173,12 @@ def test_zarr_metadata_v3_from_json(case: Expect[object, NamedConfigModelV3]) -> assert NamedConfigModelV3.from_json(case.input) == case.output +def test_zarr_metadata_v3_from_json_preserves_false_obligation() -> None: + """Explicit false is represented on the normalized model.""" + model = NamedConfigModelV3.from_json({"name": "optional", "must_understand": False}) + assert model.must_understand is False + + # --- V3 baseline ----------------------------------------------------------- @@ -181,10 +193,10 @@ def test_v3_to_json_emits_canonical_document() -> None: "node_type": "array", "shape": (10,), "fill_value": 0, - "data_type": {"name": "int32", "configuration": {}}, + "data_type": "int32", "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (10,)}}, - "codecs": ({"name": "bytes", "configuration": {}},), - "chunk_key_encoding": {"name": "default", "configuration": {}}, + "codecs": ("bytes",), + "chunk_key_encoding": "default", } @@ -233,7 +245,7 @@ def test_v3_single_storage_transformer_included() -> None: out: dict[str, object] = dict( ArrayMetadataModelV3.create_default(storage_transformers=(st,)).to_json() ) - assert out["storage_transformers"] == ({"name": "some_transformer", "configuration": {}},) + assert out["storage_transformers"] == ("some_transformer",) def test_v3_no_storage_transformers_omitted() -> None: @@ -639,11 +651,31 @@ def test_v2_roundtrip_json_model_json() -> None: def test_v3_parser_accepts_bare_string_data_type() -> None: """V3 from_json accepts a bare-string data_type and re-serializes it canonically.""" doc = ArrayMetadataModelV3.create_default().to_json() - doc["data_type"] = "int32" # bare-string form, not canonical object form + doc["data_type"] = "int32" model = ArrayMetadataModelV3.from_json(doc) - # parses correctly, re-serializes to canonical object form assert model.data_type == NamedConfigModelV3(name="int32", configuration={}) - assert model.to_json()["data_type"] == {"name": "int32", "configuration": {}} + assert model.to_json()["data_type"] == "int32" + + +@pytest.mark.parametrize("name", ["bytes", "ANY string", "urn:example:codec"]) +def test_metadata_field_accepts_any_string_name(name: str) -> None: + """The structural layer checks the name type, not syntax or registration.""" + assert validate_metadata_field_v3({"name": name}) == [] + + +@pytest.mark.parametrize("value", [0, 1, "false", None]) +def test_metadata_field_must_understand_must_be_boolean(value: object) -> None: + """must_understand is a JSON boolean, not a truthy scalar.""" + problems = validate_metadata_field_v3({"name": "x", "must_understand": value}) + assert [(problem.loc, problem.kind) for problem in problems] == [ + (("must_understand",), "invalid_type") + ] + + +def test_metadata_field_rejects_unknown_envelope_member() -> None: + """Unknown envelope keys cannot be silently discarded during normalization.""" + problems = validate_metadata_field_v3({"name": "x", "typo": 1}) + assert [(problem.loc, problem.kind) for problem in problems] == [(("typo",), "invalid_value")] def test_v2_roundtrip_with_compressor_and_filters() -> None: diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 81b9b962af..9ddbc86ded 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -65,7 +65,7 @@ def test_field_type_validates_and_dumps_canonically( model = adapter.validate_python(doc) assert type(model) is model_cls assert adapter.validate_python(model) is model - assert adapter.dump_python(model) == dict(model.to_json()) + assert adapter.dump_python(model) == model.to_json() def test_core_instances_interoperate() -> None: @@ -115,6 +115,15 @@ class Manifest(BaseModel): assert Manifest.model_validate_json(manifest.model_dump_json()) == manifest +def test_metadata_field_serializes_shorthand_and_false_object() -> None: + """The optional integration exposes the core model's canonical extension form.""" + adapter = TypeAdapter(zmp.MetadataFieldV3) + assert adapter.dump_python(adapter.validate_python({"name": "bytes"})) == "bytes" + assert adapter.dump_python( + adapter.validate_python({"name": "optional", "must_understand": False}) + ) == {"name": "optional", "must_understand": False} + + def test_core_package_does_not_import_pydantic() -> None: """Importing zarr_metadata (in a fresh interpreter) must not import pydantic: the integration is opt-in via zarr_metadata.pydantic.""" From a2167f1b219a51c7a662b5a23f634fdfa1701faf Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 14:55:43 +0200 Subject: [PATCH 50/70] fix(zarr-metadata): enforce v3 core extension rules Assisted-by: Codex:gpt-5 --- .../src/zarr_metadata/model/_validation.py | 13 ++++++++- .../zarr-metadata/tests/model/test_array.py | 27 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 1c8834d3f2..faaaa19511 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -449,13 +449,24 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) for key in ("data_type", "chunk_grid", "chunk_key_encoding"): if key in doc: - problems.extend(_prefix(key, validate_metadata_field_v3(doc[key]))) + problems.extend( + _prefix( + key, + validate_metadata_field_v3(doc[key], allow_must_understand_false=False), + ) + ) for key in ("codecs", "storage_transformers"): if key in doc: entries = doc[key] if isinstance(entries, str) or not isinstance(entries, Sequence): problems.append(ValidationProblem((key,), "expected a sequence", "invalid_type")) else: + if key == "codecs" and len(entries) == 0: + problems.append( + ValidationProblem( + ("codecs",), "expected at least one codec", "invalid_value" + ) + ) for index, entry in enumerate(cast("Sequence[object]", entries)): problems.extend(_prefix(key, _prefix(index, validate_metadata_field_v3(entry)))) if "attributes" in doc: diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index dc235fb2dd..fa732c6786 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -678,6 +678,33 @@ def test_metadata_field_rejects_unknown_envelope_member() -> None: assert [(problem.loc, problem.kind) for problem in problems] == [(("typo",), "invalid_value")] +@pytest.mark.parametrize("field", ["codecs", "storage_transformers"]) +def test_optional_extension_points_allow_must_understand_false(field: str) -> None: + """Codecs and storage transformers may be explicitly ignorable.""" + doc: dict[str, object] = dict(ArrayMetadataModelV3.create_default().to_json()) + doc[field] = ({"name": "optional", "must_understand": False},) + assert validate_array_metadata_v3(doc) == [] + + +@pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) +def test_required_extension_points_reject_must_understand_false(field: str) -> None: + """Core extension points needed to locate or decode chunks cannot be ignored.""" + doc: dict[str, object] = dict(ArrayMetadataModelV3.create_default().to_json()) + doc[field] = {"name": "optional", "must_understand": False} + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + ((field, "must_understand"), "invalid_value") + ] + + +def test_v3_codecs_cannot_be_empty() -> None: + """The core document requires at least one array-to-bytes codec.""" + doc: dict[str, object] = dict(ArrayMetadataModelV3.create_default().to_json()) + doc["codecs"] = () + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + (("codecs",), "invalid_value") + ] + + def test_v2_roundtrip_with_compressor_and_filters() -> None: # Non-None compressor/filters must round-trip; extra assertion on .compressor. """A v2 model with non-None compressor and filters round-trips.""" From 5bc85320cfe1cb7619de6e0f48b71979de19daaf Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 14:57:47 +0200 Subject: [PATCH 51/70] fix(zarr-metadata): align v3 additional field types Assisted-by: Codex:gpt-5 --- .../src/zarr_metadata/model/_validation.py | 7 +--- .../src/zarr_metadata/v3/array.py | 40 ++++--------------- .../src/zarr_metadata/v3/group.py | 2 +- .../zarr-metadata/tests/model/test_array.py | 2 + .../zarr-metadata/tests/model/test_group.py | 8 ++++ 5 files changed, 20 insertions(+), 39 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index faaaa19511..491eb30aca 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -356,14 +356,9 @@ def _is_canonical_array_metadata_v3(value: object) -> bool: _is_canonical_metadata_field_v3(item) for item in cast("tuple[object, ...]", doc["codecs"]) ): return False - if "storage_transformers" in doc and not all( + return "storage_transformers" not in doc or all( _is_canonical_metadata_field_v3(item) for item in cast("tuple[object, ...]", doc["storage_transformers"]) - ): - return False - return all( - key in ARRAY_METADATA_STANDARD_KEYS_V3 or isinstance(item, dict) - for key, item in doc.items() ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/array.py b/packages/zarr-metadata/src/zarr_metadata/v3/array.py index a3fec24a6c..3226707665 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/array.py @@ -1,51 +1,27 @@ """Zarr v3 array metadata types.""" from collections.abc import Mapping -from typing import Literal, NotRequired +from typing import Literal, NotRequired, TypeAlias from typing_extensions import TypedDict from zarr_metadata._common import JSONValue from zarr_metadata.v3._common import MetadataV3 +ExtensionFieldV3: TypeAlias = JSONValue +"""The JSON value of an unknown top-level v3 metadata field. -class ExtensionFieldV3(TypedDict, extra_items=JSONValue): - """ - Required shape of any extension field on a v3 metadata document. - - The Zarr v3 spec permits extra keys on array and group metadata - documents, provided each value is an object with a `must_understand` - boolean key. This TypedDict captures that constraint and is used as - the `extra_items=` parameter on `ArrayMetadataV3` and `GroupMetadataV3`. - - `must_understand` is typed as `bool` rather than `Literal[False]` so - that applications which understand a particular extension can produce - or consume it with `must_understand: true` (signalling that readers - that don't recognize the extension MUST refuse to open the document). - The common case is still `false`, signalling that unknown readers may - safely ignore the field. - - Spec interpretation: this type follows the original Zarr v3.0 reading - of the spec, under which any object with a `must_understand` key is a - valid extension field. The v3.1 spec rewrite added language requiring - extension fields to also include a `name: str` key (the "Extension - definition" form). Under the strict v3.1 reading, real-world extension - fields written by zarr-python and zarrs (notably `consolidated_metadata`, - which has no `name` field) are out of spec. The community consensus at - the time of writing is that this is a regression to be reverted; this - package models the v3.0 / pre-revert interpretation. See - https://github.com/zarr-developers/zarr-specs/issues/371 for the - ongoing discussion. - """ - - must_understand: bool +An object carrying the literal member `must_understand: false` may be ignored. +Every other JSON shape implicitly requires understanding; recognition itself +belongs to the reader rather than this structural type. +""" class ArrayMetadataV3(TypedDict, extra_items=ExtensionFieldV3): """ Zarr v3 array metadata document (the `zarr.json` content for an array). - Extra keys are permitted if they conform to `ExtensionFieldV3`. + Extra keys may contain arbitrary JSON values. See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#array-metadata """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/group.py b/packages/zarr-metadata/src/zarr_metadata/v3/group.py index be990b1ae7..0593896f37 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/group.py @@ -16,7 +16,7 @@ class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): """ Zarr v3 group metadata document (the `zarr.json` content for a group). - Extra keys are permitted if they conform to `ExtensionFieldV3`. + Extra keys may contain arbitrary JSON values. See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#group-metadata """ diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index fa732c6786..f6b08fdcf1 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1331,6 +1331,8 @@ def test_v3_json_extension_without_waiver_is_preserved_as_must_understand() -> N """A JSON extension without an explicit false waiver remains must-understand.""" doc = dict(ArrayMetadataModelV3.create_default().to_json()) doc["ext"] = 1 + parsed = parse_array_metadata_v3(doc) + assert is_array_metadata_v3(parsed) model = ArrayMetadataModelV3.from_json(doc) assert model.extra_fields["ext"] == 1 assert model.must_understand_fields == {"ext": 1} diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 08adb19d40..fbbcbcb875 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -58,6 +58,14 @@ def test_group_v3_extra_fields_roundtrip() -> None: assert model.to_json() == doc +def test_group_v3_json_extra_field_roundtrips_as_must_understand() -> None: + """A non-object extra field is preserved and implicitly requires understanding.""" + doc = {"zarr_format": 3, "node_type": "group", "ext": [1, 2]} + model = GroupMetadataModelV3.from_json(doc) + assert model.to_json()["ext"] == (1, 2) + assert model.must_understand_fields == {"ext": (1, 2)} + + def test_group_v3_extra_fields_overlap_rejected() -> None: """Constructing a v3 group model with extra_fields shadowing a standard key raises.""" with pytest.raises(ValueError, match="Extra fields"): From fbaf9ad959a0f6de40efe71e27be447deee623f5 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 15:04:08 +0200 Subject: [PATCH 52/70] chore(zarr-metadata): finalize v3 model conformance Assisted-by: Codex:gpt-5 --- ...7-22-zarr-v3-metadata-model-conformance.md | 398 ++++++++++++++++++ ...rr-v3-metadata-model-conformance-design.md | 30 +- .../src/zarr_metadata/_common.py | 4 +- .../src/zarr_metadata/model/_array.py | 14 +- .../src/zarr_metadata/model/_group.py | 8 +- .../src/zarr_metadata/model/_validation.py | 7 +- .../src/zarr_metadata/pydantic.py | 2 +- .../src/zarr_metadata/v3/_common.py | 2 +- .../src/zarr_metadata/v3/consolidated.py | 12 +- .../tests/model/test_pydantic.py | 14 +- 10 files changed, 446 insertions(+), 45 deletions(-) create mode 100644 docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md diff --git a/docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md b/docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md new file mode 100644 index 0000000000..98d1cd896c --- /dev/null +++ b/docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md @@ -0,0 +1,398 @@ +# Zarr v3 Metadata Model Conformance Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the extension-agnostic v3 metadata models preserve Zarr 3.1 extension envelopes, enforce core envelope/cardinality rules, and keep raw types, parsers, guards, and serializers consistent. + +**Architecture:** Keep one normalized `NamedConfigModelV3` for every extension point. It parses shorthand and object forms, stores opaque configuration plus `must_understand`, and applies one field-independent shorthand heuristic. The array validator supplies only core rules that vary by extension point; it never resolves an extension name or configuration. + +**Tech Stack:** Python 3.11+, frozen dataclasses, `TypedDict`/PEP 728, pytest, Ruff, Pyright, and optional Pydantic v2 integration. + +## Global Constraints + +- Zarr v3.1 is normative; zarrs is interoperability evidence. +- Validate extension names only as strings; do not check syntax or registry membership. +- Do not interpret extension configurations, fill values, or codec kinds/order. +- Reject `must_understand=False` for data types, chunk grids, and chunk-key encodings. +- Require a non-empty codec list. +- Preserve the `consolidated_metadata: null` tolerant-read repair and never emit null. +- Observe each new behavioral test fail before changing production code. +- Commits must be conventional and include `Assisted-by: Codex:gpt-5`. + +--- + +### Task 1: Preserve and canonically serialize extension envelopes + +**Files:** +- Modify: `packages/zarr-metadata/tests/model/test_array.py` +- Modify: `packages/zarr-metadata/tests/model/test_pydantic_module.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/_common.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_array.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_validation.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/pydantic.py` + +**Interfaces:** +- Produces: `NamedConfigV3` with optional `must_understand: bool`. +- Produces: `NamedConfigModelV3(name: str, configuration: dict[str, JSONValue], must_understand: bool = True)`. +- Produces: `validate_metadata_field_v3(value: object, *, allow_must_understand_false: bool = True) -> list[ValidationProblem]`. + +- [x] **Step 1: Write failing normalized-model tests** + +Update the existing case tables and canonical-document expectation: + +```python +ZARR_TO_JSON_CASES = [ + Expect( + NamedConfigModelV3(name="regular", configuration={"chunk_shape": [1]}), + {"name": "regular", "configuration": {"chunk_shape": [1]}}, + id="with-configuration", + ), + Expect( + NamedConfigModelV3(name="bytes", configuration={}), + "bytes", + id="empty-configuration-shorthand", + ), + Expect( + NamedConfigModelV3(name="optional", configuration={}, must_understand=False), + {"name": "optional", "must_understand": False}, + id="false-obligation-needs-object", + ), +] +``` + +Add `from_json` cases for implicit true and explicit false. Expect the default +array document to contain `"uint8"`, `("bytes",)`, and `"default"`, while its +configured regular grid remains an object. + +- [x] **Step 2: Run the focused tests and verify RED** + +```bash +packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ + -k 'zarr_metadata_v3 or canonical_document or explicit_false' -q +``` + +Expected: empty configurations remain objects, the constructor lacks +`must_understand`, or explicit false is lost. + +- [x] **Step 3: Write failing envelope-validation tests** + +```python +@pytest.mark.parametrize("name", ["bytes", "ANY string", "urn:example:codec"]) +def test_metadata_field_accepts_any_string_name(name: str) -> None: + assert validate_metadata_field_v3({"name": name}) == [] + + +@pytest.mark.parametrize("value", [0, 1, "false", None]) +def test_metadata_field_must_understand_must_be_boolean(value: object) -> None: + problems = validate_metadata_field_v3({"name": "x", "must_understand": value}) + assert [(problem.loc, problem.kind) for problem in problems] == [ + (("must_understand",), "invalid_type") + ] + + +def test_metadata_field_rejects_unknown_envelope_member() -> None: + problems = validate_metadata_field_v3({"name": "x", "typo": 1}) + assert [(problem.loc, problem.kind) for problem in problems] == [ + (("typo",), "invalid_value") + ] +``` + +- [x] **Step 4: Run the envelope tests and verify RED** + +```bash +packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ + -k 'metadata_field_accepts or must_understand_must_be or unknown_envelope' -q +``` + +Expected: invalid obligation values and the unknown member are accepted before the fix. + +- [x] **Step 5: Implement the raw type, validator, and normalized model** + +```python +class NamedConfigV3(TypedDict): + name: str + configuration: NotRequired[Mapping[str, JSONValue]] + must_understand: NotRequired[bool] +``` + +Reject envelope keys outside `name`, `configuration`, and `must_understand`; +require a real boolean for the last member; keep all string names valid. + +```python +@dataclass(frozen=True, slots=True, kw_only=True) +class NamedConfigModelV3: + name: str + configuration: dict[str, JSONValue] + must_understand: bool = True + + def to_json(self) -> MetadataV3: + if not self.configuration and self.must_understand: + return self.name + out: NamedConfigV3 = {"name": self.name} + if self.configuration: + out["configuration"] = self.configuration + if not self.must_understand: + out["must_understand"] = False + return out +``` + +`from_json` sets true for shorthand/absence and preserves explicit false. + +- [x] **Step 6: Update and test the Pydantic serializer** + +Change `MetadataFieldV3`'s serializer return type from `dict` to `str | dict`. + +```python +def test_metadata_field_serializes_shorthand_and_false_object() -> None: + adapter = TypeAdapter(zmp.MetadataFieldV3) + assert adapter.dump_python(adapter.validate_python({"name": "bytes"})) == "bytes" + assert adapter.dump_python( + adapter.validate_python({"name": "optional", "must_understand": False}) + ) == {"name": "optional", "must_understand": False} +``` + +- [x] **Step 7: Run Task 1 tests and verify GREEN** + +```bash +packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ + packages/zarr-metadata/tests/model/test_pydantic_module.py -q +``` + +Expected: both files pass. + +- [x] **Step 8: Commit Task 1** + +```bash +git add packages/zarr-metadata/src/zarr_metadata/_common.py \ + packages/zarr-metadata/src/zarr_metadata/model/_array.py \ + packages/zarr-metadata/src/zarr_metadata/model/_validation.py \ + packages/zarr-metadata/src/zarr_metadata/pydantic.py \ + packages/zarr-metadata/tests/model/test_array.py \ + packages/zarr-metadata/tests/model/test_pydantic_module.py +git commit -m "fix(zarr-metadata): preserve v3 extension obligations" \ + -m "Assisted-by: Codex:gpt-5" +``` + +### Task 2: Enforce context-sensitive core array rules + +**Files:** +- Modify: `packages/zarr-metadata/tests/model/test_array.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_validation.py` + +**Interfaces:** +- Consumes: `validate_metadata_field_v3(..., allow_must_understand_false=...)`. +- Produces: rejection of forbidden false obligations and empty codecs. + +- [x] **Step 1: Write failing context and cardinality tests** + +```python +@pytest.mark.parametrize("field", ["codecs", "storage_transformers"]) +def test_optional_extension_points_allow_must_understand_false(field: str) -> None: + doc = dict(ArrayMetadataModelV3.create_default().to_json()) + doc[field] = ({"name": "optional", "must_understand": False},) + assert validate_array_metadata_v3(doc) == [] + + +@pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) +def test_required_extension_points_reject_must_understand_false(field: str) -> None: + doc = dict(ArrayMetadataModelV3.create_default().to_json()) + doc[field] = {"name": "optional", "must_understand": False} + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + ((field, "must_understand"), "invalid_value") + ] + + +def test_v3_codecs_cannot_be_empty() -> None: + doc = dict(ArrayMetadataModelV3.create_default().to_json()) + doc["codecs"] = () + assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ + (("codecs",), "invalid_value") + ] +``` + +- [x] **Step 2: Run Task 2 tests and verify RED** + +```bash +packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ + -k 'optional_extension_points or required_extension_points or codecs_cannot' -q +``` + +Expected: required points and empty codecs are accepted before the fix. + +- [x] **Step 3: Implement the context rules** + +Call `validate_metadata_field_v3(..., allow_must_understand_false=False)` for +`data_type`, `chunk_grid`, and `chunk_key_encoding`. Leave codecs and storage +transformers at the default. After establishing that `codecs` is a sequence, +add one field-level `invalid_value` problem when it is empty. + +- [x] **Step 4: Run the complete array model tests and verify GREEN** + +```bash +packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py -q +``` + +Expected: the file passes. + +- [x] **Step 5: Commit Task 2** + +```bash +git add packages/zarr-metadata/src/zarr_metadata/model/_validation.py \ + packages/zarr-metadata/tests/model/test_array.py +git commit -m "fix(zarr-metadata): enforce v3 core extension rules" \ + -m "Assisted-by: Codex:gpt-5" +``` + +### Task 3: Align additional-field types, guards, and models + +**Files:** +- Modify: `packages/zarr-metadata/tests/model/test_array.py` +- Modify: `packages/zarr-metadata/tests/model/test_group.py` +- Modify: `packages/zarr-metadata/tests/test_partial_equivalence.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/array.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/group.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_array.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_group.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_validation.py` + +**Interfaces:** +- Produces: public `ExtensionFieldV3` alias for arbitrary `JSONValue`. +- Produces: `extra_fields: dict[str, JSONValue]` and matching guards/parsers. + +- [x] **Step 1: Write failing parser/guard agreement tests** + +```python +def test_v3_scalar_extra_field_agrees_across_parser_guard_and_model() -> None: + raw = dict(ArrayMetadataModelV3.create_default().to_json()) | {"ext": 1} + parsed = parse_array_metadata_v3(raw) + assert is_array_metadata_v3(parsed) + model = ArrayMetadataModelV3.from_json(raw) + assert model.extra_fields == {"ext": 1} + assert model.must_understand_fields == {"ext": 1} + + +def test_group_v3_scalar_extra_field_roundtrips_as_must_understand() -> None: + raw = {"zarr_format": 3, "node_type": "group", "ext": [1, 2]} + model = GroupMetadataModelV3.from_json(raw) + assert model.to_json()["ext"] == (1, 2) + assert model.must_understand_fields == {"ext": (1, 2)} +``` + +- [x] **Step 2: Run the agreement tests and verify RED** + +```bash +packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ + packages/zarr-metadata/tests/model/test_group.py -k 'scalar_extra_field' -q +``` + +Expected: the canonical array guard rejects the parsed scalar extra field. + +- [x] **Step 3: Correct raw/model annotations and the guard** + +Replace the object-shaped `ExtensionFieldV3` TypedDict with a public alias to +`JSONValue`; use it as `extra_items` on full and partial array/group types. +Update model partials, dataclass fields, helper signatures, and casts to +`dict[str, JSONValue]`. Remove the canonical array guard's requirement that +every non-standard value be a dict; JSON validation already establishes the +domain and `arrays_to_tuples` establishes canonical sequences. + +- [x] **Step 4: Run selected integration tests and verify GREEN** + +```bash +packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ + packages/zarr-metadata/tests/model/test_group.py \ + packages/zarr-metadata/tests/test_partial_equivalence.py \ + packages/zarr-metadata/tests/test_public_api.py -q +``` + +Expected: all selected files pass and `ExtensionFieldV3` remains exported. + +- [x] **Step 5: Commit Task 3** + +```bash +git add packages/zarr-metadata/src/zarr_metadata/v3/array.py \ + packages/zarr-metadata/src/zarr_metadata/v3/group.py \ + packages/zarr-metadata/src/zarr_metadata/model/_array.py \ + packages/zarr-metadata/src/zarr_metadata/model/_group.py \ + packages/zarr-metadata/src/zarr_metadata/model/_validation.py \ + packages/zarr-metadata/tests/model/test_array.py \ + packages/zarr-metadata/tests/model/test_group.py \ + packages/zarr-metadata/tests/test_partial_equivalence.py +git commit -m "fix(zarr-metadata): align v3 additional field types" \ + -m "Assisted-by: Codex:gpt-5" +``` + +### Task 4: Documentation reconciliation and full verification + +**Files:** +- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/_common.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/array.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/model/__init__.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_array.py` +- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_group.py` +- Modify: `docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md` +- Create: `docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md` + +**Interfaces:** +- Consumes: all prior tasks. +- Produces: documentation matching semantic normalization and the extension-agnostic boundary. + +- [x] **Step 1: Reconcile documentation** + +Remove object-only and v3.0-over-v3.1 claims. Document the exact heuristic: + +```text +empty configuration + must_understand true -> shorthand string +otherwise -> object containing name and non-default members +``` + +Document arbitrary JSON additional fields, type-only name validation, and the +known non-core consolidated extension. Retain the historical null repair. + +- [x] **Step 2: Run the complete package suite** + +```bash +packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests -q +``` + +Expected: zero failures. + +- [x] **Step 3: Run format, lint, typing, and whitespace checks** + +```bash +.venv/bin/ruff format --check packages/zarr-metadata +.venv/bin/ruff check packages/zarr-metadata +cd packages/zarr-metadata +uvx --from pyright==1.1.404 pyright --pythonpath .venv/bin/python src +git diff --check +``` + +Expected: every command exits zero with no diagnostics. + +- [x] **Step 4: Audit scope against the final diff** + +Confirm there is no registry lookup, name-syntax validation, extension +configuration interpretation, fill-value semantics, or codec-kind/order +validation. Confirm `consolidated_metadata: null` is still accepted on read +and never emitted. + +- [x] **Step 5: Commit docs and final integration adjustments** + +```bash +git add -f docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md \ + docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md +git add packages/zarr-metadata +git commit -m "docs(zarr-metadata): document v3 model conformance" \ + -m "Assisted-by: Codex:gpt-5" +``` + +- [x] **Step 6: Verify committed state** + +```bash +git status --short --branch +git log -4 --format='%h %s%n%(trailers:key=Assisted-by,valueonly)' +``` + +Expected: only the user's pre-existing untracked files remain; every new +commit has a conventional subject and `Assisted-by: Codex:gpt-5`. diff --git a/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md b/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md index de352a374d..49e54b2888 100644 --- a/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md +++ b/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md @@ -1,6 +1,6 @@ # Design: Zarr v3 metadata model conformance -**Status:** Approved design; implementation pending. +**Status:** Approved and implemented. **Scope:** `packages/zarr-metadata`, principally the raw v3 metadata types, the immutable model layer, and their validators and tests. **Normative authority:** Zarr core protocol v3.1. The behavior of zarrs and @@ -88,21 +88,21 @@ during normalization. - `must_understand: bool`, normalized to `True` when absent or when the input is a shorthand string. -`to_json()` will continue emitting the canonical object form with a -configuration mapping. It will omit `must_understand` when true and emit -`"must_understand": false` when false. Thus the model is semantically -lossless, but deliberately not spelling-preserving. +`to_json()` will use one field-independent canonicalization rule. When the +configuration is empty and `must_understand` is true, it emits the shorthand +name string. Otherwise it emits an object, omitting `configuration` when empty +and omitting `must_understand` when true. Thus the model is semantically +lossless, but deliberately not spelling-preserving. The same rule naturally +emits core data types as strings without giving the data-type field a separate +model or serializer. ### Name validation -Names will be checked syntactically without attempting to query or freeze the -external extension registry: - -- registered-name syntax is `^[a-z][a-z0-9-_.]+$`; and -- legacy URI names remain accepted for v3.0 compatibility. - -Registry membership is not a structural property that an extension-agnostic -offline model can determine. Extension resolution remains a reader concern. +Names will be checked only to ensure that they are strings. The model will not +validate registered-name syntax, URI syntax, or membership in the external +extension registry. Those properties are not part of the extension-agnostic +structural boundary and are left to applications that resolve extension +names. ## Context-sensitive `must_understand` rules @@ -202,7 +202,7 @@ corrections to annotations that did not match accepted runtime data: - v3 additional-field annotations widen to arbitrary JSON. Documents currently accepted only because an extension envelope contains -unknown members, an invalid name, a forbidden false `must_understand`, or an +unknown members, a non-string name, a forbidden false `must_understand`, or an empty codec list will become validation errors. Those documents violate the agreed core grammar; accepting them is not a compatibility contract to retain. @@ -213,7 +213,7 @@ Implementation will be test-first and cover: 1. shorthand and object normalization, including implicit true; 2. explicit true/false round trips; 3. invalid `must_understand` types and unknown envelope members; -4. registered and legacy URI name forms plus invalid names; +4. arbitrary string names accepted and non-string names rejected; 5. false `must_understand` rejected at mandatory extension points; 6. false accepted for codecs and storage transformers; 7. empty codecs rejected without interpreting non-empty pipelines; diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index 5c530fcebf..4e8d974ab9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -28,8 +28,8 @@ class NamedConfigV3(TypedDict): """ Externally-tagged union member for a metadata field. - The `configuration` mapping holds arbitrary JSON-encodable values; - it is typed as `Mapping[str, JSONValue]`. + The optional `configuration` mapping holds arbitrary JSON-encodable + values. `must_understand` is implicitly true when absent. """ name: str diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index ed2767c851..d1e151053a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -91,9 +91,10 @@ def from_json(cls, data: object) -> NamedConfigModelV3: This is the role-named alias for annotation positions: model fields and consumer signatures should say `MetadataFieldModelV3` (the logical meaning) rather than `NamedConfigModelV3` (the serialized form the field currently -takes). Today every metadata field normalizes to a named configuration, so +takes). Today every metadata field normalizes to a named configuration plus +its reader obligation, so the alias is exactly `NamedConfigModelV3`; if a future spec revision adds a -field form that cannot be normalized to name + configuration, this alias +field form that cannot be normalized to those values, this alias widens to a union and annotation sites do not change. Mirrors the raw-layer split between `NamedConfigV3` (shape) and `MetadataV3` (field union). """ @@ -156,12 +157,13 @@ class ArrayMetadataModelV3Partial(TypedDict, total=False): class ArrayMetadataModelV3: """In-memory model of a v3 array metadata document. - A canonical, lossless representation of the `zarr.json` content for an + A canonical, semantically lossless representation of the `zarr.json` content for an array. Extension points (`data_type`, `chunk_grid`, `chunk_key_encoding`, `codecs`, `storage_transformers`) are held as `MetadataFieldModelV3` - values (currently always `NamedConfigModelV3` name + configuration pairs) - and are never interpreted; `fill_value` is held - verbatim in its JSON form. + values (currently `NamedConfigModelV3` name, configuration, and obligation + records) and are never interpreted; `fill_value` is held verbatim in its + JSON form. Equivalent extension spellings normalize to shorthand strings + when configuration is empty and understanding is required. """ zarr_format: Literal[3] = field(default=3, init=False) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 81bbbad270..5a545a23bf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -131,10 +131,7 @@ def to_json(self) -> GroupMetadataV3: if len(self.attributes) > 0: out["attributes"] = self.attributes if self.consolidated_metadata is not UNSET: - # The consolidated-metadata shape ({kind, must_understand, metadata}, - # no `name`) predates the strict v3.1 extension-field rules, so it is - # not assignable to `ExtensionFieldV3`; see the discussion on - # `zarr_metadata.v3.consolidated`. + # Consolidated metadata is a known non-core top-level JSON field. out[CONSOLIDATED_METADATA_KEY_V3] = cast( "ExtensionFieldV3", self.consolidated_metadata.to_json() ) @@ -145,8 +142,7 @@ def to_json(self) -> GroupMetadataV3: @classmethod def from_json(cls, data: object) -> GroupMetadataModelV3: parsed = parse_group_metadata_v3(arrays_to_tuples(data)) - # Cast to object: the TypedDict's extra_items type does not admit null, - # but wild documents (historical zarr-python) contain it. + # Cast for narrowing across standard and arbitrary extra TypedDict items. consolidated_raw = cast("object", parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET)) consolidated: ConsolidatedMetadataModelV3 | UNSET if consolidated_raw is UNSET or consolidated_raw is None: diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 491eb30aca..f087de6e90 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -187,7 +187,8 @@ def validate_metadata_field_v3( ) -> list[ValidationProblem]: """Return every reason `value` is not a v3 metadata field. - A metadata field is a bare name string or a `{name, configuration}` mapping. + A metadata field is a bare name string or a mapping containing `name` and + optional `configuration` and `must_understand` members. """ if isinstance(value, str): return [] @@ -195,7 +196,7 @@ def validate_metadata_field_v3( return [ ValidationProblem( (), - "expected a metadata field (string or {name, configuration})", + "expected a metadata field (string or extension object)", "invalid_type", ) ] @@ -456,7 +457,7 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: if isinstance(entries, str) or not isinstance(entries, Sequence): problems.append(ValidationProblem((key,), "expected a sequence", "invalid_type")) else: - if key == "codecs" and len(entries) == 0: + if key == "codecs" and len(cast("Sequence[object]", entries)) == 0: problems.append( ValidationProblem( ("codecs",), "expected at least one codec", "invalid_value" diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index a19e192eb4..2c15cf51ce 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -118,7 +118,7 @@ def coerce(value: object) -> _M: PlainSerializer(NamedConfigModelV3.to_json, return_type=str | dict), WithJsonSchema(_FIELD_SCHEMA | {"title": "MetadataFieldV3"}), ] -"""Field type for one v3 metadata field (bare name string or name + configuration).""" +"""Field type for one normalized v3 metadata extension envelope.""" __all__ = [ "ArrayMetadataV2", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_common.py b/packages/zarr-metadata/src/zarr_metadata/v3/_common.py index 3424587a43..3e0d2b054c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_common.py @@ -9,7 +9,7 @@ MetadataV3 = str | NamedConfigV3 """The JSON shape of any v3 metadata extension-point entry: either a bare -short-hand name string or a `{name, configuration}` envelope. +short-hand name string or a `{name, configuration, must_understand}` envelope. Used for `data_type`, `chunk_grid`, `chunk_key_encoding`, individual codec entries, and `storage_transformers` in v3 array metadata, and for diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py index 486a0897a5..79d4b9692d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py @@ -5,14 +5,10 @@ implementation (and zarrs), where consolidated metadata is embedded as an extension field on a group's `zarr.json`. -The shape modeled here (`{kind, must_understand, metadata}` with no `name` -field) reflects the original Zarr v3.0 reading of the extension-field -rules. Under the strict Zarr v3.1 reading, every extension field must -also include a `name: str` key, which would make this shape — and every -real-world consolidated metadata document in the wild — out of spec. -See `ExtensionFieldV3` and -https://github.com/zarr-developers/zarr-specs/issues/371 for the -ongoing discussion. +This is a known non-core interoperability extension. Its +`{kind, must_understand, metadata}` payload is an unknown top-level JSON value +to the core document model; implementations that recognize the convention may +interpret it through this dedicated type. """ from collections.abc import Mapping diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index 7e80d52ee8..6d9b82b3e3 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -121,8 +121,8 @@ def test_dump_emits_canonical_document() -> None: manifest = ArrayManifest.model_validate({"path": "a/b", "metadata": VALID_DOC}) dumped = manifest.model_dump() assert dumped["metadata"] == manifest.metadata.to_json() - # the bare-string data_type was normalized to the canonical object form - assert dumped["metadata"]["data_type"] == {"name": "uint8", "configuration": {}} + # Empty configurations use the extension-definition shorthand form. + assert dumped["metadata"]["data_type"] == "uint8" def test_json_roundtrip_through_pydantic() -> None: @@ -194,10 +194,11 @@ def build_and_use() -> None: class NamedConfig(BaseModel): - """Pydantic mirror of a normalized metadata field (name + configuration).""" + """Pydantic mirror of a normalized metadata extension envelope.""" name: str configuration: dict[str, JSONValue] = {} + must_understand: bool = True class ArrayMetadataV3Spec(BaseModel, Generic[AttrsT]): @@ -229,6 +230,13 @@ def _canonicalize(cls, data: object) -> object: normalization; pydantic then parses only canonical documents.""" if isinstance(data, Mapping): doc = dict(ArrayMetadataModelV3.from_json(data).to_json()) + for key in ("data_type", "chunk_grid", "chunk_key_encoding"): + if isinstance(doc[key], str): + doc[key] = {"name": doc[key]} + for key in ("codecs", "storage_transformers"): + doc[key] = tuple( + {"name": item} if isinstance(item, str) else item for item in doc.get(key, ()) + ) doc.setdefault("attributes", {}) return doc return data From 7a88406d42ad877ec3ad89cc6d417457c2309d9d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 15:35:33 +0200 Subject: [PATCH 53/70] docs: remove llm docs --- ...7-22-zarr-v3-metadata-model-conformance.md | 398 ------------------ ...rr-v3-metadata-model-conformance-design.md | 234 ---------- 2 files changed, 632 deletions(-) delete mode 100644 docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md delete mode 100644 docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md diff --git a/docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md b/docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md deleted file mode 100644 index 1bf8fec1cf..0000000000 --- a/docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md +++ /dev/null @@ -1,398 +0,0 @@ -# Zarr v3 Metadata Model Conformance Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make the extension-agnostic v3 metadata models preserve Zarr 3.1 extension envelopes, enforce core envelope/cardinality rules, and keep raw types, parsers, guards, and serializers consistent. - -**Architecture:** Keep one normalized `ZarrV3NamedConfig` for every extension point. It parses shorthand and object forms, stores opaque configuration plus `must_understand`, and applies one field-independent shorthand heuristic. The array validator supplies only core rules that vary by extension point; it never resolves an extension name or configuration. - -**Tech Stack:** Python 3.11+, frozen dataclasses, `TypedDict`/PEP 728, pytest, Ruff, Pyright, and optional Pydantic v2 integration. - -## Global Constraints - -- Zarr v3.1 is normative; zarrs is interoperability evidence. -- Validate extension names only as strings; do not check syntax or registry membership. -- Do not interpret extension configurations, fill values, or codec kinds/order. -- Reject `must_understand=False` for data types, chunk grids, and chunk-key encodings. -- Require a non-empty codec list. -- Preserve the `consolidated_metadata: null` tolerant-read repair and never emit null. -- Observe each new behavioral test fail before changing production code. -- Commits must be conventional and include `Assisted-by: Codex:gpt-5`. - ---- - -### Task 1: Preserve and canonically serialize extension envelopes - -**Files:** -- Modify: `packages/zarr-metadata/tests/model/test_array.py` -- Modify: `packages/zarr-metadata/tests/model/test_pydantic_module.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/_common.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_array.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_validation.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/pydantic.py` - -**Interfaces:** -- Produces: `ZarrV3NamedConfigJSON` with optional `must_understand: bool`. -- Produces: `ZarrV3NamedConfig(name: str, configuration: dict[str, JSONValue], must_understand: bool = True)`. -- Produces: `validate_metadata_field_v3(value: object, *, allow_must_understand_false: bool = True) -> list[ValidationProblem]`. - -- [x] **Step 1: Write failing normalized-model tests** - -Update the existing case tables and canonical-document expectation: - -```python -ZARR_TO_JSON_CASES = [ - Expect( - ZarrV3NamedConfig(name="regular", configuration={"chunk_shape": [1]}), - {"name": "regular", "configuration": {"chunk_shape": [1]}}, - id="with-configuration", - ), - Expect( - ZarrV3NamedConfig(name="bytes", configuration={}), - "bytes", - id="empty-configuration-shorthand", - ), - Expect( - ZarrV3NamedConfig(name="optional", configuration={}, must_understand=False), - {"name": "optional", "must_understand": False}, - id="false-obligation-needs-object", - ), -] -``` - -Add `from_json` cases for implicit true and explicit false. Expect the default -array document to contain `"uint8"`, `("bytes",)`, and `"default"`, while its -configured regular grid remains an object. - -- [x] **Step 2: Run the focused tests and verify RED** - -```bash -packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ - -k 'zarr_metadata_v3 or canonical_document or explicit_false' -q -``` - -Expected: empty configurations remain objects, the constructor lacks -`must_understand`, or explicit false is lost. - -- [x] **Step 3: Write failing envelope-validation tests** - -```python -@pytest.mark.parametrize("name", ["bytes", "ANY string", "urn:example:codec"]) -def test_metadata_field_accepts_any_string_name(name: str) -> None: - assert validate_metadata_field_v3({"name": name}) == [] - - -@pytest.mark.parametrize("value", [0, 1, "false", None]) -def test_metadata_field_must_understand_must_be_boolean(value: object) -> None: - problems = validate_metadata_field_v3({"name": "x", "must_understand": value}) - assert [(problem.loc, problem.kind) for problem in problems] == [ - (("must_understand",), "invalid_type") - ] - - -def test_metadata_field_rejects_unknown_envelope_member() -> None: - problems = validate_metadata_field_v3({"name": "x", "typo": 1}) - assert [(problem.loc, problem.kind) for problem in problems] == [ - (("typo",), "invalid_value") - ] -``` - -- [x] **Step 4: Run the envelope tests and verify RED** - -```bash -packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ - -k 'metadata_field_accepts or must_understand_must_be or unknown_envelope' -q -``` - -Expected: invalid obligation values and the unknown member are accepted before the fix. - -- [x] **Step 5: Implement the raw type, validator, and normalized model** - -```python -class ZarrV3NamedConfigJSON(TypedDict): - name: str - configuration: NotRequired[Mapping[str, JSONValue]] - must_understand: NotRequired[bool] -``` - -Reject envelope keys outside `name`, `configuration`, and `must_understand`; -require a real boolean for the last member; keep all string names valid. - -```python -@dataclass(frozen=True, slots=True, kw_only=True) -class ZarrV3NamedConfig: - name: str - configuration: dict[str, JSONValue] - must_understand: bool = True - - def to_json(self) -> ZarrV3MetadataFieldJSON: - if not self.configuration and self.must_understand: - return self.name - out: ZarrV3NamedConfigJSON = {"name": self.name} - if self.configuration: - out["configuration"] = self.configuration - if not self.must_understand: - out["must_understand"] = False - return out -``` - -`from_json` sets true for shorthand/absence and preserves explicit false. - -- [x] **Step 6: Update and test the Pydantic serializer** - -Change `ZarrV3MetadataField`'s serializer return type from `dict` to `str | dict`. - -```python -def test_metadata_field_serializes_shorthand_and_false_object() -> None: - adapter = TypeAdapter(zmp.ZarrV3MetadataField) - assert adapter.dump_python(adapter.validate_python({"name": "bytes"})) == "bytes" - assert adapter.dump_python( - adapter.validate_python({"name": "optional", "must_understand": False}) - ) == {"name": "optional", "must_understand": False} -``` - -- [x] **Step 7: Run Task 1 tests and verify GREEN** - -```bash -packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ - packages/zarr-metadata/tests/model/test_pydantic_module.py -q -``` - -Expected: both files pass. - -- [x] **Step 8: Commit Task 1** - -```bash -git add packages/zarr-metadata/src/zarr_metadata/_common.py \ - packages/zarr-metadata/src/zarr_metadata/model/_array.py \ - packages/zarr-metadata/src/zarr_metadata/model/_validation.py \ - packages/zarr-metadata/src/zarr_metadata/pydantic.py \ - packages/zarr-metadata/tests/model/test_array.py \ - packages/zarr-metadata/tests/model/test_pydantic_module.py -git commit -m "fix(zarr-metadata): preserve v3 extension obligations" \ - -m "Assisted-by: Codex:gpt-5" -``` - -### Task 2: Enforce context-sensitive core array rules - -**Files:** -- Modify: `packages/zarr-metadata/tests/model/test_array.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_validation.py` - -**Interfaces:** -- Consumes: `validate_metadata_field_v3(..., allow_must_understand_false=...)`. -- Produces: rejection of forbidden false obligations and empty codecs. - -- [x] **Step 1: Write failing context and cardinality tests** - -```python -@pytest.mark.parametrize("field", ["codecs", "storage_transformers"]) -def test_optional_extension_points_allow_must_understand_false(field: str) -> None: - doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) - doc[field] = ({"name": "optional", "must_understand": False},) - assert validate_array_metadata_v3(doc) == [] - - -@pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) -def test_required_extension_points_reject_must_understand_false(field: str) -> None: - doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) - doc[field] = {"name": "optional", "must_understand": False} - assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ - ((field, "must_understand"), "invalid_value") - ] - - -def test_v3_codecs_cannot_be_empty() -> None: - doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) - doc["codecs"] = () - assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ - (("codecs",), "invalid_value") - ] -``` - -- [x] **Step 2: Run Task 2 tests and verify RED** - -```bash -packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ - -k 'optional_extension_points or required_extension_points or codecs_cannot' -q -``` - -Expected: required points and empty codecs are accepted before the fix. - -- [x] **Step 3: Implement the context rules** - -Call `validate_metadata_field_v3(..., allow_must_understand_false=False)` for -`data_type`, `chunk_grid`, and `chunk_key_encoding`. Leave codecs and storage -transformers at the default. After establishing that `codecs` is a sequence, -add one field-level `invalid_value` problem when it is empty. - -- [x] **Step 4: Run the complete array model tests and verify GREEN** - -```bash -packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py -q -``` - -Expected: the file passes. - -- [x] **Step 5: Commit Task 2** - -```bash -git add packages/zarr-metadata/src/zarr_metadata/model/_validation.py \ - packages/zarr-metadata/tests/model/test_array.py -git commit -m "fix(zarr-metadata): enforce v3 core extension rules" \ - -m "Assisted-by: Codex:gpt-5" -``` - -### Task 3: Align additional-field types, guards, and models - -**Files:** -- Modify: `packages/zarr-metadata/tests/model/test_array.py` -- Modify: `packages/zarr-metadata/tests/model/test_group.py` -- Modify: `packages/zarr-metadata/tests/test_partial_equivalence.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/array.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/group.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_array.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_group.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_validation.py` - -**Interfaces:** -- Produces: public `ZarrV3ExtensionField` alias for arbitrary `JSONValue`. -- Produces: `extra_fields: dict[str, JSONValue]` and matching guards/parsers. - -- [x] **Step 1: Write failing parser/guard agreement tests** - -```python -def test_v3_scalar_extra_field_agrees_across_parser_guard_and_model() -> None: - raw = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"ext": 1} - parsed = parse_array_metadata_v3(raw) - assert is_array_metadata_v3(parsed) - model = ZarrV3ArrayMetadata.from_json(raw) - assert model.extra_fields == {"ext": 1} - assert model.must_understand_fields == {"ext": 1} - - -def test_group_v3_scalar_extra_field_roundtrips_as_must_understand() -> None: - raw = {"zarr_format": 3, "node_type": "group", "ext": [1, 2]} - model = ZarrV3GroupMetadata.from_json(raw) - assert model.to_json()["ext"] == (1, 2) - assert model.must_understand_fields == {"ext": (1, 2)} -``` - -- [x] **Step 2: Run the agreement tests and verify RED** - -```bash -packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ - packages/zarr-metadata/tests/model/test_group.py -k 'scalar_extra_field' -q -``` - -Expected: the canonical array guard rejects the parsed scalar extra field. - -- [x] **Step 3: Correct raw/model annotations and the guard** - -Replace the object-shaped `ZarrV3ExtensionField` TypedDict with a public alias to -`JSONValue`; use it as `extra_items` on full and partial array/group types. -Update model partials, dataclass fields, helper signatures, and casts to -`dict[str, JSONValue]`. Remove the canonical array guard's requirement that -every non-standard value be a dict; JSON validation already establishes the -domain and `arrays_to_tuples` establishes canonical sequences. - -- [x] **Step 4: Run selected integration tests and verify GREEN** - -```bash -packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests/model/test_array.py \ - packages/zarr-metadata/tests/model/test_group.py \ - packages/zarr-metadata/tests/test_partial_equivalence.py \ - packages/zarr-metadata/tests/test_public_api.py -q -``` - -Expected: all selected files pass and `ZarrV3ExtensionField` remains exported. - -- [x] **Step 5: Commit Task 3** - -```bash -git add packages/zarr-metadata/src/zarr_metadata/v3/array.py \ - packages/zarr-metadata/src/zarr_metadata/v3/group.py \ - packages/zarr-metadata/src/zarr_metadata/model/_array.py \ - packages/zarr-metadata/src/zarr_metadata/model/_group.py \ - packages/zarr-metadata/src/zarr_metadata/model/_validation.py \ - packages/zarr-metadata/tests/model/test_array.py \ - packages/zarr-metadata/tests/model/test_group.py \ - packages/zarr-metadata/tests/test_partial_equivalence.py -git commit -m "fix(zarr-metadata): align v3 additional field types" \ - -m "Assisted-by: Codex:gpt-5" -``` - -### Task 4: Documentation reconciliation and full verification - -**Files:** -- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/_common.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/array.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/model/__init__.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_array.py` -- Modify: `packages/zarr-metadata/src/zarr_metadata/model/_group.py` -- Modify: `docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md` -- Create: `docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md` - -**Interfaces:** -- Consumes: all prior tasks. -- Produces: documentation matching semantic normalization and the extension-agnostic boundary. - -- [x] **Step 1: Reconcile documentation** - -Remove object-only and v3.0-over-v3.1 claims. Document the exact heuristic: - -```text -empty configuration + must_understand true -> shorthand string -otherwise -> object containing name and non-default members -``` - -Document arbitrary JSON additional fields, type-only name validation, and the -known non-core consolidated extension. Retain the historical null repair. - -- [x] **Step 2: Run the complete package suite** - -```bash -packages/zarr-metadata/.venv/bin/pytest packages/zarr-metadata/tests -q -``` - -Expected: zero failures. - -- [x] **Step 3: Run format, lint, typing, and whitespace checks** - -```bash -.venv/bin/ruff format --check packages/zarr-metadata -.venv/bin/ruff check packages/zarr-metadata -cd packages/zarr-metadata -uvx --from pyright==1.1.404 pyright --pythonpath .venv/bin/python src -git diff --check -``` - -Expected: every command exits zero with no diagnostics. - -- [x] **Step 4: Audit scope against the final diff** - -Confirm there is no registry lookup, name-syntax validation, extension -configuration interpretation, fill-value semantics, or codec-kind/order -validation. Confirm `consolidated_metadata: null` is still accepted on read -and never emitted. - -- [x] **Step 5: Commit docs and final integration adjustments** - -```bash -git add -f docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md \ - docs/superpowers/plans/2026-07-22-zarr-v3-metadata-model-conformance.md -git add packages/zarr-metadata -git commit -m "docs(zarr-metadata): document v3 model conformance" \ - -m "Assisted-by: Codex:gpt-5" -``` - -- [x] **Step 6: Verify committed state** - -```bash -git status --short --branch -git log -4 --format='%h %s%n%(trailers:key=Assisted-by,valueonly)' -``` - -Expected: only the user's pre-existing untracked files remain; every new -commit has a conventional subject and `Assisted-by: Codex:gpt-5`. diff --git a/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md b/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md deleted file mode 100644 index 504ff10a6a..0000000000 --- a/docs/superpowers/specs/2026-07-22-zarr-v3-metadata-model-conformance-design.md +++ /dev/null @@ -1,234 +0,0 @@ -# Design: Zarr v3 metadata model conformance - -**Status:** Approved and implemented. -**Scope:** `packages/zarr-metadata`, principally the raw v3 metadata types, -the immutable model layer, and their validators and tests. -**Normative authority:** Zarr core protocol v3.1. The behavior of zarrs and -zarr-python is interoperability evidence where the specification leaves an -implementation boundary or where a documented legacy-read exception is -required. - -## Goal - -Make the v3 array and group metadata models conform to the core Zarr v3.1 -document grammar without turning the generic models into interpreters for -individual extensions. - -The generic layer will validate required and optional core fields, JSON value -and container types, fixed literals and core cross-field constraints, the -common extension envelope, the core rules governing `must_understand`, and the -forward-compatibility obligations for unknown top-level fields. - -It will not validate the configuration semantics of a data type, chunk grid, -chunk key encoding, codec, or storage transformer. Those belong to typed -extension models and readers that resolve extension names. - -## Sources and precedence - -The implementation will be checked against: - -1. [Zarr core protocol v3.1](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html), - especially Array metadata, Group metadata, Codecs, and Extensions. -2. [zarrs](https://github.com/zarrs/zarrs), especially its `ZarrV3MetadataFieldJSON`, - `AdditionalFieldV3`, array-opening validation, and consolidated-metadata - extension handling. -3. Existing zarr-python metadata emitted in the wild, but only for explicit - tolerant-read paths. Legacy data does not redefine what this package emits - or calls conformant. - -If these disagree, the current v3.1 specification wins. An interoperability -exception must be narrow, documented as non-conformant input, normalized on -read, and never emitted. - -## Chosen approach - -Evolve the existing normalized `ZarrV3NamedConfig` instead of introducing a -new hierarchy of role-specific extension classes or preserving every source -JSON spelling. - -This keeps the public shape small and minimizes compatibility churn. -Validation remains context-aware: the same extension envelope is used at all -extension points, while the array-document validator supplies the few rules -that vary by field. - -Two alternatives were rejected: - -- A spelling-preserving model would retain shorthand versus object form and - absent versus empty configuration. That fidelity is not needed by this - semantic model and would substantially change its API. -- Separate required/optional extension classes would encode - `must_understand` restrictions in types, but would duplicate behavior and - force callers to use different classes for otherwise identical envelopes. - -## Extension envelope - -### Raw type - -`ZarrV3NamedConfigJSON` will describe the v3.1 object form: - -- required `name: str`; -- optional `configuration: Mapping[str, JSONValue]`; and -- optional `must_understand: bool`. - -`ZarrV3MetadataFieldJSON` remains the union of a shorthand name string and this object -form. - -The object form accepts only those three members. Extension-owned fields must -be placed inside `configuration`. Rejecting other envelope members matches -zarrs' `deny_unknown_fields` behavior and prevents silently discarding data -during normalization. - -### Normalized model - -`ZarrV3NamedConfig` will hold: - -- `name: str`; -- `configuration: dict[str, JSONValue]`, normalized to an empty mapping when - absent; and -- `must_understand: bool`, normalized to `True` when absent or when the input - is a shorthand string. - -`to_json()` will use one field-independent canonicalization rule. When the -configuration is empty and `must_understand` is true, it emits the shorthand -name string. Otherwise it emits an object, omitting `configuration` when empty -and omitting `must_understand` when true. Thus the model is semantically -lossless, but deliberately not spelling-preserving. The same rule naturally -emits core data types as strings without giving the data-type field a separate -model or serializer. - -### Name validation - -Names will be checked only to ensure that they are strings. The model will not -validate registered-name syntax, URI syntax, or membership in the external -extension registry. Those properties are not part of the extension-agnostic -structural boundary and are left to applications that resolve extension -names. - -## Context-sensitive `must_understand` rules - -An omitted `must_understand` value and every shorthand name mean `True`. - -The array validator will reject `must_understand=False` for `data_type`, -`chunk_grid`, and `chunk_key_encoding`. It will permit false for individual -codecs and storage transformers. - -This validation is structural only. The model does not decide whether an -implementation recognizes a name and does not remove or skip optional -extensions. A reader that resolves extensions owns that decision. - -## Core array and group constraints - -The generic array model will enforce the core constraints that do not require -interpreting an extension: - -- `zarr_format` is exactly `3` and `node_type` is exactly `"array"`; -- `shape` contains non-negative JSON integers (booleans excluded); -- all mandatory fields are present; -- `fill_value` is JSON, while its data-type-dependent meaning remains opaque; -- `codecs` is a non-empty sequence of valid extension envelopes; -- `storage_transformers`, when present, is a sequence of valid envelopes; -- `attributes`, when present, is a string-keyed JSON object; and -- `dimension_names`, when present, contains strings or null and has the same - length as `shape`. - -The model will not enforce regular-grid rank or positive chunk lengths, -data-type-specific fill values, codec kinds or ordering, or storage-transformer -behavior. These rules require resolving an extension name and therefore live -outside the generic model. - -The generic group model will enforce the corresponding core document rules: -fixed literals, required fields, JSON attributes, string top-level keys, and -unknown-field handling. - -## Unknown top-level fields - -Unknown array and group members will be represented as arbitrary `JSONValue`, -not as a TypedDict that requires an object. - -For reader obligations: - -- an object whose `must_understand` member is the literal JSON boolean `false` - is explicitly waivable; and -- every other value, including scalars, arrays, objects with no such member, - and objects with a non-boolean value, implicitly requires understanding. - -This matches the v3.1 implicit-true rule and zarrs' `AdditionalFieldV3` -behavior. The raw and model annotations, `is_*` guards, parsers, and -`must_understand_fields` property must all agree on this representation. - -The model itself will not fail merely because such a field requires -understanding. It has no extension registry. It exposes the partition so the -reader can fail when a required field is unrecognized. - -## Consolidated metadata - -Inline consolidated metadata is a known interoperability extension rather -than a core v3.1 group field. Its dedicated raw and model types will remain, -and group parsing may continue recognizing it explicitly. - -The standard emitted form is an object containing `kind`, `metadata`, and a -`must_understand` marker. The extension model validates its own payload and -recursively validates embedded v3 nodes. This special handling must not cause -the generic unknown-field type to become object-only. - -Historical zarr-python versions wrote `"consolidated_metadata": null`. -zarrs also accepts this input as a compatibility hotfix. `from_json()` will -continue repairing that exact value to absence, and `to_json()` will never -write it. Documentation and tests will identify this as a tolerant-read -exception, not valid core metadata. - -## Errors and normalization - -All newly enforced constraints use the existing aggregated -`MetadataValidationError`/`ValidationProblem` mechanism. Locations must point -to the envelope member or array entry that failed, and problem kinds remain -machine-readable. - -Parsing still recursively converts JSON arrays to the tuple-backed public -representation. Validation and type guards must agree on canonical runtime -containers after normalization. - -No constructor-wide semantic validation will be added to `update()`; -`from_json()` remains the validated ingestion boundary. Direct construction -and `update()` continue to support building or repairing intermediate models. - -## Public API compatibility - -The existing class names remain. The intended public changes are additive or -corrections to annotations that did not match accepted runtime data: - -- `ZarrV3NamedConfigJSON` gains optional `must_understand`; -- `ZarrV3NamedConfig` gains a defaulted `must_understand` field; and -- v3 additional-field annotations widen to arbitrary JSON. - -Documents currently accepted only because an extension envelope contains -unknown members, a non-string name, a forbidden false `must_understand`, or an -empty codec list will become validation errors. Those documents violate the -agreed core grammar; accepting them is not a compatibility contract to retain. - -## Test strategy - -Implementation will be test-first and cover: - -1. shorthand and object normalization, including implicit true; -2. explicit true/false round trips; -3. invalid `must_understand` types and unknown envelope members; -4. arbitrary string names accepted and non-string names rejected; -5. false `must_understand` rejected at mandatory extension points; -6. false accepted for codecs and storage transformers; -7. empty codecs rejected without interpreting non-empty pipelines; -8. arbitrary JSON unknown top-level fields and their obligation partition; -9. raw type, parser, type-guard, model, and serializer agreement; -10. consolidated-metadata parsing and the legacy-null read repair; and -11. representative metadata serialized by zarrs. - -The package test suite, formatting, linting, and static type checking will run -before the implementation commit. - -## Completion criteria - -The work is complete when every normative core-document rule in scope has an -explicit validator or a documented extension-owned boundary; raw annotations -and runtime behavior agree; tests demonstrate `must_understand` at every -extension point; zarrs-compatible metadata round-trips semantically; and the -package tests and quality checks pass. From 989c3c1dea0561ab6ea5714f2d636b54eaa8f2b3 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 15:44:30 +0200 Subject: [PATCH 54/70] chore(zarr-metadata): correct changelog PR number Assisted-by: Codex:gpt-5 --- .../zarr-metadata/changes/{210.feature.md => 4119.feature.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{210.feature.md => 4119.feature.md} (100%) diff --git a/packages/zarr-metadata/changes/210.feature.md b/packages/zarr-metadata/changes/4119.feature.md similarity index 100% rename from packages/zarr-metadata/changes/210.feature.md rename to packages/zarr-metadata/changes/4119.feature.md From 2586faa1fc2ad19b80e21559aa5e262f3c29b47a Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 17:00:20 +0200 Subject: [PATCH 55/70] fix(metadata): enforce canonical document boundaries Assisted-by: Codex:GPT-5 --- packages/zarr-metadata/README.md | 28 +++-- .../zarr-metadata/changes/4119.feature.md | 28 +++-- packages/zarr-metadata/pyproject.toml | 2 +- .../src/zarr_metadata/model/_array.py | 12 +- .../src/zarr_metadata/model/_group.py | 32 +++-- .../src/zarr_metadata/model/_validation.py | 115 ++++++++++++++---- .../src/zarr_metadata/pydantic.py | 78 ++++++++---- .../zarr-metadata/tests/model/test_array.py | 92 +++++++++++++- .../zarr-metadata/tests/model/test_group.py | 74 +++++++++++ .../tests/model/test_pydantic_module.py | 23 +++- 10 files changed, 384 insertions(+), 100 deletions(-) diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index f2884782fd..bcd30b742d 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -9,6 +9,9 @@ JSON shapes specified by the [Zarr v2](https://zarr-specs.readthedocs.io/en/late and [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) specifications, plus types for [`zarr-extensions`](https://github.com/zarr-developers/zarr-extensions/) and a few widely-used-but-unspecified entities (e.g. consolidated metadata). +It also provides canonical frozen-dataclass models, structural validators, +parsers, store-key serialization, and optional Pydantic field integrations. +The optional integration requires Pydantic 2.13 or newer. ## What this is for @@ -28,17 +31,24 @@ with open("zarr.json", "rb") as f: metadata = TypeAdapter(ZarrV3ArrayMetadataJSON).validate_python(raw) ``` -## What this is *not* +For a normalized model with loc-aware validation and serialization: -- Not a parser or builder. There are no `make_array_metadata(...)` factories — - that surface belongs to consumer libraries. -- Not a runtime validator on its own. Pair with `pydantic`, `msgspec`, or - similar to enforce shapes at decode time. +```python +from zarr_metadata.model import ZarrV3ArrayMetadata + +model = ZarrV3ArrayMetadata.from_json(raw) +encoded = model.to_key_value()["zarr.json"] +``` + +## Validation boundary -Even with a runtime validator, these types only describe **structural** -shape — they will not flag *semantically* invalid metadata, like a 3D v3 -array whose `dimension_names` has 4 entries instead of 3. That's a job -for downstream validator routines. +The model validators enforce the declared document structure and a small set +of context-free consistency rules, including fixed format literals, finite +JSON numbers, non-negative dimensions, non-empty v3 codec pipelines, and one +`dimension_names` entry per array dimension. They do not interpret extension +names or configurations, resolve codec pipelines, or decide whether a data +type, chunk grid, codec, or storage transformer is supported. Those decisions +belong to consumer implementations. ## Scope diff --git a/packages/zarr-metadata/changes/4119.feature.md b/packages/zarr-metadata/changes/4119.feature.md index fda1a88cf7..b32caf7b1a 100644 --- a/packages/zarr-metadata/changes/4119.feature.md +++ b/packages/zarr-metadata/changes/4119.feature.md @@ -1,13 +1,15 @@ Added `zarr_metadata.model`: frozen-dataclass models (`ZarrV2ArrayMetadata`, `ZarrV3ArrayMetadata`, `ZarrV2GroupMetadata`, `ZarrV3GroupMetadata`, `ZarrV2ConsolidatedMetadata`, `ZarrV3ConsolidatedMetadata`, `ZarrV3NamedConfig`) -that are canonical, lossless representations of Zarr metadata documents, plus -structural validators (`validate_*` / `is_*` / `parse_*`). Every v3 extension -point (data type, chunk grid, chunk key encoding, codecs, storage transformers) -is held as a name + configuration pair; nothing is interpreted. Model fields -are annotated with the role alias `ZarrV3MetadataField` (today exactly -`ZarrV3NamedConfig`), so the annotations convey the logical meaning of the -field and stay put if the spec ever adds a new field form. +that are canonical, semantically lossless representations of Zarr metadata +documents, plus structural validators (`validate_*` / `is_*` / `parse_*`). +Every v3 extension point (data type, chunk grid, chunk key encoding, codecs, +storage transformers) is held as `ZarrV3NamedConfig`: a name, configuration, +and `must_understand` obligation; nothing is interpreted. On the wire, an +empty configuration with the default obligation uses the spec's plain-string +shorthand. Model fields are annotated with the role alias +`ZarrV3MetadataField` (today exactly `ZarrV3NamedConfig`), so annotations +convey the logical meaning and stay put if the spec adds another field form. Validation is strict about what the types declare: v2 `dtype` / `order` / `compressor` / `filters` / `dimension_separator` shapes and the fixed @@ -21,6 +23,8 @@ adversarial review added further structural checks: JSON booleans are not accepted as dimension lengths, dimensions are non-negative, `dimension_names` must have one entry per dimension of `shape`, `attributes` and `configuration` values are JSON-checked recursively (like `fill_value`), +non-finite floats and non-standard JSON constants are rejected, abstract +mappings and sequences normalize to encoder-safe canonical containers, and the inline consolidated-metadata envelope and entries are deep-validated so the group validator's verdict always agrees with the model constructor. @@ -31,11 +35,13 @@ duty by subtracting the extension names they recognize; the model only partitions by obligation, since recognition is reader-specific. Optional pydantic integration ships as `zarr_metadata.pydantic` (importing it -requires pydantic v2; the core package does not depend on it): one `Annotated` +requires pydantic 2.13 or newer; the core package does not depend on it): one +`Annotated` field type per model, validating raw documents through `from_json`, passing -core-model instances through unchanged, and serializing via `to_json`. The -instances are the core model classes, so values interoperate freely with -non-pydantic code. +core-model instances through unchanged, serializing via `to_json`, and +publishing JSON Schemas derived from the raw document TypedDicts. The instances +are the core model classes, so values interoperate freely with non-pydantic +code. `create_default` keeps its output self-consistent: overriding `shape` without a chunk grid derives one regular chunk covering the array (v3 diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 60f5c4aa95..23905a5bb7 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -47,7 +47,7 @@ Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/z Documentation = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-metadata/README.md" [dependency-groups] -test = ["pytest", "pydantic>=2"] +test = ["pytest", "pydantic>=2.13"] [tool.hatch.version] source = "vcs" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index fea75d6e96..14ca92e8fc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -3,7 +3,6 @@ from __future__ import annotations import dataclasses -import json from collections.abc import Mapping from dataclasses import dataclass, field from typing import TYPE_CHECKING, Final, Literal, TypeAlias, cast @@ -16,6 +15,7 @@ MetadataValidationError, ValidationProblem, arrays_to_tuples, + dump_store_json, load_store_json, parse_array_metadata_v2, parse_array_metadata_v3, @@ -318,9 +318,7 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3ArrayMetadata: return cls.from_json(load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V3)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: - return { - ARRAY_METADATA_STORE_KEY_V3: json.dumps(self.to_json(), indent=indent).encode("utf-8") - } + return {ARRAY_METADATA_STORE_KEY_V3: dump_store_json(self.to_json(), indent=indent)} class ZarrV2ArrayMetadataPartial(TypedDict, total=False): @@ -472,9 +470,7 @@ def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes # document must exclude them. The `.zattrs` key is present exactly # when attributes are set (even empty) — UNSET emits no file. zarray = {k: v for k, v in self.to_json().items() if k != "attributes"} - out = {ARRAY_METADATA_STORE_KEY_V2: json.dumps(zarray, indent=indent).encode("utf-8")} + out = {ARRAY_METADATA_STORE_KEY_V2: dump_store_json(zarray, indent=indent)} if self.attributes is not UNSET: - out[ATTRIBUTES_STORE_KEY_V2] = json.dumps(self.attributes, indent=indent).encode( - "utf-8" - ) + out[ATTRIBUTES_STORE_KEY_V2] = dump_store_json(self.attributes, indent=indent) return out diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index ae43955d91..39c4da5b1e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -3,7 +3,6 @@ from __future__ import annotations import dataclasses -import json from collections.abc import Mapping from dataclasses import dataclass, field from typing import TYPE_CHECKING, Final, Literal, cast @@ -21,6 +20,7 @@ MetadataValidationError, ValidationProblem, arrays_to_tuples, + dump_store_json, load_store_json, parse_group_metadata_v2, parse_group_metadata_v3, @@ -184,9 +184,7 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3GroupMetadata: return cls.from_json(load_store_json(mapping, GROUP_METADATA_STORE_KEY_V3)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: - return { - GROUP_METADATA_STORE_KEY_V3: json.dumps(self.to_json(), indent=indent).encode("utf-8") - } + return {GROUP_METADATA_STORE_KEY_V3: dump_store_json(self.to_json(), indent=indent)} @dataclass(frozen=True, slots=True, kw_only=True) @@ -228,10 +226,11 @@ def to_json(self) -> ZarrV3ConsolidatedMetadataJSON: @classmethod def from_json(cls, data: object) -> ZarrV3ConsolidatedMetadata: - problems = validate_consolidated_metadata_v3(data) + normalized = arrays_to_tuples(data) + problems = validate_consolidated_metadata_v3(normalized) if problems: raise MetadataValidationError(problems) - env = cast("Mapping[str, object]", data) + env = cast("Mapping[str, object]", normalized) entries: dict[str, ZarrV3ArrayMetadata | ZarrV3GroupMetadata] = {} for key, entry in cast("Mapping[str, object]", env["metadata"]).items(): node_type = cast("Mapping[str, object]", entry).get("node_type") @@ -330,11 +329,9 @@ def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes # document must exclude them. The `.zattrs` key is present exactly # when attributes are set (even empty) — UNSET emits no file. zgroup = {k: v for k, v in self.to_json().items() if k != "attributes"} - out = {GROUP_METADATA_STORE_KEY_V2: json.dumps(zgroup, indent=indent).encode("utf-8")} + out = {GROUP_METADATA_STORE_KEY_V2: dump_store_json(zgroup, indent=indent)} if self.attributes is not UNSET: - out[ATTRIBUTES_STORE_KEY_V2] = json.dumps(self.attributes, indent=indent).encode( - "utf-8" - ) + out[ATTRIBUTES_STORE_KEY_V2] = dump_store_json(self.attributes, indent=indent) return out @@ -360,16 +357,21 @@ def to_json(self) -> dict[str, JSONValue]: @classmethod def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata: - if not isinstance(data, Mapping): + normalized = arrays_to_tuples(data) + if not isinstance(normalized, Mapping): raise MetadataValidationError( [ValidationProblem((), "expected a mapping", "invalid_type")] ) - doc = cast("Mapping[str, object]", data) + doc = cast("Mapping[str, object]", normalized) problems: list[ValidationProblem] = [ ValidationProblem((key,), "missing required key", "missing_key") for key in ("zarr_consolidated_format", "metadata") if key not in doc ] + problems.extend( + ValidationProblem((key,), "unexpected document member", "invalid_value") + for key in doc.keys() - {"zarr_consolidated_format", "metadata"} + ) if "zarr_consolidated_format" in doc and ( not isinstance(doc["zarr_consolidated_format"], int) or isinstance(doc["zarr_consolidated_format"], bool) @@ -413,8 +415,4 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ConsolidatedMetad return cls.from_json(load_store_json(mapping, CONSOLIDATED_METADATA_STORE_KEY_V2)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: - return { - CONSOLIDATED_METADATA_STORE_KEY_V2: json.dumps(self.to_json(), indent=indent).encode( - "utf-8" - ) - } + return {CONSOLIDATED_METADATA_STORE_KEY_V2: dump_store_json(self.to_json(), indent=indent)} diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 049a399685..5e8f669a1c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -14,9 +14,10 @@ from __future__ import annotations import json +import math from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import Any, Final, Literal, cast +from typing import Any, Final, Literal, NoReturn, cast from typing_extensions import TypeIs @@ -76,7 +77,11 @@ def _prefix(loc_head: str | int, problems: list[ValidationProblem]) -> list[Vali def validate_json(value: object) -> list[ValidationProblem]: """Return every reason `value` is not JSON-serializable (recursively).""" - if isinstance(value, (str, int, float, bool)) or value is None: + if isinstance(value, float): + if math.isfinite(value): + return [] + return [ValidationProblem((), f"non-finite float {value!r} is not JSON", "invalid_value")] + if isinstance(value, (str, int, bool)) or value is None: return [] problems: list[ValidationProblem] = [] if isinstance(value, Mapping): @@ -101,11 +106,12 @@ def is_json(value: object) -> TypeIs[JSONValue]: def parse_json(value: object) -> JSONValue: - """Return `value` narrowed to `JSONValue`, or raise `MetadataValidationError`.""" - problems = validate_json(value) + """Return a canonical `JSONValue`, or raise `MetadataValidationError`.""" + normalized = arrays_to_tuples(value) + problems = validate_json(normalized) if problems: raise MetadataValidationError(problems) - return cast(JSONValue, value) + return cast(JSONValue, normalized) # The standard top-level keys of a v3 array metadata document. Anything outside @@ -124,6 +130,12 @@ def parse_json(value: object) -> JSONValue: ARRAY_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( ZarrV2ArrayMetadataJSON.__required_keys__ ) +ARRAY_METADATA_OPTIONAL_KEYS_V2: Final[frozenset[str]] = frozenset( + ZarrV2ArrayMetadataJSON.__optional_keys__ +) +ARRAY_METADATA_STANDARD_KEYS_V2: Final[frozenset[str]] = ( + ARRAY_METADATA_REQUIRED_KEYS_V2 | ARRAY_METADATA_OPTIONAL_KEYS_V2 +) # The standard top-level keys of a v3 group metadata document. Anything outside # this set is an extension field. @@ -140,6 +152,12 @@ def parse_json(value: object) -> JSONValue: GROUP_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( ZarrV2GroupMetadataJSON.__required_keys__ ) +GROUP_METADATA_OPTIONAL_KEYS_V2: Final[frozenset[str]] = frozenset( + ZarrV2GroupMetadataJSON.__optional_keys__ +) +GROUP_METADATA_STANDARD_KEYS_V2: Final[frozenset[str]] = ( + GROUP_METADATA_REQUIRED_KEYS_V2 | GROUP_METADATA_OPTIONAL_KEYS_V2 +) def _missing_keys(required: frozenset[str], doc: Mapping[str, object]) -> list[ValidationProblem]: @@ -150,11 +168,28 @@ def _missing_keys(required: frozenset[str], doc: Mapping[str, object]) -> list[V ] +def _unexpected_keys( + allowed: frozenset[str], doc: Mapping[object, object] +) -> list[ValidationProblem]: + """One problem per member outside a closed document's declared shape.""" + problems: list[ValidationProblem] = [] + for key in doc: + if not isinstance(key, str): + problems.append( + ValidationProblem((), f"non-string document key {key!r}", "invalid_type") + ) + elif key not in allowed: + problems.append( + ValidationProblem((key,), "unexpected document member", "invalid_value") + ) + return problems + + def _check_literal( doc: Mapping[str, object], key: str, expected: object ) -> list[ValidationProblem]: """One `invalid_value` problem if `doc[key]` is present but not `expected`.""" - if key in doc and doc[key] != expected: + if key in doc and (type(doc[key]) is not type(expected) or doc[key] != expected): return [ ValidationProblem((key,), f"expected {expected!r}, got {doc[key]!r}", "invalid_value") ] @@ -251,10 +286,11 @@ def is_metadata_field_v3(value: object) -> TypeIs[ZarrV3MetadataFieldJSON]: def parse_metadata_field_v3(value: object) -> ZarrV3MetadataFieldJSON: """Return `value` narrowed to `ZarrV3MetadataFieldJSON`, or raise `MetadataValidationError`.""" - problems = validate_metadata_field_v3(value) + normalized = arrays_to_tuples(value) + problems = validate_metadata_field_v3(normalized) if problems: raise MetadataValidationError(problems) - return cast(ZarrV3MetadataFieldJSON, value) + return cast(ZarrV3MetadataFieldJSON, normalized) def _is_int_sequence(value: object) -> bool: @@ -523,6 +559,9 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V2, doc) + problems.extend( + _unexpected_keys(ARRAY_METADATA_STANDARD_KEYS_V2, cast("Mapping[object, object]", value)) + ) problems.extend(_check_literal(doc, "zarr_format", 2)) problems.extend(_validate_dim_sequence(doc, "shape")) problems.extend(_validate_dim_sequence(doc, "chunks")) @@ -606,6 +645,12 @@ def validate_consolidated_metadata_v3(value: object) -> list[ValidationProblem]: for key in ("kind", "must_understand", "metadata") if key not in env ] + problems.extend( + _unexpected_keys( + frozenset({"kind", "must_understand", "metadata"}), + cast("Mapping[object, object]", value), + ) + ) problems.extend(_check_literal(env, "kind", "inline")) if "must_understand" in env and env["must_understand"] is not False: problems.append(ValidationProblem(("must_understand",), "expected False", "invalid_value")) @@ -681,15 +726,18 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: def is_group_metadata_v3(value: object) -> TypeIs[ZarrV3GroupMetadataJSON]: """Whether `value` is a structurally-valid v3 group metadata document.""" - return not validate_group_metadata_v3(value) + return isinstance(value, dict) and not validate_group_metadata_v3( + cast("dict[object, object]", value) + ) def parse_group_metadata_v3(value: object) -> ZarrV3GroupMetadataJSON: """Return `value` narrowed to `ZarrV3GroupMetadataJSON`, or raise `MetadataValidationError`.""" - problems = validate_group_metadata_v3(value) + normalized = arrays_to_tuples(value) + problems = validate_group_metadata_v3(normalized) if problems: raise MetadataValidationError(problems) - return cast(ZarrV3GroupMetadataJSON, value) + return cast(ZarrV3GroupMetadataJSON, normalized) def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: @@ -702,6 +750,9 @@ def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) problems: list[ValidationProblem] = _missing_keys(GROUP_METADATA_REQUIRED_KEYS_V2, doc) + problems.extend( + _unexpected_keys(GROUP_METADATA_STANDARD_KEYS_V2, cast("Mapping[object, object]", value)) + ) problems.extend(_check_literal(doc, "zarr_format", 2)) if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) @@ -710,15 +761,23 @@ def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: def is_group_metadata_v2(value: object) -> TypeIs[ZarrV2GroupMetadataJSON]: """Whether `value` is a structurally-valid v2 group metadata document.""" - return not validate_group_metadata_v2(value) + return isinstance(value, dict) and not validate_group_metadata_v2( + cast("dict[object, object]", value) + ) def parse_group_metadata_v2(value: object) -> ZarrV2GroupMetadataJSON: """Return `value` narrowed to `ZarrV2GroupMetadataJSON`, or raise `MetadataValidationError`.""" - problems = validate_group_metadata_v2(value) + normalized = arrays_to_tuples(value) + problems = validate_group_metadata_v2(normalized) if problems: raise MetadataValidationError(problems) - return cast(ZarrV2GroupMetadataJSON, value) + return cast(ZarrV2GroupMetadataJSON, normalized) + + +def _reject_json_constant(constant: str) -> NoReturn: + """Reject the JavaScript constants accepted by Python's JSON decoder.""" + raise ValueError(f"non-standard JSON constant {constant!r}") def load_store_json(mapping: Mapping[str, bytes], key: str) -> Any: @@ -734,23 +793,35 @@ def load_store_json(mapping: Mapping[str, bytes], key: str) -> Any: [ValidationProblem((key,), "missing store key", "missing_key")] ) try: - return json.loads(mapping[key]) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: + return json.loads(mapping[key], parse_constant=_reject_json_constant) + except (UnicodeDecodeError, ValueError) as exc: raise MetadataValidationError( [ValidationProblem((key,), f"invalid JSON: {exc}", "invalid_json")] ) from exc +def dump_store_json(value: object, *, indent: int | str | None = None) -> bytes: + """Encode a metadata document as strict RFC 8259 JSON bytes.""" + return json.dumps(value, indent=indent, allow_nan=False).encode("utf-8") + + def arrays_to_tuples(obj: object) -> object: - """Recursively convert every list in a JSON-decoded structure to a tuple.""" - if isinstance(obj, list): - return tuple(arrays_to_tuples(item) for item in cast("list[object]", obj)) - if isinstance(obj, dict): - mapping = cast("dict[object, object]", obj) + """Recursively materialize mappings and convert array-like values to tuples.""" + if isinstance(obj, Sequence) and not isinstance(obj, (str, bytes, bytearray)): + sequence = cast("Sequence[object]", obj) + converted_sequence = tuple(arrays_to_tuples(item) for item in sequence) + if isinstance(obj, tuple) and all( + converted is original + for converted, original in zip(converted_sequence, sequence, strict=True) + ): + return cast("tuple[object, ...]", obj) + return converted_sequence + if isinstance(obj, Mapping): + mapping = cast("Mapping[object, object]", obj) converted: dict[object, object] = { key: arrays_to_tuples(value) for key, value in mapping.items() } - if all(converted[key] is value for key, value in mapping.items()): + if isinstance(obj, dict) and all(converted[key] is value for key, value in mapping.items()): return cast("object", obj) return converted return obj diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index f3bd4e75db..048bbe766f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -30,9 +30,16 @@ class ArrayManifest(BaseModel): from typing import TYPE_CHECKING, Annotated, TypeVar -from pydantic import BeforeValidator, InstanceOf, PlainSerializer, WithJsonSchema +from pydantic import BeforeValidator, InstanceOf, PlainSerializer from zarr_metadata import model as _model +from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON +from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON +from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON if TYPE_CHECKING: from collections.abc import Callable @@ -51,68 +58,85 @@ def coerce(value: object) -> _M: return coerce -# JSON schemas describe the DOCUMENT form each field accepts (the validation -# input), not the in-memory model shape. -_DOCUMENT_SCHEMA = {"type": "object"} -_FIELD_SCHEMA = {"anyOf": [{"type": "string"}, {"type": "object"}]} - ZarrV3ArrayMetadata = Annotated[ InstanceOf[_model.ZarrV3ArrayMetadata], - BeforeValidator(_coerce_to(_model.ZarrV3ArrayMetadata, _model.ZarrV3ArrayMetadata.from_json)), - PlainSerializer(_model.ZarrV3ArrayMetadata.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV3ArrayMetadata"}), + BeforeValidator( + _coerce_to(_model.ZarrV3ArrayMetadata, _model.ZarrV3ArrayMetadata.from_json), + json_schema_input_type=ZarrV3ArrayMetadataJSON, + ), + PlainSerializer(_model.ZarrV3ArrayMetadata.to_json, return_type=ZarrV3ArrayMetadataJSON), ] """Field type for a v3 array metadata document (`zarr.json` content).""" ZarrV2ArrayMetadata = Annotated[ InstanceOf[_model.ZarrV2ArrayMetadata], - BeforeValidator(_coerce_to(_model.ZarrV2ArrayMetadata, _model.ZarrV2ArrayMetadata.from_json)), - PlainSerializer(_model.ZarrV2ArrayMetadata.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV2ArrayMetadata"}), + BeforeValidator( + _coerce_to(_model.ZarrV2ArrayMetadata, _model.ZarrV2ArrayMetadata.from_json), + json_schema_input_type=ZarrV2ArrayMetadataJSON, + ), + PlainSerializer(_model.ZarrV2ArrayMetadata.to_json, return_type=ZarrV2ArrayMetadataJSON), ] """Field type for a v2 array metadata document (merged `.zarray` + `.zattrs` form).""" ZarrV3GroupMetadata = Annotated[ InstanceOf[_model.ZarrV3GroupMetadata], - BeforeValidator(_coerce_to(_model.ZarrV3GroupMetadata, _model.ZarrV3GroupMetadata.from_json)), - PlainSerializer(_model.ZarrV3GroupMetadata.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV3GroupMetadata"}), + BeforeValidator( + _coerce_to(_model.ZarrV3GroupMetadata, _model.ZarrV3GroupMetadata.from_json), + json_schema_input_type=ZarrV3GroupMetadataJSON, + ), + PlainSerializer(_model.ZarrV3GroupMetadata.to_json, return_type=ZarrV3GroupMetadataJSON), ] """Field type for a v3 group metadata document (`zarr.json` content).""" ZarrV2GroupMetadata = Annotated[ InstanceOf[_model.ZarrV2GroupMetadata], - BeforeValidator(_coerce_to(_model.ZarrV2GroupMetadata, _model.ZarrV2GroupMetadata.from_json)), - PlainSerializer(_model.ZarrV2GroupMetadata.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV2GroupMetadata"}), + BeforeValidator( + _coerce_to(_model.ZarrV2GroupMetadata, _model.ZarrV2GroupMetadata.from_json), + json_schema_input_type=ZarrV2GroupMetadataJSON, + ), + PlainSerializer(_model.ZarrV2GroupMetadata.to_json, return_type=ZarrV2GroupMetadataJSON), ] """Field type for a v2 group metadata document (merged `.zgroup` + `.zattrs` form).""" ZarrV3ConsolidatedMetadata = Annotated[ InstanceOf[_model.ZarrV3ConsolidatedMetadata], BeforeValidator( - _coerce_to(_model.ZarrV3ConsolidatedMetadata, _model.ZarrV3ConsolidatedMetadata.from_json) + _coerce_to( + _model.ZarrV3ConsolidatedMetadata, + _model.ZarrV3ConsolidatedMetadata.from_json, + ), + json_schema_input_type=ZarrV3ConsolidatedMetadataJSON, + ), + PlainSerializer( + _model.ZarrV3ConsolidatedMetadata.to_json, + return_type=ZarrV3ConsolidatedMetadataJSON, ), - PlainSerializer(_model.ZarrV3ConsolidatedMetadata.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV3ConsolidatedMetadata"}), ] """Field type for v3 inline consolidated metadata.""" ZarrV2ConsolidatedMetadata = Annotated[ InstanceOf[_model.ZarrV2ConsolidatedMetadata], BeforeValidator( - _coerce_to(_model.ZarrV2ConsolidatedMetadata, _model.ZarrV2ConsolidatedMetadata.from_json) + _coerce_to( + _model.ZarrV2ConsolidatedMetadata, + _model.ZarrV2ConsolidatedMetadata.from_json, + ), + json_schema_input_type=ZarrV2ConsolidatedMetadataJSON, + ), + PlainSerializer( + _model.ZarrV2ConsolidatedMetadata.to_json, + return_type=ZarrV2ConsolidatedMetadataJSON, ), - PlainSerializer(_model.ZarrV2ConsolidatedMetadata.to_json, return_type=dict), - WithJsonSchema(_DOCUMENT_SCHEMA | {"title": "ZarrV2ConsolidatedMetadata"}), ] """Field type for a v2 `.zmetadata` document.""" ZarrV3MetadataField = Annotated[ InstanceOf[_model.ZarrV3NamedConfig], - BeforeValidator(_coerce_to(_model.ZarrV3NamedConfig, _model.ZarrV3NamedConfig.from_json)), - PlainSerializer(_model.ZarrV3NamedConfig.to_json, return_type=str | dict), - WithJsonSchema(_FIELD_SCHEMA | {"title": "ZarrV3MetadataField"}), + BeforeValidator( + _coerce_to(_model.ZarrV3NamedConfig, _model.ZarrV3NamedConfig.from_json), + json_schema_input_type=ZarrV3MetadataFieldJSON, + ), + PlainSerializer(_model.ZarrV3NamedConfig.to_json, return_type=ZarrV3MetadataFieldJSON), ] """Field type for one normalized v3 metadata extension envelope.""" diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index f84dac40c9..571e4beebf 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -2,6 +2,7 @@ import dataclasses import json +from collections import UserDict from collections.abc import Callable from typing import TYPE_CHECKING @@ -787,8 +788,9 @@ def test_v2_from_json_nested_arrays_in_attributes_become_tuples() -> None: Expect(None, frozenset(), id="none"), Expect({"a": [1, {"b": None}], "c": "x"}, frozenset(), id="nested-containers"), Expect((1, 2, 3), frozenset(), id="tuple-array"), - Expect(float("nan"), frozenset(), id="nan"), - Expect(float("inf"), frozenset(), id="inf"), + Expect(float("nan"), frozenset({()}), id="nan"), + Expect(float("inf"), frozenset({()}), id="inf"), + Expect(float("-inf"), frozenset({()}), id="negative-inf"), Expect(object(), frozenset({()}), id="object"), Expect(b"abc", frozenset({()}), id="bytes"), Expect(bytearray(b"abc"), frozenset({()}), id="bytearray"), @@ -816,12 +818,36 @@ def test_validate_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) - def test_parse_json(case: Expect[object, frozenset[tuple[str | int, ...]]]) -> None: """parse_json returns valid JSON values and raises on invalid ones.""" if case.output == frozenset(): - assert parse_json(case.input) is case.input + parsed = parse_json(case.input) + assert arrays_to_tuples(parsed) == arrays_to_tuples(case.input) else: with pytest.raises(MetadataValidationError): parse_json(case.input) +def test_parse_json_materializes_abstract_containers() -> None: + """Accepted Mapping and Sequence values normalize to JSON encoder containers.""" + value = UserDict({"values": range(3)}) + + parsed = parse_json(value) + + assert parsed == {"values": (0, 1, 2)} + assert type(parsed) is dict + assert type(parsed["values"]) is tuple + json.dumps(parsed, allow_nan=False) + + +def test_parse_metadata_field_materializes_abstract_containers() -> None: + """Named-config parsing produces canonical containers at every nesting level.""" + value = UserDict({"name": "example", "configuration": UserDict({"values": range(2)})}) + + parsed = parse_metadata_field_v3(value) + + assert isinstance(parsed, dict) + assert parsed == {"name": "example", "configuration": {"values": (0, 1)}} + assert type(parsed["configuration"]) is dict + + def test_validate_json_reports_json_in_message() -> None: """validate_json's message for a non-JSON value mentions JSON.""" problems = validate_json(object()) @@ -1180,6 +1206,66 @@ def test_v3_zarr_format_literal_enforced() -> None: assert [(p.loc, p.kind) for p in problems] == [(("zarr_format",), "invalid_value")] +@pytest.mark.parametrize( + ("document", "validate"), + [ + pytest.param( + dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"zarr_format": 2.0}, + validate_array_metadata_v2, + id="v2", + ), + pytest.param( + dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"zarr_format": 3.0}, + validate_array_metadata_v3, + id="v3", + ), + ], +) +def test_array_zarr_format_rejects_float( + document: object, validate: Callable[[object], list[ValidationProblem]] +) -> None: + """Integer-valued floats do not satisfy integer format literals.""" + assert [(p.loc, p.kind) for p in validate(document)] == [(("zarr_format",), "invalid_value")] + + +def test_array_v2_rejects_unknown_document_member() -> None: + """The closed v2 merged-document shape rejects undeclared members.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"unexpected": 1} + + assert [(p.loc, p.kind) for p in validate_array_metadata_v2(doc)] == [ + (("unexpected",), "invalid_value") + ] + + +def test_array_v3_from_json_materializes_abstract_containers() -> None: + """A flexible input mapping becomes the canonical dict/tuple model shape.""" + doc = UserDict(dict(ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json())) + doc["shape"] = range(2) + + model = ZarrV3ArrayMetadata.from_json(doc) + + assert model.shape == (0, 1) + assert type(model.shape) is tuple + + +def test_from_key_value_rejects_non_standard_json_constant() -> None: + """Store JSON decoding rejects JavaScript NaN/Infinity constants.""" + doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) + doc["fill_value"] = float("nan") + raw = json.dumps(doc) + + with pytest.raises(MetadataValidationError, match="invalid JSON"): + ZarrV3ArrayMetadata.from_key_value({"zarr.json": raw.encode()}) + + +def test_to_key_value_rejects_non_finite_model_value() -> None: + """Strict encoding prevents directly-constructed models from writing invalid JSON.""" + model = ZarrV3ArrayMetadata.create_default(fill_value=float("nan")) + + with pytest.raises(ValueError, match="JSON compliant"): + model.to_key_value() + + def test_v3_node_type_literal_enforced() -> None: """A v3 array document claiming node_type 'group' is rejected.""" doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"node_type": "group"} diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 0ce89180c9..52b27ac131 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -2,6 +2,8 @@ import dataclasses import json +from collections import UserDict +from collections.abc import Callable import pytest @@ -17,6 +19,7 @@ ) from zarr_metadata.model._validation import ( MetadataValidationError, + ValidationProblem, parse_group_metadata_v2, parse_group_metadata_v3, validate_group_metadata_v2, @@ -98,6 +101,56 @@ def test_group_v3_bad_attributes() -> None: parse_group_metadata_v3({"zarr_format": 3, "node_type": "group", "attributes": 5}) +@pytest.mark.parametrize( + ("document", "validate"), + [ + pytest.param( + {"zarr_format": 2.0}, + validate_group_metadata_v2, + id="v2", + ), + pytest.param( + {"zarr_format": 3.0, "node_type": "group"}, + validate_group_metadata_v3, + id="v3", + ), + ], +) +def test_group_zarr_format_rejects_float( + document: object, validate: Callable[[object], list[ValidationProblem]] +) -> None: + """Integer-valued floats do not satisfy integer format literals.""" + assert [(p.loc, p.kind) for p in validate(document)] == [(("zarr_format",), "invalid_value")] + + +def test_group_v2_rejects_unknown_document_member() -> None: + """The closed v2 merged-document shape rejects undeclared members.""" + assert [(p.loc, p.kind) for p in validate_group_metadata_v2({"zarr_format": 2, "x": 1})] == [ + (("x",), "invalid_value") + ] + + +@pytest.mark.parametrize( + ("parse", "document"), + [ + pytest.param(parse_group_metadata_v2, {"zarr_format": 2}, id="v2"), + pytest.param( + parse_group_metadata_v3, + {"zarr_format": 3, "node_type": "group"}, + id="v3", + ), + ], +) +def test_group_parser_materializes_abstract_mapping( + parse: Callable[[object], object], document: dict[str, object] +) -> None: + """A successful group parser always returns the declared concrete TypedDict shape.""" + parsed = parse(UserDict(document)) + + assert type(parsed) is dict + assert parsed == document + + def test_group_v3_extension_fields_are_validated() -> None: """Group extension payloads must be JSON values with a must-understand flag.""" doc = { @@ -396,6 +449,27 @@ def test_group_v3_valid_consolidated_passes_validator() -> None: assert validate_group_metadata_v3(doc) == [] +def test_v3_consolidated_rejects_unknown_envelope_member() -> None: + """The inline consolidated envelope is closed and never drops accepted members.""" + doc = { + "kind": "inline", + "must_understand": False, + "metadata": {}, + "unexpected": 1, + } + + with pytest.raises(MetadataValidationError, match="unexpected"): + ZarrV3ConsolidatedMetadata.from_json(doc) + + +def test_v2_consolidated_rejects_unknown_document_member() -> None: + """The v2 consolidated document is closed and never drops accepted members.""" + doc = {"zarr_consolidated_format": 1, "metadata": {}, "unexpected": 1} + + with pytest.raises(MetadataValidationError, match="unexpected"): + ZarrV2ConsolidatedMetadata.from_json(doc) + + # --- must_understand partition ------------------------------------------------ diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 4ee72da16d..9010344d5a 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -101,8 +101,27 @@ class Manifest(BaseModel): codec: zmp.ZarrV3MetadataField schema = Manifest.model_json_schema() - assert schema["properties"]["metadata"] == {"type": "object", "title": "ZarrV3ArrayMetadata"} - assert schema["properties"]["codec"]["anyOf"] == [{"type": "string"}, {"type": "object"}] + metadata_schema = schema["$defs"]["ZarrV3ArrayMetadataJSON"] + assert schema["properties"]["metadata"]["$ref"] == "#/$defs/ZarrV3ArrayMetadataJSON" + assert metadata_schema["required"] == [ + "zarr_format", + "node_type", + "data_type", + "shape", + "chunk_grid", + "chunk_key_encoding", + "fill_value", + "codecs", + ] + assert metadata_schema["properties"]["zarr_format"] == { + "const": 3, + "title": "Zarr Format", + "type": "integer", + } + assert schema["properties"]["codec"]["anyOf"] == [ + {"type": "string"}, + {"$ref": "#/$defs/ZarrV3NamedConfigJSON"}, + ] def test_json_roundtrip() -> None: From 15f9b5807ca445a5f813dccc6c2b07005066edba Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 17:32:14 +0200 Subject: [PATCH 56/70] docs(metadata): record review fix design Assisted-by: Codex:gpt-5 --- ...07-22-zarr-metadata-review-fixes-design.md | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-22-zarr-metadata-review-fixes-design.md diff --git a/docs/superpowers/specs/2026-07-22-zarr-metadata-review-fixes-design.md b/docs/superpowers/specs/2026-07-22-zarr-metadata-review-fixes-design.md new file mode 100644 index 0000000000..c169a822d9 --- /dev/null +++ b/docs/superpowers/specs/2026-07-22-zarr-metadata-review-fixes-design.md @@ -0,0 +1,118 @@ +# Zarr Metadata Review Fixes Design + +## Goal + +Correct the significant conformance and public-contract defects found by the +independent branch review without changing the documented ability to construct +temporarily invalid model instances for metadata repair workflows. + +## Scope + +This change fixes four areas: + +1. Validation of raw Zarr v2 `.zarray` and `.zgroup` documents must be distinct + from validation of the library's merged in-memory document representation. +2. Public `TypeIs` predicates must narrow only values that actually inhabit the + type named in their return annotation. +3. The public Zarr v2 dtype type must represent nested structured dtypes. +4. Pydantic-generated JSON schemas must encode the structural constraints that + the corresponding runtime parsers enforce. + +The inaccurate consolidated-metadata docstring is minor and outside this fix. +Existing model-instance pass-through in the Pydantic integration remains +unchanged because it is part of the documented repair-state design. + +## Raw V2 Documents and Merged Models + +The public `ZarrV2ArrayMetadataJSON` and `ZarrV2GroupMetadataJSON` types describe +an in-memory representation in which an optional `attributes` member has been +merged from `.zattrs`. They do not describe the raw `.zarray` and `.zgroup` +objects stored on disk. + +Private raw-document parsers will enforce the storage specification before any +merge occurs: + +- `.zarray` accepts exactly the required array metadata members plus the + optional `dimension_separator` member. Every other member, including + `attributes`, is an error. +- `.zgroup` accepts exactly `zarr_format`. Every other member, including + `attributes`, is an error. +- `.zattrs`, when present, must be a JSON object with string keys and JSON + values. It is then added as the merged `attributes` member. +- An absent `.zattrs` remains distinguishable from an explicitly empty object, + preserving the existing `UNSET` behavior. + +The existing public merged-document validators remain strict. This avoids +silently discarding data and keeps their declared `TypedDict` contracts sound. + +## Sound Type Guards + +Validation and parsing intentionally accept abstract `Mapping` and `Sequence` +inputs. Parsers materialize those inputs into canonical dictionaries and +tuples. Type guards cannot make the same promise because `TypeIs[T]` asserts +that the original object already inhabits `T`. + +`is_json` will therefore require recursively canonical JSON container types: +`dict`, `list`, and `tuple`, with string dictionary keys and canonical children. +It will reject abstract containers such as `UserDict` and `range`, even though +`parse_json` continues to accept and normalize them. + +`is_metadata_field_v3` will likewise require either a string or a concrete +dictionary whose nested values are already canonical. Parsing continues to +accept abstract mappings and materialize them. + +## Recursive V2 Dtype Type + +`ZarrV2DataTypeMetadata` will be a named recursive alias. A structured field's +datatype may be either a dtype string or another structured dtype tuple. The +optional subarray shape remains a tuple of integers. Runtime validation already +accepts this structure, so this change aligns static typing and generated schema +with existing spec-compliant behavior. + +## Pydantic JSON Schema Parity + +The Pydantic field aliases will retain their current runtime behavior: raw +documents are parsed through the core model constructors, and existing core +model instances pass through unchanged. + +Their JSON schema input definitions will be replaced or annotated with +constraints that express the runtime document rules, including: + +- non-negative shape and chunk dimensions; +- at least one v3 codec; +- closed named-configuration objects; +- closed v2 merged documents and consolidated payloads where runtime parsing is + closed; +- `must_understand` constraints at mandatory v3 extension points; and +- the recursive v2 structured-dtype representation. + +Constraints that depend on semantic interpretation outside this package's +structural validators will not be added. The generated schema should describe +accepted raw document inputs, not the intentionally permissive state of an +already-constructed core model instance. + +## Error Handling + +All new raw-v2 failures use `MetadataValidationError` and preserve precise +locations. An illegal `.zarray` member named `attributes`, for example, reports +the location `("attributes",)` and the existing `invalid_value` kind. +Malformed `.zattrs` values report locations beneath `attributes` after the +merge boundary, consistent with the public merged representation. + +## Testing + +Implementation follows red-green-refactor cycles. Regression coverage will +prove: + +- raw `.zarray` and `.zgroup` reject `attributes` and other extra members; +- valid sibling `.zattrs` still merges, including absent-versus-empty behavior; +- `is_json(range(3))` and `is_metadata_field_v3(UserDict(...))` are false while + the corresponding parsers still normalize those inputs; +- nested structured v2 dtypes type-check through Pydantic and validate at + runtime; and +- generated schemas reject representative inputs that runtime parsing rejects: + empty codecs, negative dimensions, forbidden extra members, and unsupported + `must_understand: false` values. + +Focused package tests, formatting, linting, type checking where configured, and +the full repository test suite will run before the implementation is committed. From d1c6dda3ea7094b318a79443210145f71667bc99 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 18:10:42 +0200 Subject: [PATCH 57/70] fix(metadata): align validation and schemas Assisted-by: Codex:gpt-5 --- .../src/zarr_metadata/_pydantic_schema.py | 119 ++++++++++++++++++ .../src/zarr_metadata/model/_array.py | 10 ++ .../src/zarr_metadata/model/_group.py | 10 ++ .../src/zarr_metadata/model/_validation.py | 23 +++- .../src/zarr_metadata/pydantic.py | 67 ++++++---- .../src/zarr_metadata/v2/array.py | 11 +- .../zarr-metadata/tests/model/test_array.py | 29 +++++ .../zarr-metadata/tests/model/test_group.py | 13 ++ .../tests/model/test_pydantic_module.py | 84 +++++++++++++ 9 files changed, 339 insertions(+), 27 deletions(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py diff --git a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py new file mode 100644 index 0000000000..e7f94262c9 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py @@ -0,0 +1,119 @@ +"""Private input types used only to generate accurate Pydantic JSON schemas.""" + +from __future__ import annotations + +from collections.abc import Mapping # noqa: TC003 # resolved by Pydantic at runtime +from typing import Annotated, Literal, NotRequired + +from pydantic import ConfigDict, Field, with_config +from typing_extensions import TypedDict + +from zarr_metadata._common import JSONValue +from zarr_metadata.v2.array import ( # noqa: TC001 # resolved by Pydantic at runtime + ZarrV2DataTypeMetadata, +) +from zarr_metadata.v2.codec import ( # noqa: TC001 # resolved by Pydantic at runtime + ZarrV2CodecMetadata, +) + +NonNegativeInt = Annotated[int, Field(ge=0)] + + +@with_config(ConfigDict(extra="forbid")) +class ZarrV3NamedConfigJSON(TypedDict): + """Closed v3 named configuration accepted at optional extension points.""" + + name: str + configuration: NotRequired[Mapping[str, JSONValue]] + must_understand: NotRequired[bool] + + +@with_config(ConfigDict(extra="forbid")) +class ZarrV3MandatoryNamedConfigJSON(TypedDict): + """Closed named configuration accepted where understanding is mandatory.""" + + name: str + configuration: NotRequired[Mapping[str, JSONValue]] + must_understand: NotRequired[Literal[True]] + + +ZarrV3MetadataFieldJSON = str | ZarrV3NamedConfigJSON +ZarrV3MandatoryMetadataFieldJSON = str | ZarrV3MandatoryNamedConfigJSON +ZarrV3CodecPipelineJSON = Annotated[tuple[ZarrV3MetadataFieldJSON, ...], Field(min_length=1)] + + +class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=JSONValue): + """Schema input for a v3 array document, including arbitrary extensions.""" + + zarr_format: Literal[3] + node_type: Literal["array"] + data_type: ZarrV3MandatoryMetadataFieldJSON + shape: tuple[NonNegativeInt, ...] + chunk_grid: ZarrV3MandatoryMetadataFieldJSON + chunk_key_encoding: ZarrV3MandatoryMetadataFieldJSON + fill_value: JSONValue + codecs: ZarrV3CodecPipelineJSON + attributes: NotRequired[Mapping[str, JSONValue]] + storage_transformers: NotRequired[tuple[ZarrV3MetadataFieldJSON, ...]] + dimension_names: NotRequired[tuple[str | None, ...]] + + +@with_config(ConfigDict(extra="forbid")) +class ZarrV3ConsolidatedMetadataJSON(TypedDict): + """Schema input for the closed inline consolidated-metadata envelope.""" + + kind: Literal["inline"] + must_understand: Literal[False] + metadata: Mapping[str, ZarrV3ArrayMetadataJSON | ZarrV3GroupMetadataJSON] + + +class ZarrV3GroupMetadataJSON(TypedDict, extra_items=JSONValue): + """Schema input for a v3 group document, including arbitrary extensions.""" + + zarr_format: Literal[3] + node_type: Literal["group"] + attributes: NotRequired[Mapping[str, JSONValue]] + consolidated_metadata: NotRequired[ZarrV3ConsolidatedMetadataJSON | None] + + +@with_config(ConfigDict(extra="forbid")) +class ZarrV2ArrayMetadataJSON(TypedDict): + """Schema input for the closed, merged v2 array representation.""" + + zarr_format: Literal[2] + shape: tuple[NonNegativeInt, ...] + chunks: tuple[NonNegativeInt, ...] + dtype: ZarrV2DataTypeMetadata + compressor: ZarrV2CodecMetadata | None + fill_value: JSONValue + order: Literal["C", "F"] + filters: tuple[ZarrV2CodecMetadata, ...] | None + dimension_separator: NotRequired[Literal[".", "/"]] + attributes: NotRequired[Mapping[str, JSONValue]] + + +@with_config(ConfigDict(extra="forbid")) +class ZarrV2GroupMetadataJSON(TypedDict): + """Schema input for the closed, merged v2 group representation.""" + + zarr_format: Literal[2] + attributes: NotRequired[Mapping[str, JSONValue]] + + +@with_config(ConfigDict(extra="forbid")) +class ZarrV2ConsolidatedMetadataJSON(TypedDict): + """Schema input matching the v2 consolidated model's structural parser.""" + + zarr_consolidated_format: Literal[1] + metadata: Mapping[str, JSONValue] + + +__all__ = [ + "ZarrV2ArrayMetadataJSON", + "ZarrV2ConsolidatedMetadataJSON", + "ZarrV2GroupMetadataJSON", + "ZarrV3ArrayMetadataJSON", + "ZarrV3ConsolidatedMetadataJSON", + "ZarrV3GroupMetadataJSON", + "ZarrV3MetadataFieldJSON", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 14ca92e8fc..bc6f2592f1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -460,6 +460,16 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ArrayMetadata: if not isinstance(zarray_raw, Mapping): return cls.from_json(zarray_raw) zarray = cast("Mapping[str, object]", zarray_raw) + if "attributes" in zarray: + raise MetadataValidationError( + [ + ValidationProblem( + ("attributes",), + "unexpected document member", + "invalid_value", + ) + ] + ) if ATTRIBUTES_STORE_KEY_V2 in mapping: zattrs = cast("object", load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2)) return cls.from_json({**zarray, "attributes": zattrs}) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 39c4da5b1e..811cc7b81e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -319,6 +319,16 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2GroupMetadata: if not isinstance(zgroup_raw, Mapping): return cls.from_json(zgroup_raw) zgroup = cast("Mapping[str, object]", zgroup_raw) + if "attributes" in zgroup: + raise MetadataValidationError( + [ + ValidationProblem( + ("attributes",), + "unexpected document member", + "invalid_value", + ) + ] + ) if ATTRIBUTES_STORE_KEY_V2 in mapping: zattrs = cast("object", load_store_json(mapping, ATTRIBUTES_STORE_KEY_V2)) return cls.from_json({**zgroup, "attributes": zattrs}) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 5e8f669a1c..3c3e4a1fea 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -100,9 +100,22 @@ def validate_json(value: object) -> list[ValidationProblem]: return [ValidationProblem((), f"not a JSON-serializable value: {value!r}", "invalid_type")] +def _is_canonical_json(value: object) -> TypeIs[JSONValue]: + """Whether `value` already uses the concrete containers in `JSONValue`.""" + if isinstance(value, float): + return math.isfinite(value) + if isinstance(value, (str, int, bool)) or value is None: + return True + if isinstance(value, (list, tuple)): + return all(_is_canonical_json(item) for item in value) + if isinstance(value, dict): + return all(isinstance(key, str) and _is_canonical_json(item) for key, item in value.items()) + return False + + def is_json(value: object) -> TypeIs[JSONValue]: - """Whether `value` is a JSON-serializable structure (recursively).""" - return not validate_json(value) + """Whether `value` is a canonical JSON structure (recursively).""" + return _is_canonical_json(value) def parse_json(value: object) -> JSONValue: @@ -281,7 +294,11 @@ def validate_metadata_field_v3( def is_metadata_field_v3(value: object) -> TypeIs[ZarrV3MetadataFieldJSON]: """Whether `value` is a v3 metadata field: a bare name or a named config.""" - return not validate_metadata_field_v3(value) + return isinstance(value, str) or ( + isinstance(value, dict) + and _is_canonical_json(value) + and not validate_metadata_field_v3(value) + ) def parse_metadata_field_v3(value: object) -> ZarrV3MetadataFieldJSON: diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index 048bbe766f..8584efa570 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -33,13 +33,38 @@ class ArrayManifest(BaseModel): from pydantic import BeforeValidator, InstanceOf, PlainSerializer from zarr_metadata import model as _model -from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON -from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON -from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON -from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON -from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON -from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON -from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON +from zarr_metadata._pydantic_schema import ( + ZarrV2ArrayMetadataJSON as _ZarrV2ArrayMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV2ConsolidatedMetadataJSON as _ZarrV2ConsolidatedMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV2GroupMetadataJSON as _ZarrV2GroupMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV3ArrayMetadataJSON as _ZarrV3ArrayMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV3ConsolidatedMetadataJSON as _ZarrV3ConsolidatedMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV3GroupMetadataJSON as _ZarrV3GroupMetadataSchema, +) +from zarr_metadata._pydantic_schema import ( + ZarrV3MetadataFieldJSON as _ZarrV3MetadataFieldSchema, +) +from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON as _ZarrV2ArrayMetadataJSON +from zarr_metadata.v2.consolidated import ( + ZarrV2ConsolidatedMetadataJSON as _ZarrV2ConsolidatedMetadataJSON, +) +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON as _ZarrV2GroupMetadataJSON +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON as _ZarrV3MetadataFieldJSON +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON as _ZarrV3ArrayMetadataJSON +from zarr_metadata.v3.consolidated import ( + ZarrV3ConsolidatedMetadataJSON as _ZarrV3ConsolidatedMetadataJSON, +) +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON as _ZarrV3GroupMetadataJSON if TYPE_CHECKING: from collections.abc import Callable @@ -62,9 +87,9 @@ def coerce(value: object) -> _M: InstanceOf[_model.ZarrV3ArrayMetadata], BeforeValidator( _coerce_to(_model.ZarrV3ArrayMetadata, _model.ZarrV3ArrayMetadata.from_json), - json_schema_input_type=ZarrV3ArrayMetadataJSON, + json_schema_input_type=_ZarrV3ArrayMetadataSchema, ), - PlainSerializer(_model.ZarrV3ArrayMetadata.to_json, return_type=ZarrV3ArrayMetadataJSON), + PlainSerializer(_model.ZarrV3ArrayMetadata.to_json, return_type=_ZarrV3ArrayMetadataJSON), ] """Field type for a v3 array metadata document (`zarr.json` content).""" @@ -72,9 +97,9 @@ def coerce(value: object) -> _M: InstanceOf[_model.ZarrV2ArrayMetadata], BeforeValidator( _coerce_to(_model.ZarrV2ArrayMetadata, _model.ZarrV2ArrayMetadata.from_json), - json_schema_input_type=ZarrV2ArrayMetadataJSON, + json_schema_input_type=_ZarrV2ArrayMetadataSchema, ), - PlainSerializer(_model.ZarrV2ArrayMetadata.to_json, return_type=ZarrV2ArrayMetadataJSON), + PlainSerializer(_model.ZarrV2ArrayMetadata.to_json, return_type=_ZarrV2ArrayMetadataJSON), ] """Field type for a v2 array metadata document (merged `.zarray` + `.zattrs` form).""" @@ -82,9 +107,9 @@ def coerce(value: object) -> _M: InstanceOf[_model.ZarrV3GroupMetadata], BeforeValidator( _coerce_to(_model.ZarrV3GroupMetadata, _model.ZarrV3GroupMetadata.from_json), - json_schema_input_type=ZarrV3GroupMetadataJSON, + json_schema_input_type=_ZarrV3GroupMetadataSchema, ), - PlainSerializer(_model.ZarrV3GroupMetadata.to_json, return_type=ZarrV3GroupMetadataJSON), + PlainSerializer(_model.ZarrV3GroupMetadata.to_json, return_type=_ZarrV3GroupMetadataJSON), ] """Field type for a v3 group metadata document (`zarr.json` content).""" @@ -92,9 +117,9 @@ def coerce(value: object) -> _M: InstanceOf[_model.ZarrV2GroupMetadata], BeforeValidator( _coerce_to(_model.ZarrV2GroupMetadata, _model.ZarrV2GroupMetadata.from_json), - json_schema_input_type=ZarrV2GroupMetadataJSON, + json_schema_input_type=_ZarrV2GroupMetadataSchema, ), - PlainSerializer(_model.ZarrV2GroupMetadata.to_json, return_type=ZarrV2GroupMetadataJSON), + PlainSerializer(_model.ZarrV2GroupMetadata.to_json, return_type=_ZarrV2GroupMetadataJSON), ] """Field type for a v2 group metadata document (merged `.zgroup` + `.zattrs` form).""" @@ -105,11 +130,11 @@ def coerce(value: object) -> _M: _model.ZarrV3ConsolidatedMetadata, _model.ZarrV3ConsolidatedMetadata.from_json, ), - json_schema_input_type=ZarrV3ConsolidatedMetadataJSON, + json_schema_input_type=_ZarrV3ConsolidatedMetadataSchema, ), PlainSerializer( _model.ZarrV3ConsolidatedMetadata.to_json, - return_type=ZarrV3ConsolidatedMetadataJSON, + return_type=_ZarrV3ConsolidatedMetadataJSON, ), ] """Field type for v3 inline consolidated metadata.""" @@ -121,11 +146,11 @@ def coerce(value: object) -> _M: _model.ZarrV2ConsolidatedMetadata, _model.ZarrV2ConsolidatedMetadata.from_json, ), - json_schema_input_type=ZarrV2ConsolidatedMetadataJSON, + json_schema_input_type=_ZarrV2ConsolidatedMetadataSchema, ), PlainSerializer( _model.ZarrV2ConsolidatedMetadata.to_json, - return_type=ZarrV2ConsolidatedMetadataJSON, + return_type=_ZarrV2ConsolidatedMetadataJSON, ), ] """Field type for a v2 `.zmetadata` document.""" @@ -134,9 +159,9 @@ def coerce(value: object) -> _M: InstanceOf[_model.ZarrV3NamedConfig], BeforeValidator( _coerce_to(_model.ZarrV3NamedConfig, _model.ZarrV3NamedConfig.from_json), - json_schema_input_type=ZarrV3MetadataFieldJSON, + json_schema_input_type=_ZarrV3MetadataFieldSchema, ), - PlainSerializer(_model.ZarrV3NamedConfig.to_json, return_type=ZarrV3MetadataFieldJSON), + PlainSerializer(_model.ZarrV3NamedConfig.to_json, return_type=_ZarrV3MetadataFieldJSON), ] """Field type for one normalized v3 metadata extension envelope.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/array.py b/packages/zarr-metadata/src/zarr_metadata/v2/array.py index 741f37acba..28085e4fbb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/array.py @@ -3,18 +3,23 @@ from collections.abc import Mapping from typing import Final, Literal, NotRequired -from typing_extensions import TypedDict +from typing_extensions import TypeAliasType, TypedDict from zarr_metadata._common import JSONValue from zarr_metadata.v2.codec import ZarrV2CodecMetadata -ZarrV2DataTypeMetadata = str | tuple[tuple[str, str] | tuple[str, str, tuple[int, ...]], ...] +ZarrV2DataTypeMetadata = TypeAliasType( + "ZarrV2DataTypeMetadata", + "str | tuple[tuple[str, ZarrV2DataTypeMetadata] | " + "tuple[str, ZarrV2DataTypeMetadata, tuple[int, ...]], ...]", +) """The v2 dtype representation. Either a numpy-style dtype string (e.g. `" None: assert model.shape == (10,) +@pytest.mark.parametrize("extra_key", ["attributes", "vendor_extension"]) +def test_v2_from_key_value_rejects_zarray_extra_members(extra_key: str) -> None: + """Raw `.zarray` documents reject every non-spec member.""" + doc: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default().to_json()) + doc.pop("attributes", None) + doc[extra_key] = {} + + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) + + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + ((extra_key,), "invalid_value") + ] + + def test_v2_zattrs_presence_round_trips() -> None: """The .zattrs file's presence is part of the store: an absent file reads as UNSET and emits no .zattrs; an explicit empty file reads as {} and @@ -837,6 +852,12 @@ def test_parse_json_materializes_abstract_containers() -> None: json.dumps(parsed, allow_nan=False) +def test_json_type_guard_rejects_abstract_sequence() -> None: + """A guard cannot narrow an abstract sequence that only the parser materializes.""" + assert not is_json(range(3)) + assert parse_json(range(3)) == (0, 1, 2) + + def test_parse_metadata_field_materializes_abstract_containers() -> None: """Named-config parsing produces canonical containers at every nesting level.""" value = UserDict({"name": "example", "configuration": UserDict({"values": range(2)})}) @@ -848,6 +869,14 @@ def test_parse_metadata_field_materializes_abstract_containers() -> None: assert type(parsed["configuration"]) is dict +def test_metadata_field_type_guard_rejects_abstract_mapping() -> None: + """A metadata-field guard only narrows concrete TypedDict-shaped objects.""" + value = UserDict({"name": "bytes"}) + + assert not is_metadata_field_v3(value) + assert parse_metadata_field_v3(value) == {"name": "bytes"} + + def test_validate_json_reports_json_in_message() -> None: """validate_json's message for a non-JSON value mentions JSON.""" problems = validate_json(object()) diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 52b27ac131..99f99de935 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -189,6 +189,19 @@ def test_group_v2_key_value_split() -> None: assert ZarrV2GroupMetadata.from_key_value(kv) == model +@pytest.mark.parametrize("extra_key", ["attributes", "vendor_extension"]) +def test_v2_group_from_key_value_rejects_zgroup_extra_members(extra_key: str) -> None: + """Raw `.zgroup` documents reject every non-spec member.""" + doc: dict[str, object] = {"zarr_format": 2, extra_key: {}} + + with pytest.raises(MetadataValidationError) as exc_info: + ZarrV2GroupMetadata.from_key_value({".zgroup": json.dumps(doc).encode()}) + + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + ((extra_key,), "invalid_value") + ] + + def test_group_v2_zattrs_presence_round_trips() -> None: """A v2 group with no .zattrs file parses with UNSET attributes and emits no .zattrs; an explicit empty .zattrs stays a file — the stores remain diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 9010344d5a..0b096b4b68 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -5,7 +5,10 @@ parallel hierarchy), so values interoperate freely with non-pydantic code. """ +import json + import pytest +from jsonschema import Draft202012Validator from pydantic import BaseModel, TypeAdapter, ValidationError import zarr_metadata.pydantic as zmp @@ -124,6 +127,87 @@ class Manifest(BaseModel): ] +def test_v2_recursive_structured_dtype_is_in_pydantic_schema() -> None: + """The schema accepts nested structured dtypes supported by the v2 specification.""" + doc = json.loads(json.dumps(V2_ARRAY_DOC)) + doc["dtype"] = [["outer", [["inner", " None: + adapter = TypeAdapter(field_type) + with pytest.raises(ValidationError): + adapter.validate_python(document) + assert list(Draft202012Validator(adapter.json_schema()).iter_errors(document)) + + +def test_v3_array_schema_rejects_empty_codecs() -> None: + """The generated schema mirrors the runtime non-empty codec pipeline rule.""" + doc = json.loads(json.dumps(V3_ARRAY_DOC)) + doc["codecs"] = [] + + _assert_runtime_and_schema_reject(zmp.ZarrV3ArrayMetadata, doc) + + +def test_array_schemas_reject_negative_dimensions() -> None: + """Both array schemas mirror the runtime non-negative dimension rule.""" + for field_type, source in ( + (zmp.ZarrV3ArrayMetadata, V3_ARRAY_DOC), + (zmp.ZarrV2ArrayMetadata, V2_ARRAY_DOC), + ): + doc = json.loads(json.dumps(source)) + doc["shape"] = [-1] + _assert_runtime_and_schema_reject(field_type, doc) + + +@pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) +def test_v3_array_schema_rejects_false_at_mandatory_extension_points(field: str) -> None: + """Mandatory v3 extension points cannot opt out of understanding.""" + doc = json.loads(json.dumps(V3_ARRAY_DOC)) + doc[field] = {"name": "example", "must_understand": False} + + _assert_runtime_and_schema_reject(zmp.ZarrV3ArrayMetadata, doc) + + +def test_metadata_field_schema_rejects_unknown_members() -> None: + """Named-configuration envelopes are closed in both runtime and schema validation.""" + _assert_runtime_and_schema_reject( + zmp.ZarrV3MetadataField, + {"name": "example", "unexpected": 1}, + ) + + +@pytest.mark.parametrize( + ("field_type", "source"), + [ + (zmp.ZarrV2ArrayMetadata, V2_ARRAY_DOC), + (zmp.ZarrV2GroupMetadata, V2_GROUP_DOC), + (zmp.ZarrV2ConsolidatedMetadata, V2_CONSOLIDATED_DOC), + ], +) +def test_v2_schema_rejects_unknown_document_members( + field_type: object, source: dict[str, object] +) -> None: + """Closed v2 merged documents expose their runtime boundary in JSON Schema.""" + doc = json.loads(json.dumps(source)) + doc["unexpected"] = 1 + + _assert_runtime_and_schema_reject(field_type, doc) + + +def test_v3_array_schema_allows_unknown_extension_fields() -> None: + """Schema constraints do not close the v3 top-level extension namespace.""" + doc = json.loads(json.dumps(V3_ARRAY_DOC)) + doc["vendor_extension"] = {"anything": [1, 2]} + adapter = TypeAdapter(zmp.ZarrV3ArrayMetadata) + + assert adapter.validate_python(doc).extra_fields == {"vendor_extension": {"anything": (1, 2)}} + assert Draft202012Validator(adapter.json_schema()).is_valid(doc) + + def test_json_roundtrip() -> None: """model_dump_json output re-validates to an equal pydantic model.""" From f4db45b576cdd04d56571c865f70022c2aa1d793 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 20:58:16 +0200 Subject: [PATCH 58/70] fix(metadata): tighten validation boundaries Assisted-by: Codex:gpt-5 --- packages/zarr-metadata/README.md | 34 +++++++++----- .../zarr-metadata/changes/4119.feature.md | 10 ++-- packages/zarr-metadata/pyproject.toml | 6 +-- .../src/zarr_metadata/_pydantic_schema.py | 25 ++++------ .../src/zarr_metadata/model/_group.py | 2 +- .../src/zarr_metadata/model/_validation.py | 46 +++++++++++++++---- .../src/zarr_metadata/v3/consolidated.py | 6 +-- .../zarr-metadata/tests/model/test_array.py | 37 +++++++++++++++ .../zarr-metadata/tests/model/test_group.py | 13 ++++++ .../tests/model/test_pydantic_module.py | 27 +++++++++++ 10 files changed, 160 insertions(+), 46 deletions(-) diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index bcd30b742d..810d0001f0 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -15,31 +15,35 @@ The optional integration requires Pydantic 2.13 or newer. ## What this is for -These types describe the JSON shape of Zarr metadata. They are -intended for libraries that **read, write, validate, or transform** -Zarr metadata. Pair them with a runtime validator like -[pydantic](https://docs.pydantic.dev/) to check JSON loaded from disk: +The public `TypedDict` definitions describe the static JSON shape of Zarr +metadata. For strict, loc-aware validation of JSON loaded from disk, use the +model parser: ```python import json -from pydantic import TypeAdapter -from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON +from zarr_metadata.model import ZarrV3ArrayMetadata with open("zarr.json", "rb") as f: raw = json.load(f) -metadata = TypeAdapter(ZarrV3ArrayMetadataJSON).validate_python(raw) +metadata = ZarrV3ArrayMetadata.from_json(raw) ``` -For a normalized model with loc-aware validation and serialization: +The optional Pydantic integration delegates raw input to the same strict +parser and returns the same normalized model class: ```python -from zarr_metadata.model import ZarrV3ArrayMetadata +from pydantic import TypeAdapter +import zarr_metadata.pydantic as zmp -model = ZarrV3ArrayMetadata.from_json(raw) -encoded = model.to_key_value()["zarr.json"] +metadata = TypeAdapter(zmp.ZarrV3ArrayMetadata).validate_python(raw) +encoded = metadata.to_key_value()["zarr.json"] ``` +A bare `TypeAdapter` over a public document `TypedDict` is a coercive shape +adapter, not a Zarr conformance validator; it may coerce values or discard +members that the strict model parser rejects. + ## Validation boundary The model validators enforce the declared document structure and a small set @@ -50,6 +54,14 @@ names or configurations, resolve codec pipelines, or decide whether a data type, chunk grid, codec, or storage transformer is supported. Those decisions belong to consumer implementations. +The Pydantic integration's generated JSON Schemas express independently +checkable document structure and field constraints, but they are not a +replacement for runtime model validation. Standard JSON Schema treats a +mathematically integral number such as `1.0` as an integer, while the runtime +boundary requires Python `int` values, and it cannot express arbitrary +same-length relations such as `dimension_names` versus `shape` or v2 `chunks` +versus `shape`. Consumers should run the model parser after schema validation. + ## Scope At minimum, this library supports what Zarr-Python needs: the complete diff --git a/packages/zarr-metadata/changes/4119.feature.md b/packages/zarr-metadata/changes/4119.feature.md index b32caf7b1a..7da9b728bf 100644 --- a/packages/zarr-metadata/changes/4119.feature.md +++ b/packages/zarr-metadata/changes/4119.feature.md @@ -25,6 +25,9 @@ accepted as dimension lengths, dimensions are non-negative, and `configuration` values are JSON-checked recursively (like `fill_value`), non-finite floats and non-standard JSON constants are rejected, abstract mappings and sequences normalize to encoder-safe canonical containers, +v2 `shape` and `chunks` must have the same rank, non-null v2 filter pipelines +contain at least one filter, document `TypeIs` guards only narrow values that +already use the declared canonical containers, and the inline consolidated-metadata envelope and entries are deep-validated so the group validator's verdict always agrees with the model constructor. @@ -39,9 +42,10 @@ requires pydantic 2.13 or newer; the core package does not depend on it): one `Annotated` field type per model, validating raw documents through `from_json`, passing core-model instances through unchanged, serializing via `to_json`, and -publishing JSON Schemas derived from the raw document TypedDicts. The instances -are the core model classes, so values interoperate freely with non-pydantic -code. +publishing JSON Schemas derived from private constrained document types that +mirror the independently expressible runtime rules. Cross-field cardinality +relations still require runtime validation. The instances are the core model +classes, so values interoperate freely with non-pydantic code. `create_default` keeps its output self-consistent: overriding `shape` without a chunk grid derives one regular chunk covering the array (v3 diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 23905a5bb7..8dcf0a905a 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -71,9 +71,9 @@ xfail_strict = true addopts = ["-ra", "--strict-config", "--strict-markers"] filterwarnings = [ "error", - # pydantic warns about ReadOnly TypedDict items not being enforced at runtime. - # That's expected here — we rely on type-checker enforcement, not pydantic mutation guards. - "ignore::UserWarning:pydantic._internal._generate_schema", + # Pydantic validates these public immutable-shape TypedDicts correctly but + # cannot enforce the type checker's ReadOnly mutation restriction. + "ignore:Items? .* using the `ReadOnly` qualifier.*:UserWarning:pydantic._internal._generate_schema", ] [tool.numpydoc_validation] diff --git a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py index e7f94262c9..e9792d6931 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py @@ -5,22 +5,21 @@ from collections.abc import Mapping # noqa: TC003 # resolved by Pydantic at runtime from typing import Annotated, Literal, NotRequired -from pydantic import ConfigDict, Field, with_config +from pydantic import Field from typing_extensions import TypedDict from zarr_metadata._common import JSONValue from zarr_metadata.v2.array import ( # noqa: TC001 # resolved by Pydantic at runtime ZarrV2DataTypeMetadata, ) -from zarr_metadata.v2.codec import ( # noqa: TC001 # resolved by Pydantic at runtime +from zarr_metadata.v2.codec import ( # resolved by Pydantic at runtime ZarrV2CodecMetadata, ) NonNegativeInt = Annotated[int, Field(ge=0)] -@with_config(ConfigDict(extra="forbid")) -class ZarrV3NamedConfigJSON(TypedDict): +class ZarrV3NamedConfigJSON(TypedDict, closed=True): """Closed v3 named configuration accepted at optional extension points.""" name: str @@ -28,8 +27,7 @@ class ZarrV3NamedConfigJSON(TypedDict): must_understand: NotRequired[bool] -@with_config(ConfigDict(extra="forbid")) -class ZarrV3MandatoryNamedConfigJSON(TypedDict): +class ZarrV3MandatoryNamedConfigJSON(TypedDict, closed=True): """Closed named configuration accepted where understanding is mandatory.""" name: str @@ -40,6 +38,7 @@ class ZarrV3MandatoryNamedConfigJSON(TypedDict): ZarrV3MetadataFieldJSON = str | ZarrV3NamedConfigJSON ZarrV3MandatoryMetadataFieldJSON = str | ZarrV3MandatoryNamedConfigJSON ZarrV3CodecPipelineJSON = Annotated[tuple[ZarrV3MetadataFieldJSON, ...], Field(min_length=1)] +ZarrV2FilterPipelineJSON = Annotated[tuple[ZarrV2CodecMetadata, ...], Field(min_length=1)] class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=JSONValue): @@ -58,8 +57,7 @@ class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=JSONValue): dimension_names: NotRequired[tuple[str | None, ...]] -@with_config(ConfigDict(extra="forbid")) -class ZarrV3ConsolidatedMetadataJSON(TypedDict): +class ZarrV3ConsolidatedMetadataJSON(TypedDict, closed=True): """Schema input for the closed inline consolidated-metadata envelope.""" kind: Literal["inline"] @@ -76,8 +74,7 @@ class ZarrV3GroupMetadataJSON(TypedDict, extra_items=JSONValue): consolidated_metadata: NotRequired[ZarrV3ConsolidatedMetadataJSON | None] -@with_config(ConfigDict(extra="forbid")) -class ZarrV2ArrayMetadataJSON(TypedDict): +class ZarrV2ArrayMetadataJSON(TypedDict, closed=True): """Schema input for the closed, merged v2 array representation.""" zarr_format: Literal[2] @@ -87,21 +84,19 @@ class ZarrV2ArrayMetadataJSON(TypedDict): compressor: ZarrV2CodecMetadata | None fill_value: JSONValue order: Literal["C", "F"] - filters: tuple[ZarrV2CodecMetadata, ...] | None + filters: ZarrV2FilterPipelineJSON | None dimension_separator: NotRequired[Literal[".", "/"]] attributes: NotRequired[Mapping[str, JSONValue]] -@with_config(ConfigDict(extra="forbid")) -class ZarrV2GroupMetadataJSON(TypedDict): +class ZarrV2GroupMetadataJSON(TypedDict, closed=True): """Schema input for the closed, merged v2 group representation.""" zarr_format: Literal[2] attributes: NotRequired[Mapping[str, JSONValue]] -@with_config(ConfigDict(extra="forbid")) -class ZarrV2ConsolidatedMetadataJSON(TypedDict): +class ZarrV2ConsolidatedMetadataJSON(TypedDict, closed=True): """Schema input matching the v2 consolidated model's structural parser.""" zarr_consolidated_format: Literal[1] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 811cc7b81e..f1a76e2508 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -350,7 +350,7 @@ class ZarrV2ConsolidatedMetadata: """In-memory model of a v2 `.zmetadata` document. The `metadata` map holds the flat file-keyed entries (`"path/.zarray"`, - `"path/.zattrs"`, ...) verbatim, preserving byte-faithful round-tripping. + `"path/.zattrs"`, ...) verbatim, preserving the normalized JSON tree. Entries are deliberately NOT merged into per-node models: which nodes had a `.zattrs` file at all is information the canonical representation must keep. Interpreting entries into node models is consumer work. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 3c3e4a1fea..35e6904edf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -552,7 +552,11 @@ def validate_array_metadata_v3(value: object) -> list[ValidationProblem]: def is_array_metadata_v3(value: object) -> TypeIs[ZarrV3ArrayMetadataJSON]: """Whether `value` is a structurally-valid v3 array metadata document.""" - return not validate_array_metadata_v3(value) and _is_canonical_array_metadata_v3(value) + return ( + _is_canonical_json(value) + and not validate_array_metadata_v3(value) + and _is_canonical_array_metadata_v3(value) + ) def parse_array_metadata_v3(value: object) -> ZarrV3ArrayMetadataJSON: @@ -580,8 +584,26 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: _unexpected_keys(ARRAY_METADATA_STANDARD_KEYS_V2, cast("Mapping[object, object]", value)) ) problems.extend(_check_literal(doc, "zarr_format", 2)) - problems.extend(_validate_dim_sequence(doc, "shape")) - problems.extend(_validate_dim_sequence(doc, "chunks")) + shape_problems = _validate_dim_sequence(doc, "shape") + chunks_problems = _validate_dim_sequence(doc, "chunks") + problems.extend(shape_problems) + problems.extend(chunks_problems) + if ( + not shape_problems + and not chunks_problems + and _is_int_sequence(doc.get("shape")) + and _is_int_sequence(doc.get("chunks")) + ): + shape = cast("Sequence[int]", doc["shape"]) + chunks = cast("Sequence[int]", doc["chunks"]) + if len(shape) != len(chunks): + problems.append( + ValidationProblem( + ("chunks",), + "expected the same number of dimensions as shape", + "invalid_value", + ) + ) if "dtype" in doc and not _is_dtype_v2(doc["dtype"]): problems.append( ValidationProblem( @@ -615,6 +637,10 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: ) ) elif filters is not None: + if len(cast("Sequence[object]", filters)) == 0: + problems.append( + ValidationProblem(("filters",), "expected at least one filter", "invalid_value") + ) for index, item in enumerate(cast("Sequence[object]", filters)): problems.extend(_prefix("filters", _prefix(index, validate_json(item)))) if "dimension_separator" in doc and doc["dimension_separator"] not in (".", "/"): @@ -634,7 +660,11 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: def is_array_metadata_v2(value: object) -> TypeIs[ZarrV2ArrayMetadataJSON]: """Whether `value` is a structurally-valid v2 array metadata document.""" - return not validate_array_metadata_v2(value) and _is_canonical_array_metadata_v2(value) + return ( + _is_canonical_json(value) + and not validate_array_metadata_v2(value) + and _is_canonical_array_metadata_v2(value) + ) def parse_array_metadata_v2(value: object) -> ZarrV2ArrayMetadataJSON: @@ -743,9 +773,7 @@ def validate_group_metadata_v3(value: object) -> list[ValidationProblem]: def is_group_metadata_v3(value: object) -> TypeIs[ZarrV3GroupMetadataJSON]: """Whether `value` is a structurally-valid v3 group metadata document.""" - return isinstance(value, dict) and not validate_group_metadata_v3( - cast("dict[object, object]", value) - ) + return _is_canonical_json(value) and not validate_group_metadata_v3(value) def parse_group_metadata_v3(value: object) -> ZarrV3GroupMetadataJSON: @@ -778,9 +806,7 @@ def validate_group_metadata_v2(value: object) -> list[ValidationProblem]: def is_group_metadata_v2(value: object) -> TypeIs[ZarrV2GroupMetadataJSON]: """Whether `value` is a structurally-valid v2 group metadata document.""" - return isinstance(value, dict) and not validate_group_metadata_v2( - cast("dict[object, object]", value) - ) + return _is_canonical_json(value) and not validate_group_metadata_v2(value) def parse_group_metadata_v2(value: object) -> ZarrV2GroupMetadataJSON: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py index bc29cd145e..bcbe675947 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py @@ -24,9 +24,9 @@ class ZarrV3ConsolidatedMetadataJSON(TypedDict): """ Inline consolidated metadata embedded in a v3 group. - The `metadata` map contains only v3 array and group entries - v2 - entries are excluded by design. Mixing v2 entries into a v3 - consolidated metadata document is invalid per spec. + The `metadata` map contains only v3 array and group entries. V2 entries + are excluded from this interoperability convention by design; the v3 core + specification does not define consolidated metadata. """ kind: Literal["inline"] diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 20764a85eb..f0ff473df1 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1214,6 +1214,30 @@ def test_v2_filters_must_be_codec_sequence_or_none() -> None: assert [(p.loc, p.kind) for p in problems] == [(("filters",), "invalid_type")], bad +def test_v2_shape_and_chunks_must_have_equal_rank() -> None: + """Raw v2 metadata requires one chunk length per array dimension.""" + doc = dict(ZarrV2ArrayMetadata.create_default(shape=(2, 3)).to_json()) + doc["chunks"] = (1,) + + assert [(p.loc, p.kind) for p in validate_array_metadata_v2(doc)] == [ + (("chunks",), "invalid_value") + ] + with pytest.raises(MetadataValidationError, match="same number of dimensions"): + ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) + + +def test_v2_filters_must_be_nonempty_when_present() -> None: + """A non-null v2 filter sequence contains one or more codec configurations.""" + doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) + doc["filters"] = () + + assert [(p.loc, p.kind) for p in validate_array_metadata_v2(doc)] == [ + (("filters",), "invalid_value") + ] + with pytest.raises(MetadataValidationError, match="at least one filter"): + ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) + + def test_v2_dimension_separator_literal_enforced() -> None: """A dimension_separator other than '.' or '/' is rejected.""" doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"dimension_separator": "-"} @@ -1492,6 +1516,19 @@ def test_array_parsers_normalize_json_lists_before_narrowing() -> None: assert isinstance(v2_parsed["chunks"], tuple) +def test_array_guards_reject_noncanonical_nested_json() -> None: + """Document guards cannot narrow values that only parsers can materialize.""" + v3 = dict(ZarrV3ArrayMetadata.create_default().to_json()) + v3["fill_value"] = range(2) + v2 = dict(ZarrV2ArrayMetadata.create_default().to_json()) + v2["fill_value"] = range(2) + + assert not is_array_metadata_v3(v3) + assert not is_array_metadata_v2(v2) + assert parse_array_metadata_v3(v3)["fill_value"] == (0, 1) + assert parse_array_metadata_v2(v2)["fill_value"] == (0, 1) + + # --- must_understand partition (spec: MUST fail to open unrecognized fields) -- diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 99f99de935..f38efeeb8a 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -20,6 +20,8 @@ from zarr_metadata.model._validation import ( MetadataValidationError, ValidationProblem, + is_group_metadata_v2, + is_group_metadata_v3, parse_group_metadata_v2, parse_group_metadata_v3, validate_group_metadata_v2, @@ -151,6 +153,17 @@ def test_group_parser_materializes_abstract_mapping( assert parsed == document +def test_group_guards_reject_noncanonical_nested_json() -> None: + """Document guards cannot narrow values that only parsers can materialize.""" + v3 = {"zarr_format": 3, "node_type": "group", "extension": range(2)} + v2 = {"zarr_format": 2, "attributes": {"values": range(2)}} + + assert not is_group_metadata_v3(v3) + assert not is_group_metadata_v2(v2) + assert parse_group_metadata_v3(v3)["extension"] == (0, 1) + assert parse_group_metadata_v2(v2)["attributes"] == {"values": (0, 1)} + + def test_group_v3_extension_fields_are_validated() -> None: """Group extension payloads must be JSON values with a must-understand flag.""" doc = { diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 0b096b4b68..d15b3f118c 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -6,6 +6,7 @@ """ import json +import warnings import pytest from jsonschema import Draft202012Validator @@ -127,6 +128,24 @@ class Manifest(BaseModel): ] +def test_json_schema_generation_emits_no_warnings() -> None: + """Consumers can generate every public integration schema without warning filters.""" + field_types = ( + zmp.ZarrV3ArrayMetadata, + zmp.ZarrV2ArrayMetadata, + zmp.ZarrV3GroupMetadata, + zmp.ZarrV2GroupMetadata, + zmp.ZarrV3ConsolidatedMetadata, + zmp.ZarrV2ConsolidatedMetadata, + zmp.ZarrV3MetadataField, + ) + + with warnings.catch_warnings(): + warnings.simplefilter("error") + for field_type in field_types: + TypeAdapter(field_type).json_schema() + + def test_v2_recursive_structured_dtype_is_in_pydantic_schema() -> None: """The schema accepts nested structured dtypes supported by the v2 specification.""" doc = json.loads(json.dumps(V2_ARRAY_DOC)) @@ -163,6 +182,14 @@ def test_array_schemas_reject_negative_dimensions() -> None: _assert_runtime_and_schema_reject(field_type, doc) +def test_v2_array_schema_rejects_empty_filters() -> None: + """The v2 schema mirrors the runtime one-or-more filter rule.""" + doc = json.loads(json.dumps(V2_ARRAY_DOC)) + doc["filters"] = [] + + _assert_runtime_and_schema_reject(zmp.ZarrV2ArrayMetadata, doc) + + @pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) def test_v3_array_schema_rejects_false_at_mandatory_extension_points(field: str) -> None: """Mandatory v3 extension points cannot opt out of understanding.""" From 693c9be800133efbc0a20c5686528a1e6bf1a262 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 22 Jul 2026 21:28:05 +0200 Subject: [PATCH 59/70] fix(metadata): repair package CI Assisted-by: Codex:gpt-5 --- packages/zarr-metadata/pyproject.toml | 2 +- .../src/zarr_metadata/_common.py | 9 ++++++++- .../src/zarr_metadata/model/_validation.py | 19 ++++++++++++------- .../src/zarr_metadata/v2/array.py | 8 ++++++-- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 8dcf0a905a..10b97f1c1e 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -47,7 +47,7 @@ Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/z Documentation = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-metadata/README.md" [dependency-groups] -test = ["pytest", "pydantic>=2.13"] +test = ["pytest", "pydantic>=2.13", "jsonschema"] [tool.hatch.version] source = "vcs" diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index 23bdfd00ca..13c2457489 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -13,7 +13,14 @@ JSONValue = TypeAliasType( "JSONValue", - "int | float | bool | None | str | list[JSONValue] | tuple[JSONValue, ...] | Mapping[str, JSONValue]", + int + | float + | bool + | None + | str + | list["JSONValue"] + | tuple["JSONValue", ...] + | Mapping[str, "JSONValue"], ) """A recursive type alias for JSON-encodable values. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 35e6904edf..a12e1911b1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -107,9 +107,13 @@ def _is_canonical_json(value: object) -> TypeIs[JSONValue]: if isinstance(value, (str, int, bool)) or value is None: return True if isinstance(value, (list, tuple)): - return all(_is_canonical_json(item) for item in value) + sequence = cast("list[object] | tuple[object, ...]", value) + return all(_is_canonical_json(item) for item in sequence) if isinstance(value, dict): - return all(isinstance(key, str) and _is_canonical_json(item) for key, item in value.items()) + mapping = cast("dict[object, object]", value) + return all( + isinstance(key, str) and _is_canonical_json(item) for key, item in mapping.items() + ) return False @@ -294,11 +298,12 @@ def validate_metadata_field_v3( def is_metadata_field_v3(value: object) -> TypeIs[ZarrV3MetadataFieldJSON]: """Whether `value` is a v3 metadata field: a bare name or a named config.""" - return isinstance(value, str) or ( - isinstance(value, dict) - and _is_canonical_json(value) - and not validate_metadata_field_v3(value) - ) + if isinstance(value, str): + return True + if not isinstance(value, dict): + return False + field = cast("dict[object, object]", value) + return _is_canonical_json(field) and not validate_metadata_field_v3(field) def parse_metadata_field_v3(value: object) -> ZarrV3MetadataFieldJSON: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/array.py b/packages/zarr-metadata/src/zarr_metadata/v2/array.py index 28085e4fbb..96c8b75036 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/array.py @@ -10,8 +10,12 @@ ZarrV2DataTypeMetadata = TypeAliasType( "ZarrV2DataTypeMetadata", - "str | tuple[tuple[str, ZarrV2DataTypeMetadata] | " - "tuple[str, ZarrV2DataTypeMetadata, tuple[int, ...]], ...]", + str + | tuple[ + tuple[str, "ZarrV2DataTypeMetadata"] + | tuple[str, "ZarrV2DataTypeMetadata", tuple[int, ...]], + ..., + ], ) """The v2 dtype representation. From faba4bc8f2cdf379879179778cc3c9b202136be8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 23:42:13 +0200 Subject: [PATCH 60/70] chore(deps): bump setuptools from 82.0.1 to 83.0.0 (#4176) Bumps [setuptools](https://github.com/pypa/setuptools) from 82.0.1 to 83.0.0. - [Release notes](https://github.com/pypa/setuptools/releases) - [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst) - [Commits](https://github.com/pypa/setuptools/compare/v82.0.1...v83.0.0) --- updated-dependencies: - dependency-name: setuptools dependency-version: 83.0.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 7a0d713873..725f9f2a20 100644 --- a/uv.lock +++ b/uv.lock @@ -2894,11 +2894,11 @@ wheels = [ [[package]] name = "setuptools" -version = "82.0.1" +version = "83.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4f/db/cfac1baf10650ab4d1c111714410d2fbb77ac5a616db26775db562c8fab2/setuptools-82.0.1.tar.gz", hash = "sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9", size = 1152316, upload-time = "2026-03-09T12:47:17.221Z" } +sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", size = 1006223, upload-time = "2026-03-09T12:47:15.026Z" }, + { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, ] [[package]] From 80e00ae006fa3466c6405ad2d3f6de26e3c5ac65 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 21:57:33 +0000 Subject: [PATCH 61/70] chore(deps): bump pillow from 12.2.0 to 12.3.0 (#4169) Bumps [pillow](https://github.com/python-pillow/Pillow) from 12.2.0 to 12.3.0. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/12.2.0...12.3.0) --- updated-dependencies: - dependency-name: pillow dependency-version: 12.3.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Davis Bennett --- uv.lock | 132 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 67 insertions(+), 65 deletions(-) diff --git a/uv.lock b/uv.lock index 725f9f2a20..6035acc616 100644 --- a/uv.lock +++ b/uv.lock @@ -2015,71 +2015,73 @@ wheels = [ [[package]] name = "pillow" -version = "12.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8c/21/c2bcdd5906101a30244eaffc1b6e6ce71a31bd0742a01eb89e660ebfac2d/pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5", size = 46987819, upload-time = "2026-04-01T14:46:17.687Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/58/be/7482c8a5ebebbc6470b3eb791812fff7d5e0216c2be3827b30b8bb6603ed/pillow-12.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d192a155bbcec180f8564f693e6fd9bccff5a7af9b32e2e4bf8c9c69dbad6b5", size = 5308279, upload-time = "2026-04-01T14:43:13.246Z" }, - { url = "https://files.pythonhosted.org/packages/d8/95/0a351b9289c2b5cbde0bacd4a83ebc44023e835490a727b2a3bd60ddc0f4/pillow-12.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3f40b3c5a968281fd507d519e444c35f0ff171237f4fdde090dd60699458421", size = 4695490, upload-time = "2026-04-01T14:43:15.584Z" }, - { url = "https://files.pythonhosted.org/packages/de/af/4e8e6869cbed569d43c416fad3dc4ecb944cb5d9492defaed89ddd6fe871/pillow-12.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03e7e372d5240cc23e9f07deca4d775c0817bffc641b01e9c3af208dbd300987", size = 6284462, upload-time = "2026-04-01T14:43:18.268Z" }, - { url = "https://files.pythonhosted.org/packages/e9/9e/c05e19657fd57841e476be1ab46c4d501bffbadbafdc31a6d665f8b737b6/pillow-12.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b86024e52a1b269467a802258c25521e6d742349d760728092e1bc2d135b4d76", size = 8094744, upload-time = "2026-04-01T14:43:20.716Z" }, - { url = "https://files.pythonhosted.org/packages/2b/54/1789c455ed10176066b6e7e6da1b01e50e36f94ba584dc68d9eebfe9156d/pillow-12.2.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7371b48c4fa448d20d2714c9a1f775a81155050d383333e0a6c15b1123dda005", size = 6398371, upload-time = "2026-04-01T14:43:23.443Z" }, - { url = "https://files.pythonhosted.org/packages/43/e3/fdc657359e919462369869f1c9f0e973f353f9a9ee295a39b1fea8ee1a77/pillow-12.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62f5409336adb0663b7caa0da5c7d9e7bdbaae9ce761d34669420c2a801b2780", size = 7087215, upload-time = "2026-04-01T14:43:26.758Z" }, - { url = "https://files.pythonhosted.org/packages/8b/f8/2f6825e441d5b1959d2ca5adec984210f1ec086435b0ed5f52c19b3b8a6e/pillow-12.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:01afa7cf67f74f09523699b4e88c73fb55c13346d212a59a2db1f86b0a63e8c5", size = 6509783, upload-time = "2026-04-01T14:43:29.56Z" }, - { url = "https://files.pythonhosted.org/packages/67/f9/029a27095ad20f854f9dba026b3ea6428548316e057e6fc3545409e86651/pillow-12.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5", size = 7212112, upload-time = "2026-04-01T14:43:32.091Z" }, - { url = "https://files.pythonhosted.org/packages/be/42/025cfe05d1be22dbfdb4f264fe9de1ccda83f66e4fc3aac94748e784af04/pillow-12.2.0-cp312-cp312-win32.whl", hash = "sha256:58f62cc0f00fd29e64b29f4fd923ffdb3859c9f9e6105bfc37ba1d08994e8940", size = 6378489, upload-time = "2026-04-01T14:43:34.601Z" }, - { url = "https://files.pythonhosted.org/packages/5d/7b/25a221d2c761c6a8ae21bfa3874988ff2583e19cf8a27bf2fee358df7942/pillow-12.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f84204dee22a783350679a0333981df803dac21a0190d706a50475e361c93f5", size = 7084129, upload-time = "2026-04-01T14:43:37.213Z" }, - { url = "https://files.pythonhosted.org/packages/10/e1/542a474affab20fd4a0f1836cb234e8493519da6b76899e30bcc5d990b8b/pillow-12.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414", size = 2463612, upload-time = "2026-04-01T14:43:39.421Z" }, - { url = "https://files.pythonhosted.org/packages/4a/01/53d10cf0dbad820a8db274d259a37ba50b88b24768ddccec07355382d5ad/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8297651f5b5679c19968abefd6bb84d95fe30ef712eb1b2d9b2d31ca61267f4c", size = 4100837, upload-time = "2026-04-01T14:43:41.506Z" }, - { url = "https://files.pythonhosted.org/packages/0f/98/f3a6657ecb698c937f6c76ee564882945f29b79bad496abcba0e84659ec5/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:50d8520da2a6ce0af445fa6d648c4273c3eeefbc32d7ce049f22e8b5c3daecc2", size = 4176528, upload-time = "2026-04-01T14:43:43.773Z" }, - { url = "https://files.pythonhosted.org/packages/69/bc/8986948f05e3ea490b8442ea1c1d4d990b24a7e43d8a51b2c7d8b1dced36/pillow-12.2.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:766cef22385fa1091258ad7e6216792b156dc16d8d3fa607e7545b2b72061f1c", size = 3640401, upload-time = "2026-04-01T14:43:45.87Z" }, - { url = "https://files.pythonhosted.org/packages/34/46/6c717baadcd62bc8ed51d238d521ab651eaa74838291bda1f86fe1f864c9/pillow-12.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5d2fd0fa6b5d9d1de415060363433f28da8b1526c1c129020435e186794b3795", size = 5308094, upload-time = "2026-04-01T14:43:48.438Z" }, - { url = "https://files.pythonhosted.org/packages/71/43/905a14a8b17fdb1ccb58d282454490662d2cb89a6bfec26af6d3520da5ec/pillow-12.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:56b25336f502b6ed02e889f4ece894a72612fe885889a6e8c4c80239ff6e5f5f", size = 4695402, upload-time = "2026-04-01T14:43:51.292Z" }, - { url = "https://files.pythonhosted.org/packages/73/dd/42107efcb777b16fa0393317eac58f5b5cf30e8392e266e76e51cff28c3d/pillow-12.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f1c943e96e85df3d3478f7b691f229887e143f81fedab9b20205349ab04d73ed", size = 6280005, upload-time = "2026-04-01T14:43:54.242Z" }, - { url = "https://files.pythonhosted.org/packages/a8/68/b93e09e5e8549019e61acf49f65b1a8530765a7f812c77a7461bca7e4494/pillow-12.2.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:03f6fab9219220f041c74aeaa2939ff0062bd5c364ba9ce037197f4c6d498cd9", size = 8090669, upload-time = "2026-04-01T14:43:57.335Z" }, - { url = "https://files.pythonhosted.org/packages/4b/6e/3ccb54ce8ec4ddd1accd2d89004308b7b0b21c4ac3d20fa70af4760a4330/pillow-12.2.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cdfebd752ec52bf5bb4e35d9c64b40826bc5b40a13df7c3cda20a2c03a0f5ed", size = 6395194, upload-time = "2026-04-01T14:43:59.864Z" }, - { url = "https://files.pythonhosted.org/packages/67/ee/21d4e8536afd1a328f01b359b4d3997b291ffd35a237c877b331c1c3b71c/pillow-12.2.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eedf4b74eda2b5a4b2b2fb4c006d6295df3bf29e459e198c90ea48e130dc75c3", size = 7082423, upload-time = "2026-04-01T14:44:02.74Z" }, - { url = "https://files.pythonhosted.org/packages/78/5f/e9f86ab0146464e8c133fe85df987ed9e77e08b29d8d35f9f9f4d6f917ba/pillow-12.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:00a2865911330191c0b818c59103b58a5e697cae67042366970a6b6f1b20b7f9", size = 6505667, upload-time = "2026-04-01T14:44:05.381Z" }, - { url = "https://files.pythonhosted.org/packages/ed/1e/409007f56a2fdce61584fd3acbc2bbc259857d555196cedcadc68c015c82/pillow-12.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1e1757442ed87f4912397c6d35a0db6a7b52592156014706f17658ff58bbf795", size = 7208580, upload-time = "2026-04-01T14:44:08.39Z" }, - { url = "https://files.pythonhosted.org/packages/23/c4/7349421080b12fb35414607b8871e9534546c128a11965fd4a7002ccfbee/pillow-12.2.0-cp313-cp313-win32.whl", hash = "sha256:144748b3af2d1b358d41286056d0003f47cb339b8c43a9ea42f5fea4d8c66b6e", size = 6375896, upload-time = "2026-04-01T14:44:11.197Z" }, - { url = "https://files.pythonhosted.org/packages/3f/82/8a3739a5e470b3c6cbb1d21d315800d8e16bff503d1f16b03a4ec3212786/pillow-12.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:390ede346628ccc626e5730107cde16c42d3836b89662a115a921f28440e6a3b", size = 7081266, upload-time = "2026-04-01T14:44:13.947Z" }, - { url = "https://files.pythonhosted.org/packages/c3/25/f968f618a062574294592f668218f8af564830ccebdd1fa6200f598e65c5/pillow-12.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:8023abc91fba39036dbce14a7d6535632f99c0b857807cbbbf21ecc9f4717f06", size = 2463508, upload-time = "2026-04-01T14:44:16.312Z" }, - { url = "https://files.pythonhosted.org/packages/4d/a4/b342930964e3cb4dce5038ae34b0eab4653334995336cd486c5a8c25a00c/pillow-12.2.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:042db20a421b9bafecc4b84a8b6e444686bd9d836c7fd24542db3e7df7baad9b", size = 5309927, upload-time = "2026-04-01T14:44:18.89Z" }, - { url = "https://files.pythonhosted.org/packages/9f/de/23198e0a65a9cf06123f5435a5d95cea62a635697f8f03d134d3f3a96151/pillow-12.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:dd025009355c926a84a612fecf58bb315a3f6814b17ead51a8e48d3823d9087f", size = 4698624, upload-time = "2026-04-01T14:44:21.115Z" }, - { url = "https://files.pythonhosted.org/packages/01/a6/1265e977f17d93ea37aa28aa81bad4fa597933879fac2520d24e021c8da3/pillow-12.2.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88ddbc66737e277852913bd1e07c150cc7bb124539f94c4e2df5344494e0a612", size = 6321252, upload-time = "2026-04-01T14:44:23.663Z" }, - { url = "https://files.pythonhosted.org/packages/3c/83/5982eb4a285967baa70340320be9f88e57665a387e3a53a7f0db8231a0cd/pillow-12.2.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d362d1878f00c142b7e1a16e6e5e780f02be8195123f164edf7eddd911eefe7c", size = 8126550, upload-time = "2026-04-01T14:44:26.772Z" }, - { url = "https://files.pythonhosted.org/packages/4e/48/6ffc514adce69f6050d0753b1a18fd920fce8cac87620d5a31231b04bfc5/pillow-12.2.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c727a6d53cb0018aadd8018c2b938376af27914a68a492f59dfcaca650d5eea", size = 6433114, upload-time = "2026-04-01T14:44:29.615Z" }, - { url = "https://files.pythonhosted.org/packages/36/a3/f9a77144231fb8d40ee27107b4463e205fa4677e2ca2548e14da5cf18dce/pillow-12.2.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd8c21c98c5cc60653bcb311bef2ce0401642b7ce9d09e03a7da87c878289d4", size = 7115667, upload-time = "2026-04-01T14:44:32.773Z" }, - { url = "https://files.pythonhosted.org/packages/c1/fc/ac4ee3041e7d5a565e1c4fd72a113f03b6394cc72ab7089d27608f8aaccb/pillow-12.2.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9f08483a632889536b8139663db60f6724bfcb443c96f1b18855860d7d5c0fd4", size = 6538966, upload-time = "2026-04-01T14:44:35.252Z" }, - { url = "https://files.pythonhosted.org/packages/c0/a8/27fb307055087f3668f6d0a8ccb636e7431d56ed0750e07a60547b1e083e/pillow-12.2.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dac8d77255a37e81a2efcbd1fc05f1c15ee82200e6c240d7e127e25e365c39ea", size = 7238241, upload-time = "2026-04-01T14:44:37.875Z" }, - { url = "https://files.pythonhosted.org/packages/ad/4b/926ab182c07fccae9fcb120043464e1ff1564775ec8864f21a0ebce6ac25/pillow-12.2.0-cp313-cp313t-win32.whl", hash = "sha256:ee3120ae9dff32f121610bb08e4313be87e03efeadfc6c0d18f89127e24d0c24", size = 6379592, upload-time = "2026-04-01T14:44:40.336Z" }, - { url = "https://files.pythonhosted.org/packages/c2/c4/f9e476451a098181b30050cc4c9a3556b64c02cf6497ea421ac047e89e4b/pillow-12.2.0-cp313-cp313t-win_amd64.whl", hash = "sha256:325ca0528c6788d2a6c3d40e3568639398137346c3d6e66bb61db96b96511c98", size = 7085542, upload-time = "2026-04-01T14:44:43.251Z" }, - { url = "https://files.pythonhosted.org/packages/00/a4/285f12aeacbe2d6dc36c407dfbbe9e96d4a80b0fb710a337f6d2ad978c75/pillow-12.2.0-cp313-cp313t-win_arm64.whl", hash = "sha256:2e5a76d03a6c6dcef67edabda7a52494afa4035021a79c8558e14af25313d453", size = 2465765, upload-time = "2026-04-01T14:44:45.996Z" }, - { url = "https://files.pythonhosted.org/packages/bf/98/4595daa2365416a86cb0d495248a393dfc84e96d62ad080c8546256cb9c0/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3adc9215e8be0448ed6e814966ecf3d9952f0ea40eb14e89a102b87f450660d8", size = 4100848, upload-time = "2026-04-01T14:44:48.48Z" }, - { url = "https://files.pythonhosted.org/packages/0b/79/40184d464cf89f6663e18dfcf7ca21aae2491fff1a16127681bf1fa9b8cf/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:6a9adfc6d24b10f89588096364cc726174118c62130c817c2837c60cf08a392b", size = 4176515, upload-time = "2026-04-01T14:44:51.353Z" }, - { url = "https://files.pythonhosted.org/packages/b0/63/703f86fd4c422a9cf722833670f4f71418fb116b2853ff7da722ea43f184/pillow-12.2.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6a6e67ea2e6feda684ed370f9a1c52e7a243631c025ba42149a2cc5934dec295", size = 3640159, upload-time = "2026-04-01T14:44:53.588Z" }, - { url = "https://files.pythonhosted.org/packages/71/e0/fb22f797187d0be2270f83500aab851536101b254bfa1eae10795709d283/pillow-12.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2bb4a8d594eacdfc59d9e5ad972aa8afdd48d584ffd5f13a937a664c3e7db0ed", size = 5312185, upload-time = "2026-04-01T14:44:56.039Z" }, - { url = "https://files.pythonhosted.org/packages/ba/8c/1a9e46228571de18f8e28f16fabdfc20212a5d019f3e3303452b3f0a580d/pillow-12.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:80b2da48193b2f33ed0c32c38140f9d3186583ce7d516526d462645fd98660ae", size = 4695386, upload-time = "2026-04-01T14:44:58.663Z" }, - { url = "https://files.pythonhosted.org/packages/70/62/98f6b7f0c88b9addd0e87c217ded307b36be024d4ff8869a812b241d1345/pillow-12.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22db17c68434de69d8ecfc2fe821569195c0c373b25cccb9cbdacf2c6e53c601", size = 6280384, upload-time = "2026-04-01T14:45:01.5Z" }, - { url = "https://files.pythonhosted.org/packages/5e/03/688747d2e91cfbe0e64f316cd2e8005698f76ada3130d0194664174fa5de/pillow-12.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7b14cc0106cd9aecda615dd6903840a058b4700fcb817687d0ee4fc8b6e389be", size = 8091599, upload-time = "2026-04-01T14:45:04.5Z" }, - { url = "https://files.pythonhosted.org/packages/f6/35/577e22b936fcdd66537329b33af0b4ccfefaeabd8aec04b266528cddb33c/pillow-12.2.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cbeb542b2ebc6fcdacabf8aca8c1a97c9b3ad3927d46b8723f9d4f033288a0f", size = 6396021, upload-time = "2026-04-01T14:45:07.117Z" }, - { url = "https://files.pythonhosted.org/packages/11/8d/d2532ad2a603ca2b93ad9f5135732124e57811d0168155852f37fbce2458/pillow-12.2.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bfd07bc812fbd20395212969e41931001fd59eb55a60658b0e5710872e95286", size = 7083360, upload-time = "2026-04-01T14:45:09.763Z" }, - { url = "https://files.pythonhosted.org/packages/5e/26/d325f9f56c7e039034897e7380e9cc202b1e368bfd04d4cbe6a441f02885/pillow-12.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9aba9a17b623ef750a4d11b742cbafffeb48a869821252b30ee21b5e91392c50", size = 6507628, upload-time = "2026-04-01T14:45:12.378Z" }, - { url = "https://files.pythonhosted.org/packages/5f/f7/769d5632ffb0988f1c5e7660b3e731e30f7f8ec4318e94d0a5d674eb65a4/pillow-12.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:deede7c263feb25dba4e82ea23058a235dcc2fe1f6021025dc71f2b618e26104", size = 7209321, upload-time = "2026-04-01T14:45:15.122Z" }, - { url = "https://files.pythonhosted.org/packages/6a/7a/c253e3c645cd47f1aceea6a8bacdba9991bf45bb7dfe927f7c893e89c93c/pillow-12.2.0-cp314-cp314-win32.whl", hash = "sha256:632ff19b2778e43162304d50da0181ce24ac5bb8180122cbe1bf4673428328c7", size = 6479723, upload-time = "2026-04-01T14:45:17.797Z" }, - { url = "https://files.pythonhosted.org/packages/cd/8b/601e6566b957ca50e28725cb6c355c59c2c8609751efbecd980db44e0349/pillow-12.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:4e6c62e9d237e9b65fac06857d511e90d8461a32adcc1b9065ea0c0fa3a28150", size = 7217400, upload-time = "2026-04-01T14:45:20.529Z" }, - { url = "https://files.pythonhosted.org/packages/d6/94/220e46c73065c3e2951bb91c11a1fb636c8c9ad427ac3ce7d7f3359b9b2f/pillow-12.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:b1c1fbd8a5a1af3412a0810d060a78b5136ec0836c8a4ef9aa11807f2a22f4e1", size = 2554835, upload-time = "2026-04-01T14:45:23.162Z" }, - { url = "https://files.pythonhosted.org/packages/b6/ab/1b426a3974cb0e7da5c29ccff4807871d48110933a57207b5a676cccc155/pillow-12.2.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:57850958fe9c751670e49b2cecf6294acc99e562531f4bd317fa5ddee2068463", size = 5314225, upload-time = "2026-04-01T14:45:25.637Z" }, - { url = "https://files.pythonhosted.org/packages/19/1e/dce46f371be2438eecfee2a1960ee2a243bbe5e961890146d2dee1ff0f12/pillow-12.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d5d38f1411c0ed9f97bcb49b7bd59b6b7c314e0e27420e34d99d844b9ce3b6f3", size = 4698541, upload-time = "2026-04-01T14:45:28.355Z" }, - { url = "https://files.pythonhosted.org/packages/55/c3/7fbecf70adb3a0c33b77a300dc52e424dc22ad8cdc06557a2e49523b703d/pillow-12.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c0a9f29ca8e79f09de89293f82fc9b0270bb4af1d58bc98f540cc4aedf03166", size = 6322251, upload-time = "2026-04-01T14:45:30.924Z" }, - { url = "https://files.pythonhosted.org/packages/1c/3c/7fbc17cfb7e4fe0ef1642e0abc17fc6c94c9f7a16be41498e12e2ba60408/pillow-12.2.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1610dd6c61621ae1cf811bef44d77e149ce3f7b95afe66a4512f8c59f25d9ebe", size = 8127807, upload-time = "2026-04-01T14:45:33.908Z" }, - { url = "https://files.pythonhosted.org/packages/ff/c3/a8ae14d6defd2e448493ff512fae903b1e9bd40b72efb6ec55ce0048c8ce/pillow-12.2.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a34329707af4f73cf1782a36cd2289c0368880654a2c11f027bcee9052d35dd", size = 6433935, upload-time = "2026-04-01T14:45:36.623Z" }, - { url = "https://files.pythonhosted.org/packages/6e/32/2880fb3a074847ac159d8f902cb43278a61e85f681661e7419e6596803ed/pillow-12.2.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e9c4f5b3c546fa3458a29ab22646c1c6c787ea8f5ef51300e5a60300736905e", size = 7116720, upload-time = "2026-04-01T14:45:39.258Z" }, - { url = "https://files.pythonhosted.org/packages/46/87/495cc9c30e0129501643f24d320076f4cc54f718341df18cc70ec94c44e1/pillow-12.2.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06", size = 6540498, upload-time = "2026-04-01T14:45:41.879Z" }, - { url = "https://files.pythonhosted.org/packages/18/53/773f5edca692009d883a72211b60fdaf8871cbef075eaa9d577f0a2f989e/pillow-12.2.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f278f034eb75b4e8a13a54a876cc4a5ab39173d2cdd93a638e1b467fc545ac43", size = 7239413, upload-time = "2026-04-01T14:45:44.705Z" }, - { url = "https://files.pythonhosted.org/packages/c9/e4/4b64a97d71b2a83158134abbb2f5bd3f8a2ea691361282f010998f339ec7/pillow-12.2.0-cp314-cp314t-win32.whl", hash = "sha256:6bb77b2dcb06b20f9f4b4a8454caa581cd4dd0643a08bacf821216a16d9c8354", size = 6482084, upload-time = "2026-04-01T14:45:47.568Z" }, - { url = "https://files.pythonhosted.org/packages/ba/13/306d275efd3a3453f72114b7431c877d10b1154014c1ebbedd067770d629/pillow-12.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6562ace0d3fb5f20ed7290f1f929cae41b25ae29528f2af1722966a0a02e2aa1", size = 7225152, upload-time = "2026-04-01T14:45:50.032Z" }, - { url = "https://files.pythonhosted.org/packages/ff/6e/cf826fae916b8658848d7b9f38d88da6396895c676e8086fc0988073aaf8/pillow-12.2.0-cp314-cp314t-win_arm64.whl", hash = "sha256:aa88ccfe4e32d362816319ed727a004423aab09c5cea43c01a4b435643fa34eb", size = 2556579, upload-time = "2026-04-01T14:45:52.529Z" }, +version = "12.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/bf/fb3ebff8ddcb76aac5a01389251bbbb9519922a9b520d8247c1ca864a25d/pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965", size = 5345969, upload-time = "2026-07-01T11:54:06.397Z" }, + { url = "https://files.pythonhosted.org/packages/d8/66/9a386a92561f402389a4fc70c18838bf6d35eb5eb5c6850b4b2dc64f5048/pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7", size = 4780323, upload-time = "2026-07-01T11:54:09.351Z" }, + { url = "https://files.pythonhosted.org/packages/25/27/ac8f99618ffd3dde21db0f4d4b1d2ab00c0880595bfd17df103f7f39fd0c/pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9", size = 6266838, upload-time = "2026-07-01T11:54:11.71Z" }, + { url = "https://files.pythonhosted.org/packages/84/21/a35af28dcc61f37ed850a2d64c65c701321dfbf25085e469d5559360cbbf/pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91", size = 6940830, upload-time = "2026-07-01T11:54:13.732Z" }, + { url = "https://files.pythonhosted.org/packages/eb/51/8b08617af3ad95e33ce6d7dd2c99ed6c8298f7fb131636303956be022e25/pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c", size = 6344383, upload-time = "2026-07-01T11:54:15.756Z" }, + { url = "https://files.pythonhosted.org/packages/1d/72/cf78ac9780bb93c28328f408973845a309d4d145041665f734572ced1b52/pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df", size = 7052934, upload-time = "2026-07-01T11:54:17.721Z" }, + { url = "https://files.pythonhosted.org/packages/20/20/25e0f4dc178a6bc0696793720055519a0de89e7661dae886992decbd2f81/pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f", size = 6472684, upload-time = "2026-07-01T11:54:19.839Z" }, + { url = "https://files.pythonhosted.org/packages/45/89/da2f7971a317f83d807fdd4065c0af40208e59e692cc43d315a71a0e96d1/pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09", size = 7227137, upload-time = "2026-07-01T11:54:22.025Z" }, + { url = "https://files.pythonhosted.org/packages/de/47/4845a0a6c0dbf1db8456bd9fc791f13c5ced7ced20606d08a0aacfd25b49/pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510", size = 2568267, upload-time = "2026-07-01T11:54:24.051Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ac/31fb64e1e7efb5a4b50cd3d92049ba89ac6e4d8d3bb6a74e15048ca3353e/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89", size = 4161684, upload-time = "2026-07-01T11:54:25.934Z" }, + { url = "https://files.pythonhosted.org/packages/87/b4/9805e23d2b4d77842b468513841fda254ee42f0289d25088340e4ff46e2d/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace", size = 4255487, upload-time = "2026-07-01T11:54:27.935Z" }, + { url = "https://files.pythonhosted.org/packages/df/39/ecf519435a200c693fe053a6ee4d835b41cf963a4dfc2551c4e637cb2a71/pillow-12.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec", size = 3696433, upload-time = "2026-07-01T11:54:29.813Z" }, + { url = "https://files.pythonhosted.org/packages/42/92/2fc3ffad878ae8dd5469ec1bc8eb83b71f48e13efdf68f02709003982a32/pillow-12.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66", size = 5345889, upload-time = "2026-07-01T11:54:31.97Z" }, + { url = "https://files.pythonhosted.org/packages/10/76/8803c13605b763d33d156c4678fc77f8443389c0c51c8aef707bb02015f4/pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35", size = 4780109, upload-time = "2026-07-01T11:54:34.026Z" }, + { url = "https://files.pythonhosted.org/packages/1f/01/e18aff37cb0b4aac47ac90f016d347a49aca667ef97f190b06ac2aabc928/pillow-12.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65", size = 6263736, upload-time = "2026-07-01T11:54:36.131Z" }, + { url = "https://files.pythonhosted.org/packages/f7/62/de5bdd77d935331f4f802edc11e4d82950f642caad6cb2f949837b8560e2/pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3", size = 6937129, upload-time = "2026-07-01T11:54:38.216Z" }, + { url = "https://files.pythonhosted.org/packages/70/4d/105627a13300c5e0df1d174230b32fd1273062c96f7745fd552b945d1e1d/pillow-12.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a", size = 6339562, upload-time = "2026-07-01T11:54:40.354Z" }, + { url = "https://files.pythonhosted.org/packages/6b/1d/f13de01a553988ab895ba1c722e06cf3144d4f57656fd5b81b6d881f1179/pillow-12.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e", size = 7049439, upload-time = "2026-07-01T11:54:42.489Z" }, + { url = "https://files.pythonhosted.org/packages/c9/f9/066794cca041b969964f779ee5fa66a9498bbf34248ac39c5d7954e4198f/pillow-12.3.0-cp313-cp313-win32.whl", hash = "sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f", size = 6473287, upload-time = "2026-07-01T11:54:44.9Z" }, + { url = "https://files.pythonhosted.org/packages/a6/9b/7a58e61d62be561da3a356fe2384d4059a6345fc130e23ef1c36a5b81d24/pillow-12.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8", size = 7239691, upload-time = "2026-07-01T11:54:47.141Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b0/c4ed4f0ef8f8fa5ee8351537db6650bb8189f7e118842978dd6589065692/pillow-12.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b", size = 2568185, upload-time = "2026-07-01T11:54:49.137Z" }, + { url = "https://files.pythonhosted.org/packages/dc/01/001f65b68192f0228cc1dbbc8d2530ab5d58b61037ba0587f946fea607cd/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330", size = 4161736, upload-time = "2026-07-01T11:54:51.156Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d2/0219746d0fd16fc8a84498e79452375be3797d3ce4044596ce565164b84f/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217", size = 4255435, upload-time = "2026-07-01T11:54:53.414Z" }, + { url = "https://files.pythonhosted.org/packages/c8/02/8d0bc62ef0302318c46ff2a512822d2610e81c7aa46c9b3abe6cbaca5ad0/pillow-12.3.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930", size = 3696262, upload-time = "2026-07-01T11:54:55.739Z" }, + { url = "https://files.pythonhosted.org/packages/85/e2/73c77d218410b14f5f2d565e8a998d5317b7b9c75368d29985139f7a46f0/pillow-12.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8", size = 5350344, upload-time = "2026-07-01T11:54:57.657Z" }, + { url = "https://files.pythonhosted.org/packages/c7/da/32c752228ae345f489e3a42499d817b6c3996da7e8a3bc7a04fc806b243b/pillow-12.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0", size = 4780131, upload-time = "2026-07-01T11:54:59.713Z" }, + { url = "https://files.pythonhosted.org/packages/b1/9d/8b2c807dbef61a5197c047afe99823787eb66f63daf9fb2432f91d6f0462/pillow-12.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321", size = 6263757, upload-time = "2026-07-01T11:55:01.778Z" }, + { url = "https://files.pythonhosted.org/packages/5c/44/c85361f65dbe00eea8576ee467c768d25129989efb76e94f205e9ca9bb46/pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b", size = 6936962, upload-time = "2026-07-01T11:55:03.93Z" }, + { url = "https://files.pythonhosted.org/packages/18/7e/e483414b35800b86b6f08dbbc7803fb5cd52c4d6f897f47d53ea2c7e6f65/pillow-12.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198", size = 6339171, upload-time = "2026-07-01T11:55:05.989Z" }, + { url = "https://files.pythonhosted.org/packages/f0/f4/68c491844841ede6bed70189546b3ee9731cf9f2cbad396faff5e1ccba45/pillow-12.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130", size = 7048116, upload-time = "2026-07-01T11:55:08.131Z" }, + { url = "https://files.pythonhosted.org/packages/a3/34/77f3f793fed8efc7d243f21b33c5a3f0d1c97ee70346d3db855587e155ff/pillow-12.3.0-cp314-cp314-win32.whl", hash = "sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a", size = 6467209, upload-time = "2026-07-01T11:55:10.408Z" }, + { url = "https://files.pythonhosted.org/packages/f1/e0/492879f69d94f91f60fc8cd05ba03650e9520afebb2fb7aa12777d7c7f38/pillow-12.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d", size = 7237707, upload-time = "2026-07-01T11:55:12.745Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ac/6b11f2875f1c2ac040d84e1bbf9cf22a88038f901ca1037898b280b38365/pillow-12.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838", size = 2565995, upload-time = "2026-07-01T11:55:14.736Z" }, + { url = "https://files.pythonhosted.org/packages/52/69/c2208e56af9bfc1913afb24020297a691eb1d4ef688474c8a04913f65e04/pillow-12.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e", size = 5352503, upload-time = "2026-07-01T11:55:17.076Z" }, + { url = "https://files.pythonhosted.org/packages/07/70/e5686d753e898a45d778ff1718dba8516ead6ab6b95d85fc8c4b70650cf2/pillow-12.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17", size = 4782956, upload-time = "2026-07-01T11:55:19.448Z" }, + { url = "https://files.pythonhosted.org/packages/d5/37/25c6692f06927ee973ff18c8d9ee98ad0b4d84ee67a09610c2dd1447958e/pillow-12.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385", size = 6322855, upload-time = "2026-07-01T11:55:21.613Z" }, + { url = "https://files.pythonhosted.org/packages/cc/91/420637fcb8f1bc11029e403b4538e6694744428d8246118e45719f944556/pillow-12.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c", size = 6989642, upload-time = "2026-07-01T11:55:24.006Z" }, + { url = "https://files.pythonhosted.org/packages/10/08/b94d7811281ccf0d143a1cf768d1c49e1e54af63e7b708ab2ee3eb87face/pillow-12.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d", size = 6391281, upload-time = "2026-07-01T11:55:26.252Z" }, + { url = "https://files.pythonhosted.org/packages/d2/87/24233f785f55474dc02ce3e739c5528a77e3a862e9333d1dd7a25cc31f70/pillow-12.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931", size = 7096716, upload-time = "2026-07-01T11:55:28.318Z" }, + { url = "https://files.pythonhosted.org/packages/23/26/fcb2f6e37175b04f53570b59937867e2b80ee1685e744023153028fc14f9/pillow-12.3.0-cp314-cp314t-win32.whl", hash = "sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7", size = 6474125, upload-time = "2026-07-01T11:55:30.956Z" }, + { url = "https://files.pythonhosted.org/packages/90/de/3634abee5f1c9e13c56787b7d5517b0ba8d6de51700b95578cf338349c9f/pillow-12.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c", size = 7242939, upload-time = "2026-07-01T11:55:34.044Z" }, + { url = "https://files.pythonhosted.org/packages/ce/2a/fd13f8eb24de5714a6eb444a3d67e2842c6c576e159a43793adf23051351/pillow-12.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45", size = 2567506, upload-time = "2026-07-01T11:55:35.988Z" }, + { url = "https://files.pythonhosted.org/packages/5d/dc/8fdce34ec725a33c81c6ba122b904d6b9024e50ea9ac7bede62fab54506c/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139", size = 4162063, upload-time = "2026-07-01T11:55:37.941Z" }, + { url = "https://files.pythonhosted.org/packages/76/66/2044b9a63d3b84ff048228dfcb7cd9bf0df983e8470971bf7d4c57b693de/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402", size = 4255549, upload-time = "2026-07-01T11:55:40.022Z" }, + { url = "https://files.pythonhosted.org/packages/52/7e/1f67e6f4ece6b582ee4b539decbcc9f848dc245a93ed8cd7338bafef72f1/pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c", size = 3696331, upload-time = "2026-07-01T11:55:41.98Z" }, + { url = "https://files.pythonhosted.org/packages/12/40/d306fc2c8e4d45d7f175c77edca7063be7b86fe7fe6e68f4353bf71d808c/pillow-12.3.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f", size = 5350370, upload-time = "2026-07-01T11:55:44.028Z" }, + { url = "https://files.pythonhosted.org/packages/dd/44/668fb1437e8ce420f62d6106eb66e44a5971602a4d794615bdf79315d82d/pillow-12.3.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701", size = 4780147, upload-time = "2026-07-01T11:55:46.073Z" }, + { url = "https://files.pythonhosted.org/packages/0c/08/93fa2e70e30a2d81547e481b6ee2bb9522117221fb1e0ce4b5df70967677/pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace", size = 6273659, upload-time = "2026-07-01T11:55:48.264Z" }, + { url = "https://files.pythonhosted.org/packages/f8/6d/043e96ff814fc31a33077e4cba86082167db520c93632afdf2042febbb0c/pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4", size = 6947439, upload-time = "2026-07-01T11:55:50.503Z" }, + { url = "https://files.pythonhosted.org/packages/af/92/ba71d2ee2ac0edf3fa33bd9d5ee9ee080da70b1766f3ca3934f9938ddac9/pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39", size = 6353577, upload-time = "2026-07-01T11:55:52.697Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ce/e63064e2122923ff687c8ad792d0d736a7b3920a56a46982e81a7fdd25d6/pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71", size = 7060394, upload-time = "2026-07-01T11:55:55.149Z" }, + { url = "https://files.pythonhosted.org/packages/54/76/a09cc3ccc8d773a7283d34c38bec1708f9e3cc932093cbc4c5e71ac4060b/pillow-12.3.0-cp315-cp315-win32.whl", hash = "sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827", size = 6467375, upload-time = "2026-07-01T11:55:57.769Z" }, + { url = "https://files.pythonhosted.org/packages/3e/03/1846c49ba3b1d5550392a4bbd06d6fb4578e1cd91a803198b5c90f5f7d53/pillow-12.3.0-cp315-cp315-win_amd64.whl", hash = "sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5", size = 7237048, upload-time = "2026-07-01T11:55:59.975Z" }, + { url = "https://files.pythonhosted.org/packages/fb/bb/89f35dcc79610423f9f195504d7def7f0d1416a711541b42867e25fe3412/pillow-12.3.0-cp315-cp315-win_arm64.whl", hash = "sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658", size = 2566006, upload-time = "2026-07-01T11:56:02.143Z" }, + { url = "https://files.pythonhosted.org/packages/30/88/707027ba09942dfa2c28759b5c222d769290a41c6d20ea60ec250801941f/pillow-12.3.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf", size = 5352509, upload-time = "2026-07-01T11:56:04.2Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6d/00352fa25332c2569cd387851f568cc5a4b75a9adbfb37ac4fbce4c02eec/pillow-12.3.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64", size = 4783167, upload-time = "2026-07-01T11:56:06.631Z" }, + { url = "https://files.pythonhosted.org/packages/13/4f/9e049dfa21af7c22427275720e2490267ba8138120add5c4c574deb69782/pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e", size = 6329237, upload-time = "2026-07-01T11:56:08.868Z" }, + { url = "https://files.pythonhosted.org/packages/36/16/cf6eeaae8d0fce8dd390a33437cf68c5d5bd73834a2bc6e2f14efda0ab45/pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777", size = 6997047, upload-time = "2026-07-01T11:56:11.379Z" }, + { url = "https://files.pythonhosted.org/packages/1e/69/dbf769bdd55f48bf5733cac28edc6364ffaa072ec9ba336266e4fe66be55/pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1", size = 6400440, upload-time = "2026-07-01T11:56:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e1/ffc9cfc2eea0d178da8018e18e959301ad9d6bc9f3edb7181e748a474b97/pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9", size = 7105895, upload-time = "2026-07-01T11:56:16.575Z" }, + { url = "https://files.pythonhosted.org/packages/18/f0/a5595c1e8c3ae44b9828cb2f0fa8155e5095ef04d6327b8f61cf44a3df85/pillow-12.3.0-cp315-cp315t-win32.whl", hash = "sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8", size = 6474384, upload-time = "2026-07-01T11:56:18.855Z" }, + { url = "https://files.pythonhosted.org/packages/e4/04/62bcd9f844984c5938d3b05264a61d797a29d3e0812341a8204af70bbdee/pillow-12.3.0-cp315-cp315t-win_amd64.whl", hash = "sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418", size = 7243537, upload-time = "2026-07-01T11:56:21.214Z" }, + { url = "https://files.pythonhosted.org/packages/3d/68/1f3066acedf37673694a7141381d8f811ae97f30d34413d236abe7d489f1/pillow-12.3.0-cp315-cp315t-win_arm64.whl", hash = "sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59", size = 2567491, upload-time = "2026-07-01T11:56:23.506Z" }, ] [[package]] From e67b0016664b50292f4622952551ee883af00701 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 23 Jul 2026 16:17:13 +0200 Subject: [PATCH 62/70] chore: remove stray design notes from docs Assisted-by: ClaudeCode:claude-fable-5 --- ...07-22-zarr-metadata-review-fixes-design.md | 118 ------------------ 1 file changed, 118 deletions(-) delete mode 100644 docs/superpowers/specs/2026-07-22-zarr-metadata-review-fixes-design.md diff --git a/docs/superpowers/specs/2026-07-22-zarr-metadata-review-fixes-design.md b/docs/superpowers/specs/2026-07-22-zarr-metadata-review-fixes-design.md deleted file mode 100644 index c169a822d9..0000000000 --- a/docs/superpowers/specs/2026-07-22-zarr-metadata-review-fixes-design.md +++ /dev/null @@ -1,118 +0,0 @@ -# Zarr Metadata Review Fixes Design - -## Goal - -Correct the significant conformance and public-contract defects found by the -independent branch review without changing the documented ability to construct -temporarily invalid model instances for metadata repair workflows. - -## Scope - -This change fixes four areas: - -1. Validation of raw Zarr v2 `.zarray` and `.zgroup` documents must be distinct - from validation of the library's merged in-memory document representation. -2. Public `TypeIs` predicates must narrow only values that actually inhabit the - type named in their return annotation. -3. The public Zarr v2 dtype type must represent nested structured dtypes. -4. Pydantic-generated JSON schemas must encode the structural constraints that - the corresponding runtime parsers enforce. - -The inaccurate consolidated-metadata docstring is minor and outside this fix. -Existing model-instance pass-through in the Pydantic integration remains -unchanged because it is part of the documented repair-state design. - -## Raw V2 Documents and Merged Models - -The public `ZarrV2ArrayMetadataJSON` and `ZarrV2GroupMetadataJSON` types describe -an in-memory representation in which an optional `attributes` member has been -merged from `.zattrs`. They do not describe the raw `.zarray` and `.zgroup` -objects stored on disk. - -Private raw-document parsers will enforce the storage specification before any -merge occurs: - -- `.zarray` accepts exactly the required array metadata members plus the - optional `dimension_separator` member. Every other member, including - `attributes`, is an error. -- `.zgroup` accepts exactly `zarr_format`. Every other member, including - `attributes`, is an error. -- `.zattrs`, when present, must be a JSON object with string keys and JSON - values. It is then added as the merged `attributes` member. -- An absent `.zattrs` remains distinguishable from an explicitly empty object, - preserving the existing `UNSET` behavior. - -The existing public merged-document validators remain strict. This avoids -silently discarding data and keeps their declared `TypedDict` contracts sound. - -## Sound Type Guards - -Validation and parsing intentionally accept abstract `Mapping` and `Sequence` -inputs. Parsers materialize those inputs into canonical dictionaries and -tuples. Type guards cannot make the same promise because `TypeIs[T]` asserts -that the original object already inhabits `T`. - -`is_json` will therefore require recursively canonical JSON container types: -`dict`, `list`, and `tuple`, with string dictionary keys and canonical children. -It will reject abstract containers such as `UserDict` and `range`, even though -`parse_json` continues to accept and normalize them. - -`is_metadata_field_v3` will likewise require either a string or a concrete -dictionary whose nested values are already canonical. Parsing continues to -accept abstract mappings and materialize them. - -## Recursive V2 Dtype Type - -`ZarrV2DataTypeMetadata` will be a named recursive alias. A structured field's -datatype may be either a dtype string or another structured dtype tuple. The -optional subarray shape remains a tuple of integers. Runtime validation already -accepts this structure, so this change aligns static typing and generated schema -with existing spec-compliant behavior. - -## Pydantic JSON Schema Parity - -The Pydantic field aliases will retain their current runtime behavior: raw -documents are parsed through the core model constructors, and existing core -model instances pass through unchanged. - -Their JSON schema input definitions will be replaced or annotated with -constraints that express the runtime document rules, including: - -- non-negative shape and chunk dimensions; -- at least one v3 codec; -- closed named-configuration objects; -- closed v2 merged documents and consolidated payloads where runtime parsing is - closed; -- `must_understand` constraints at mandatory v3 extension points; and -- the recursive v2 structured-dtype representation. - -Constraints that depend on semantic interpretation outside this package's -structural validators will not be added. The generated schema should describe -accepted raw document inputs, not the intentionally permissive state of an -already-constructed core model instance. - -## Error Handling - -All new raw-v2 failures use `MetadataValidationError` and preserve precise -locations. An illegal `.zarray` member named `attributes`, for example, reports -the location `("attributes",)` and the existing `invalid_value` kind. -Malformed `.zattrs` values report locations beneath `attributes` after the -merge boundary, consistent with the public merged representation. - -## Testing - -Implementation follows red-green-refactor cycles. Regression coverage will -prove: - -- raw `.zarray` and `.zgroup` reject `attributes` and other extra members; -- valid sibling `.zattrs` still merges, including absent-versus-empty behavior; -- `is_json(range(3))` and `is_metadata_field_v3(UserDict(...))` are false while - the corresponding parsers still normalize those inputs; -- nested structured v2 dtypes type-check through Pydantic and validate at - runtime; and -- generated schemas reject representative inputs that runtime parsing rejects: - empty codecs, negative dimensions, forbidden extra members, and unsupported - `must_understand: false` values. - -Focused package tests, formatting, linting, type checking where configured, and -the full repository test suite will run before the implementation is committed. From 3f17dc0b2818c62b782572fc0429be4bb60445c7 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 23 Jul 2026 16:17:31 +0200 Subject: [PATCH 63/70] fix(zarr-metadata): deep-copy mutable state in to_json output to_json previously returned documents holding direct references to the model's internal dicts (attributes, named-config configurations, extra fields, v2 compressor/filters, consolidated entries), so mutating a serialized document silently mutated the frozen model. Every value that can hold a mutable container is now deep-copied on the way out, with parametrized tests proving mutation independence for all seven models. Assisted-by: ClaudeCode:claude-fable-5 --- .../zarr-metadata/changes/4119.feature.md | 6 +++ .../src/zarr_metadata/model/_array.py | 22 +++++---- .../src/zarr_metadata/model/_group.py | 13 +++-- packages/zarr-metadata/tests/model/_cases.py | 20 ++++++++ .../zarr-metadata/tests/model/test_array.py | 37 +++++++++++++- .../zarr-metadata/tests/model/test_group.py | 48 +++++++++++++++++++ 6 files changed, 133 insertions(+), 13 deletions(-) diff --git a/packages/zarr-metadata/changes/4119.feature.md b/packages/zarr-metadata/changes/4119.feature.md index 7da9b728bf..97e5891c34 100644 --- a/packages/zarr-metadata/changes/4119.feature.md +++ b/packages/zarr-metadata/changes/4119.feature.md @@ -77,3 +77,9 @@ The v2 models treat the `.zattrs` file's presence as part of the store: emits none), while an explicit empty `.zattrs` is `{}` and round-trips as a file. Previously `to_key_value` always emitted `.zattrs`, silently adding a file to stores that never had one. + +`to_json` returns a document that shares no mutable state with the model: +every value that can hold a mutable container (attributes, configurations, +extra fields, v2 codec configurations, fill values, consolidated entries) is +deep-copied on the way out, so editing a serialized document can never +silently mutate the frozen model that produced it. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index bc6f2592f1..595eefc184 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -2,6 +2,7 @@ from __future__ import annotations +import copy import dataclasses from collections.abc import Mapping from dataclasses import dataclass, field @@ -62,7 +63,8 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: return self.name out: ZarrV3NamedConfigJSON = {"name": self.name} if self.configuration: - out["configuration"] = self.configuration + # to_json output shares no mutable state with the model. + out["configuration"] = copy.deepcopy(self.configuration) if not self.must_understand: out["must_understand"] = False return out @@ -250,11 +252,13 @@ def __post_init__(self) -> None: ) def to_json(self) -> ZarrV3ArrayMetadataJSON: + # to_json output shares no mutable state with the model: every value + # that can hold a mutable container is deep-copied. out: ZarrV3ArrayMetadataJSON = { "zarr_format": self.zarr_format, "node_type": self.node_type, "shape": self.shape, - "fill_value": self.fill_value, + "fill_value": copy.deepcopy(self.fill_value), "data_type": self.data_type.to_json(), "chunk_grid": self.chunk_grid.to_json(), "codecs": tuple(codec.to_json() for codec in self.codecs), @@ -263,7 +267,7 @@ def to_json(self) -> ZarrV3ArrayMetadataJSON: if self.dimension_names is not UNSET: out["dimension_names"] = self.dimension_names if len(self.attributes) > 0: - out["attributes"] = self.attributes + out["attributes"] = copy.deepcopy(self.attributes) if len(self.storage_transformers) > 0: out["storage_transformers"] = tuple( transformer.to_json() for transformer in self.storage_transformers @@ -273,7 +277,7 @@ def to_json(self) -> ZarrV3ArrayMetadataJSON: # indexed-write path against `extra_items`, but not the `update(**...)` # overload. for key, value in self.extra_fields.items(): - out[key] = value + out[key] = copy.deepcopy(value) return out @classmethod @@ -424,19 +428,21 @@ def to_json(self) -> ZarrV2ArrayMetadataJSON: `attributes` (they live in the sibling `.zattrs` file). Use `to_key_value` to produce the spec-conforming split for storage. """ + # to_json output shares no mutable state with the model: every value + # that can hold a mutable container is deep-copied. out: ZarrV2ArrayMetadataJSON = { "zarr_format": self.zarr_format, "shape": self.shape, "dtype": self.dtype, "order": self.order, "chunks": self.chunks, - "fill_value": self.fill_value, + "fill_value": copy.deepcopy(self.fill_value), "dimension_separator": self.dimension_separator, - "compressor": self.compressor, - "filters": self.filters, + "compressor": copy.deepcopy(self.compressor), + "filters": copy.deepcopy(self.filters), } if self.attributes is not UNSET: - out["attributes"] = self.attributes + out["attributes"] = copy.deepcopy(self.attributes) return out @classmethod diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index f1a76e2508..4633699f7c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -2,6 +2,7 @@ from __future__ import annotations +import copy import dataclasses from collections.abc import Mapping from dataclasses import dataclass, field @@ -122,19 +123,21 @@ def update(self, **kwargs: Unpack[ZarrV3GroupMetadataPartial]) -> ZarrV3GroupMet return dataclasses.replace(self, **kwargs) def to_json(self) -> ZarrV3GroupMetadataJSON: + # to_json output shares no mutable state with the model: every value + # that can hold a mutable container is deep-copied. out: ZarrV3GroupMetadataJSON = { "zarr_format": self.zarr_format, "node_type": self.node_type, } if len(self.attributes) > 0: - out["attributes"] = self.attributes + out["attributes"] = copy.deepcopy(self.attributes) if self.consolidated_metadata is not UNSET: # Consolidated metadata is a known non-core top-level JSON field. out[CONSOLIDATED_METADATA_KEY_V3] = cast( "ZarrV3ExtensionField", self.consolidated_metadata.to_json() ) for key, value in self.extra_fields.items(): - out[key] = value + out[key] = copy.deepcopy(value) return out @classmethod @@ -303,9 +306,10 @@ def to_json(self) -> ZarrV2GroupMetadataJSON: `attributes` (they live in the sibling `.zattrs` file). Use `to_key_value` to produce the spec-conforming split for storage. """ + # to_json output shares no mutable state with the model. out: ZarrV2GroupMetadataJSON = {"zarr_format": self.zarr_format} if self.attributes is not UNSET: - out["attributes"] = self.attributes + out["attributes"] = copy.deepcopy(self.attributes) return out @classmethod @@ -360,9 +364,10 @@ class ZarrV2ConsolidatedMetadata: metadata: dict[str, JSONValue] def to_json(self) -> dict[str, JSONValue]: + # to_json output shares no mutable state with the model. return { "zarr_consolidated_format": self.zarr_consolidated_format, - "metadata": self.metadata, + "metadata": copy.deepcopy(self.metadata), } @classmethod diff --git a/packages/zarr-metadata/tests/model/_cases.py b/packages/zarr-metadata/tests/model/_cases.py index f925c69dd6..15faa65539 100644 --- a/packages/zarr-metadata/tests/model/_cases.py +++ b/packages/zarr-metadata/tests/model/_cases.py @@ -43,3 +43,23 @@ def raises(self) -> AbstractContextManager[pytest.ExceptionInfo[Exception]]: return pytest.raises(self.exception) pattern = re.escape(self.msg) if self.escape else self.msg return pytest.raises(self.exception, match=pattern) + + +def mutate_nested_containers(value: object) -> None: + """Recursively mutate every mutable container reachable inside `value`. + + Adds a marker key to every dict and appends a marker to every list, + descending through tuples. Used to prove a `to_json` document shares no + mutable state with the model that produced it. + """ + if isinstance(value, dict): + for item in value.values(): + mutate_nested_containers(item) + value["__mutated__"] = "__mutated__" + elif isinstance(value, list): + for item in value: + mutate_nested_containers(item) + value.append("__mutated__") + elif isinstance(value, tuple): + for item in value: + mutate_nested_containers(item) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index f0ff473df1..e7ae0cb752 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1,5 +1,6 @@ """Tests for the metadata models in ``zarr_metadata.model``.""" +import copy import dataclasses import json from collections import UserDict @@ -9,7 +10,7 @@ import pytest from typing_extensions import Unpack -from tests.model._cases import Expect, ExpectFail +from tests.model._cases import Expect, ExpectFail, mutate_nested_containers from zarr_metadata.model import ( ARRAY_METADATA_OPTIONAL_KEYS_V3, ARRAY_METADATA_REQUIRED_KEYS_V3, @@ -647,6 +648,40 @@ def test_v2_roundtrip_json_model_json() -> None: assert ZarrV2ArrayMetadata.from_json(doc).to_json() == doc +# --- to_json shares no mutable state with the model ------------------------ + +TO_JSON_NO_ALIASING_PARAMS = [ + pytest.param( + ZarrV3ArrayMetadata.create_default( + shape=(2,), + attributes={"a": {"b": [1]}}, + codecs=(ZarrV3NamedConfig(name="blosc", configuration={"opts": {"level": 1}}),), + extra_fields={"ext": {"must_understand": False, "cfg": {"x": [1]}}}, + ), + id="v3", + ), + pytest.param( + ZarrV2ArrayMetadata.create_default( + attributes={"a": {"b": [1]}}, + compressor={"id": "zstd", "opts": {"level": 1}}, + filters=({"id": "delta", "cfg": [1]},), + fill_value=[0, 0], + ), + id="v2", + ), +] + + +@pytest.mark.parametrize("model", TO_JSON_NO_ALIASING_PARAMS) +def test_to_json_shares_no_mutable_state_with_model( + model: ZarrV3ArrayMetadata | ZarrV2ArrayMetadata, +) -> None: + """Mutating a document returned by to_json leaves the model unchanged.""" + baseline = copy.deepcopy(model.to_json()) + mutate_nested_containers(model.to_json()) + assert model.to_json() == baseline + + def test_v3_parser_accepts_bare_string_data_type() -> None: """V3 from_json accepts a bare-string data_type and re-serializes it canonically.""" doc = ZarrV3ArrayMetadata.create_default().to_json() diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index f38efeeb8a..4b8c22b84d 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -1,5 +1,6 @@ """Tests for the group and consolidated metadata models in `zarr_metadata.model`.""" +import copy import dataclasses import json from collections import UserDict @@ -7,6 +8,7 @@ import pytest +from tests.model._cases import mutate_nested_containers from zarr_metadata.model import UNSET from zarr_metadata.model._array import ZarrV3ArrayMetadata from zarr_metadata.model._group import ( @@ -522,3 +524,49 @@ def test_group_v3_null_consolidated_metadata_repaired_to_absence() -> None: assert model.consolidated_metadata is UNSET assert "consolidated_metadata" not in model.to_json() assert model == ZarrV3GroupMetadata.from_json({"zarr_format": 3, "node_type": "group"}) + + +# --- to_json shares no mutable state with the model ------------------------ + +TO_JSON_NO_ALIASING_PARAMS = [ + pytest.param( + ZarrV3GroupMetadata.create_default( + attributes={"a": {"b": [1]}}, + consolidated_metadata=ZarrV3ConsolidatedMetadata( + metadata={ + "child": ZarrV3ArrayMetadata.create_default(attributes={"x": {"y": 1}}), + "grp": ZarrV3GroupMetadata.create_default(attributes={"x": {"y": 1}}), + } + ), + extra_fields={"ext": {"must_understand": False, "cfg": {"x": [1]}}}, + ), + id="v3-group", + ), + pytest.param( + ZarrV2GroupMetadata.create_default(attributes={"a": {"b": [1]}}), + id="v2-group", + ), + pytest.param( + ZarrV3ConsolidatedMetadata( + metadata={"child": ZarrV3ArrayMetadata.create_default(attributes={"x": {"y": 1}})} + ), + id="v3-consolidated", + ), + pytest.param( + ZarrV2ConsolidatedMetadata(metadata={"a/.zarray": {"nested": {"x": [1]}}}), + id="v2-consolidated", + ), +] + + +@pytest.mark.parametrize("model", TO_JSON_NO_ALIASING_PARAMS) +def test_to_json_shares_no_mutable_state_with_model( + model: ZarrV3GroupMetadata + | ZarrV2GroupMetadata + | ZarrV3ConsolidatedMetadata + | ZarrV2ConsolidatedMetadata, +) -> None: + """Mutating a document returned by to_json leaves the model unchanged.""" + baseline = copy.deepcopy(model.to_json()) + mutate_nested_containers(model.to_json()) + assert model.to_json() == baseline From eefa424df0e7cfd1683ca7fb27996384141fea72 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:11:51 +0200 Subject: [PATCH 64/70] chore(deps): bump the actions group with 3 updates (#4181) Bumps the actions group with 3 updates: [actions/labeler](https://github.com/actions/labeler), [actions/attest](https://github.com/actions/attest) and [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action). Updates `actions/labeler` from 6.2.0 to 7.0.0 - [Release notes](https://github.com/actions/labeler/releases) - [Commits](https://github.com/actions/labeler/compare/b8dd2d9be0f68b860e7dae5dae7d772984eacd6d...bf12e9b00b37c5c0ca2b87b79b2daf7891dbda13) Updates `actions/attest` from 4.1.1 to 4.2.0 - [Release notes](https://github.com/actions/attest/releases) - [Changelog](https://github.com/actions/attest/blob/main/RELEASE.md) - [Commits](https://github.com/actions/attest/compare/a1948c3f048ba23858d222213b7c278aabede763...f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6) Updates `zizmorcore/zizmor-action` from 0.5.7 to 0.6.0 - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/192e21d79ab29983730a13d1382995c2307fbcaa...6599ee8b7a49aef6a770f63d261d214911a7ce02) --- updated-dependencies: - dependency-name: actions/labeler dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: actions/attest dependency-version: 4.2.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: zizmorcore/zizmor-action dependency-version: 0.6.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/needs_release_notes.yml | 2 +- .github/workflows/releases.yml | 2 +- .github/workflows/zarr-metadata-release.yml | 4 ++-- .github/workflows/zizmor.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/needs_release_notes.yml b/.github/workflows/needs_release_notes.yml index fa555d1478..e001e8cd43 100644 --- a/.github/workflows/needs_release_notes.yml +++ b/.github/workflows/needs_release_notes.yml @@ -21,7 +21,7 @@ jobs: pull-requests: write # Required to add labels to PRs runs-on: ubuntu-latest steps: - - uses: actions/labeler@b8dd2d9be0f68b860e7dae5dae7d772984eacd6d # v6.2.0 + - uses: actions/labeler@bf12e9b00b37c5c0ca2b87b79b2daf7891dbda13 # v7.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} sync-labels: true diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 4d460f4a56..fe0d09f300 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -81,7 +81,7 @@ jobs: name: releases path: dist - name: Generate artifact attestation - uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0 with: subject-path: dist/* - name: Publish package to PyPI diff --git a/.github/workflows/zarr-metadata-release.yml b/.github/workflows/zarr-metadata-release.yml index 5021f79d2e..bc9ecf9871 100644 --- a/.github/workflows/zarr-metadata-release.yml +++ b/.github/workflows/zarr-metadata-release.yml @@ -82,7 +82,7 @@ jobs: path: dist - name: Generate artifact attestation - uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0 with: subject-path: dist/* @@ -107,7 +107,7 @@ jobs: path: dist - name: Generate artifact attestation - uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0 with: subject-path: dist/* diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 6250426bae..1567bea713 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -32,4 +32,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7 + uses: zizmorcore/zizmor-action@6599ee8b7a49aef6a770f63d261d214911a7ce02 # v0.6.0 From 0b727571268c8962717ab5c23965c4642fe6bd36 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 27 Jul 2026 19:31:14 -0700 Subject: [PATCH 65/70] feat: ZipStore accepts open binary file-like objects (#4187) Allows constructing a ZipStore from any seekable binary reader, enabling zip archives on remote storage: - io objects (BytesIO, fsspec file objects) are used directly - minimal readers that are not io.IOBase instances and whose read() may return buffer-protocol objects rather than bytes (e.g. obstore.ReadableFile) are adapted via a small io.RawIOBase wrapper when opened for reading clear()/move() raise NotImplementedError for file-object-backed stores. Co-authored-by: Claude Fable 5 --- changes/4187.feature.md | 4 + docs/user-guide/storage.md | 13 +++ src/zarr/storage/_zip.py | 114 +++++++++++++++++++++++-- tests/test_store/test_zip.py | 159 +++++++++++++++++++++++++++++++++++ 4 files changed, 281 insertions(+), 9 deletions(-) create mode 100644 changes/4187.feature.md diff --git a/changes/4187.feature.md b/changes/4187.feature.md new file mode 100644 index 0000000000..87133e2034 --- /dev/null +++ b/changes/4187.feature.md @@ -0,0 +1,4 @@ +`ZipStore` now accepts an open binary file-like object in place of a path, enabling +zip archives on remote storage (e.g. a file opened with `fsspec` or an +`obstore.ReadableFile`). Operations that require a filesystem location +(`clear`, `move`) raise `NotImplementedError` for file-object-backed stores. diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index 7e0154b2a0..0ba6202c76 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -124,6 +124,19 @@ array = zarr.create_array(store=store, shape=(2,), dtype='float64') print(array) ``` +In place of a path, `ZipStore` also accepts an open binary file object (for +example a file opened with `fsspec`, or an `obstore` reader), enabling zip +archives on remote storage. The file must stay open for as long as the store +is in use: + +```python exec="true" session="storage" source="above" result="ansi" +store.close() +f = open('data.zip', mode='rb') # must stay open while the store is used +array = zarr.open_array(store=zarr.storage.ZipStore(f), mode='r') +print(array[:]) +f.close() +``` + ### Remote Store The [`zarr.storage.FsspecStore`][] stores the contents of a Zarr hierarchy following the same diff --git a/src/zarr/storage/_zip.py b/src/zarr/storage/_zip.py index 430b0c3e2a..69ae18bc2c 100644 --- a/src/zarr/storage/_zip.py +++ b/src/zarr/storage/_zip.py @@ -1,12 +1,13 @@ from __future__ import annotations +import io import os import shutil import threading import time import zipfile from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal +from typing import IO, TYPE_CHECKING, Any, Literal from zarr.abc.store import ( ByteRequest, @@ -23,14 +24,67 @@ ZipStoreAccessModeLiteral = Literal["r", "w", "a"] +class _RawReaderAdapter(io.RawIOBase): + """ + Adapt a minimal seekable reader to the `io` interface `zipfile` needs. + + Some file-like objects (e.g. `obstore.ReadableFile`) implement + `read`/`seek`/`tell` but are not `io.IOBase` instances, and their + `read` may return a buffer-protocol object rather than `bytes`. + Wrapping in this adapter plus `io.BufferedReader` yields real `bytes`. + + Reads are clamped to the bytes remaining before EOF: some readers + (obstore < 0.6) raise on short reads rather than returning fewer bytes. + The size is cached, which is safe because the adapter is only used for + read-only access. + """ + + def __init__(self, fileobj: IO[bytes]) -> None: + self._fileobj = fileobj + self._size: int | None = None + + def _get_size(self) -> int: + if self._size is None: + pos = self._fileobj.tell() + self._size = self._fileobj.seek(0, os.SEEK_END) + self._fileobj.seek(pos) + return self._size + + def readable(self) -> bool: + return True + + def seekable(self) -> bool: + return True + + def seek(self, pos: int, whence: int = 0) -> int: + return self._fileobj.seek(pos, whence) + + def tell(self) -> int: + return self._fileobj.tell() + + def readinto(self, b: Any) -> int: + n_requested = min(len(b), self._get_size() - self._fileobj.tell()) + if n_requested <= 0: + return 0 + data = self._fileobj.read(n_requested) + n = len(data) + b[:n] = memoryview(data) + return n + + class ZipStore(Store): """ Store using a ZIP file. Parameters ---------- - path : str - Location of file. + path : str, Path, or IO[bytes] + Location of file, or an open binary file object. A file object must + support `read`, `seek`, and `tell`; objects that are not `io.IOBase` + instances (e.g. an `obstore` reader) are adapted automatically but + can only be used for reading (`mode="r"`). The file object must stay + open for the lifetime of the store, and operations that require a + filesystem location (`clear`, `move`, pickling) are not supported. mode : str, optional One of 'r' to read an existing file, 'w' to truncate and write a new file, 'a' to append to an existing file, or 'x' to exclusively create @@ -58,16 +112,17 @@ class ZipStore(Store): supports_deletes: bool = False supports_listing: bool = True - path: Path + path: Path | None compression: int allowZip64: bool _zf: zipfile.ZipFile _lock: threading.RLock + _fileobj: IO[bytes] | None def __init__( self, - path: Path | str, + path: Path | str | IO[bytes], *, mode: ZipStoreAccessModeLiteral = "r", read_only: bool | None = None, @@ -81,8 +136,28 @@ def __init__( if isinstance(path, str): path = Path(path) - assert isinstance(path, Path) - self.path = path # root? + if isinstance(path, Path): + self.path = path # root? + self._fileobj = None + else: + self.path = None + if not isinstance(path, io.IOBase): + if not all( + callable(getattr(path, attr, None)) for attr in ("read", "seek", "tell") + ): + raise TypeError( + f"expected a path or an open binary file object supporting " + f"read/seek/tell, got {type(path).__name__}" + ) + if mode != "r": + raise TypeError( + f"a file object that is not an io.IOBase instance can only be " + f"opened for reading (mode='r', got mode={mode!r})" + ) + # e.g. an obstore ReadableFile: readable and seekable, but + # not an io object and reads may not return bytes + path = io.BufferedReader(_RawReaderAdapter(path)) + self._fileobj = path self._zmode = mode self.compression = compression @@ -95,7 +170,7 @@ def _sync_open(self) -> None: self._lock = threading.RLock() self._zf = zipfile.ZipFile( - self.path, + self.path if self.path is not None else self._fileobj, # type: ignore[arg-type] mode=self._zmode, compression=self.compression, allowZip64=self.allowZip64, @@ -107,6 +182,13 @@ async def _open(self) -> None: self._sync_open() def __getstate__(self) -> dict[str, Any]: + if self.path is None: + # A path-backed store pickles its path and reopens the file on + # unpickling; an open file object cannot be serialized that way. + raise TypeError( + "cannot pickle a ZipStore backed by a file-like object; " + "construct the store from a path instead" + ) # We need a copy to not modify the state of the original store state = self.__dict__.copy() for attr in ["_zf", "_lock"]: @@ -130,6 +212,10 @@ async def clear(self) -> None: # docstring inherited with self._lock: self._check_writable() + if self.path is None: + raise NotImplementedError( + "clear() is not supported for a ZipStore backed by a file-like object" + ) self._zf.close() os.remove(self.path) self._zf = zipfile.ZipFile( @@ -137,13 +223,19 @@ async def clear(self) -> None: ) def __str__(self) -> str: + if self.path is None: + return f"zip://{self._fileobj!r}" return f"zip://{self.path}" def __repr__(self) -> str: return f"ZipStore('{self}')" def __eq__(self, other: object) -> bool: - return isinstance(other, type(self)) and self.path == other.path + return ( + isinstance(other, type(self)) + and self.path == other.path + and self._fileobj is other._fileobj + ) def _get( self, @@ -297,6 +389,10 @@ async def move(self, path: Path | str) -> None: """ Move the store to another path. """ + if self.path is None: + raise NotImplementedError( + "move() is not supported for a ZipStore backed by a file-like object" + ) if isinstance(path, str): path = Path(path) self.close() diff --git a/tests/test_store/test_zip.py b/tests/test_store/test_zip.py index ed69114b51..0d8dadd18a 100644 --- a/tests/test_store/test_zip.py +++ b/tests/test_store/test_zip.py @@ -1,6 +1,8 @@ from __future__ import annotations +import io import os +import pickle import shutil import tempfile import zipfile @@ -188,6 +190,163 @@ async def test_move(self, tmp_path: Path) -> None: assert np.array_equal(array[...], np.arange(10)) +class TestZipStoreFileObj: + """ZipStore backed by an open binary file-like object instead of a path.""" + + @pytest.fixture + def zip_bytes(self, tmp_path: Path) -> bytes: + path = tmp_path / "data.zip" + store = ZipStore(path, mode="w") + zarr.create_array(store, data=np.arange(10), chunks=(5,)) + store.close() + return path.read_bytes() + + def test_read_from_fileobj(self, zip_bytes: bytes) -> None: + # an existing archive can be read through any seekable binary reader + store = ZipStore(io.BytesIO(zip_bytes), mode="r") + array = zarr.open_array(store, mode="r") + assert np.array_equal(array[...], np.arange(10)) + assert store.path is None + + def test_write_to_fileobj(self) -> None: + # a writable file object receives the archive; the bytes it holds + # after close() are a complete, reopenable zip + buffer = io.BytesIO() + store = ZipStore(buffer, mode="w", read_only=False) + zarr.create_array(store, data=np.arange(4)) + store.close() + + roundtrip = ZipStore(io.BytesIO(buffer.getvalue()), mode="r") + array = zarr.open_array(roundtrip, mode="r") + assert np.array_equal(array[...], np.arange(4)) + + async def test_clear_unsupported(self, zip_bytes: bytes) -> None: + # clear() requires a filesystem location, so it raises a clear error + # for file-object-backed stores + store = ZipStore(io.BytesIO(zip_bytes), mode="a", read_only=False) + store._sync_open() + with pytest.raises(NotImplementedError, match="clear.*file-like"): + await store.clear() + + async def test_move_unsupported(self, zip_bytes: bytes) -> None: + # move() requires a filesystem location, so it raises a clear error + # for file-object-backed stores + store = ZipStore(io.BytesIO(zip_bytes), mode="a", read_only=False) + store._sync_open() + with pytest.raises(NotImplementedError, match="move.*file-like"): + await store.move("elsewhere.zip") + + def test_invalid_file_object_rejected(self) -> None: + # objects without read/seek/tell are rejected at construction, not + # deep inside zipfile + with pytest.raises(TypeError, match="read/seek/tell"): + ZipStore(42, mode="r") # type: ignore[arg-type] + + @pytest.mark.parametrize("mode", ["w", "a", "x"]) + def test_non_iobase_reader_write_modes_rejected(self, zip_bytes: bytes, mode: str) -> None: + # readers that are not io.IOBase instances are adapted for reading + # only; write modes are rejected at construction with a clear error + class MinimalReader: + def __init__(self, data: bytes) -> None: + self._buffer = io.BytesIO(data) + + def read(self, size: int, /) -> bytes: + return self._buffer.read(size) + + def seek(self, pos: int, whence: int = 0, /) -> int: + return self._buffer.seek(pos, whence) + + def tell(self) -> int: + return self._buffer.tell() + + with pytest.raises(TypeError, match="opened for reading"): + ZipStore(MinimalReader(zip_bytes), mode=mode, read_only=False) # type: ignore[arg-type] + + def test_fsspec_file(self, tmp_path: Path, zip_bytes: bytes) -> None: + # a file opened through fsspec (already an io.IOBase) is used directly; + # fsspec's local filesystem stands in for a remote one + fsspec = pytest.importorskip("fsspec") + + path = tmp_path / "fsspec.zip" + path.write_bytes(zip_bytes) + with fsspec.open(f"local://{path}", "rb") as fileobj: + store = ZipStore(fileobj, mode="r") + array = zarr.open_array(store, mode="r") + assert np.array_equal(array[...], np.arange(10)) + assert store.path is None + + def test_obstore_reader(self, tmp_path: Path, zip_bytes: bytes) -> None: + # obstore's ReadableFile is not an io.IOBase and its read() returns a + # buffer-protocol object; ZipStore adapts it via _RawReaderAdapter + obstore = pytest.importorskip("obstore") + from obstore.store import LocalStore as ObstoreLocalStore + + (tmp_path / "obstore.zip").write_bytes(zip_bytes) + reader = obstore.open_reader(ObstoreLocalStore(str(tmp_path)), "obstore.zip") + store = ZipStore(reader, mode="r") + array = zarr.open_array(store, mode="r") + assert np.array_equal(array[...], np.arange(10)) + + def test_raw_reader_adapter_eof(self) -> None: + from zarr.storage._zip import _RawReaderAdapter + + class MinimalReader: + """Non-io.IOBase reader exposing only read/seek/tell, like obstore.""" + + def __init__(self, data: bytes) -> None: + self._buffer = io.BytesIO(data) + + def read(self, size: int, /) -> bytes: + return self._buffer.read(size) + + def seek(self, pos: int, whence: int = 0, /) -> int: + return self._buffer.seek(pos, whence) + + def tell(self) -> int: + return self._buffer.tell() + + # the adapter must clamp reads to EOF: some readers (obstore < 0.6) + # raise on short reads instead of returning fewer bytes + data = b"0123456789" + adapter = _RawReaderAdapter(MinimalReader(data)) # type: ignore[arg-type] + + # A read straddling EOF returns only the remaining bytes. + adapter.seek(len(data) - 3) + buf = bytearray(8) + assert adapter.readinto(buf) == 3 + assert bytes(buf[:3]) == data[-3:] + + # A read at EOF returns 0. + assert adapter.tell() == len(data) + assert adapter.readinto(bytearray(8)) == 0 + + def test_pickle_fileobj_raises(self, zip_bytes: bytes) -> None: + # an open file object cannot be reliably serialized, so pickling a + # file-object-backed store raises with a pointer at the alternative + store = ZipStore(io.BytesIO(zip_bytes), mode="r") + with pytest.raises(TypeError, match="cannot pickle a ZipStore backed by a file-like"): + pickle.dumps(store) + + def test_pickle_path_backed_roundtrip(self, tmp_path: Path, zip_bytes: bytes) -> None: + # path-backed stores remain picklable: the path is serialized and the + # archive is reopened on unpickling + path = tmp_path / "pickled.zip" + path.write_bytes(zip_bytes) + store = ZipStore(path, mode="r") + unpickled = pickle.loads(pickle.dumps(store)) + array = zarr.open_array(unpickled, mode="r") + assert np.array_equal(array[...], np.arange(10)) + + def test_str_and_eq(self, zip_bytes: bytes) -> None: + # file-object-backed stores stringify with the object repr and + # compare equal only when backed by the very same file object + fileobj = io.BytesIO(zip_bytes) + store = ZipStore(fileobj, mode="r") + assert str(store).startswith("zip://<") + assert store == ZipStore(fileobj, mode="r") + assert store != ZipStore(io.BytesIO(zip_bytes), mode="r") + + class ZipStoreLifecycleMachine(RuleBasedStateMachine): """Drive a ZipStore through construct / open / write / close transitions. From 40ab975e4b984c2a6bc4f2debb60898c75f77c33 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 28 Jul 2026 15:09:18 +0200 Subject: [PATCH 66/70] docs(zarr-metadata): describe the widened package scope The README tagline, intro, and scope section (and the PyPI description) still presented the package as type definitions only. Restructure them around the two layers plus optional integration, extend the contribution scope to models and structural validation, and state the runtime-behavior boundary explicitly. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/README.md | 33 +++++++++++++++++++-------- packages/zarr-metadata/pyproject.toml | 2 +- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index 810d0001f0..69b80d7332 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -1,17 +1,23 @@ # zarr-metadata -Python type definitions for Zarr v2 and v3 metadata. +Python types, models, and validators for Zarr v2 and v3 metadata. ## What this is -A typed-data package: `TypedDict` definitions and `Literal` aliases for the -JSON shapes specified by the [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html) -and [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) -specifications, plus types for [`zarr-extensions`](https://github.com/zarr-developers/zarr-extensions/) -and a few widely-used-but-unspecified entities (e.g. consolidated metadata). -It also provides canonical frozen-dataclass models, structural validators, -parsers, store-key serialization, and optional Pydantic field integrations. -The optional integration requires Pydantic 2.13 or newer. +Two layers and an optional integration: + +- **Typed JSON shapes**: `TypedDict` definitions and `Literal` aliases for the + JSON documents specified by the [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html) + and [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) + specifications, plus types for [`zarr-extensions`](https://github.com/zarr-developers/zarr-extensions/) + and a few widely-used-but-unspecified entities (e.g. consolidated metadata). +- **Document models** (`zarr_metadata.model`): canonical frozen-dataclass + models of whole metadata documents, with structural validators, loc-aware + parsers, and store-key (de)serialization. A document produced by `to_json` + shares no mutable state with the model that produced it. +- **Optional Pydantic integration** (`zarr_metadata.pydantic`, requires + Pydantic 2.13 or newer): each model as a Pydantic field type that validates + raw documents through the same strict parser. ## What this is for @@ -67,7 +73,14 @@ versus `shape`. Consumers should run the model parser after schema validation. At minimum, this library supports what Zarr-Python needs: the complete Zarr v2 and v3 specs, consolidated metadata, and a subset of the metadata defined in `zarr-extensions`. We are generally open to contributions that -add types for Zarr metadata with a published spec. +add types, models, or structural validation for Zarr metadata with a +published spec. + +Runtime array behavior is out of scope: nothing here encodes or decodes +chunks, resolves codec or data type names to implementations, or performs +store I/O. The models begin and end at the metadata documents themselves — +`from_key_value` / `to_key_value` map documents to store keys and bytes, +and everything past that belongs to consumer libraries. ## Releasing diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 10b97f1c1e..edc4b696a6 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "zarr-metadata" dynamic = ["version"] -description = "Spec-defined metadata types for Zarr v2 and v3." +description = "Spec-defined metadata types, models, and validators for Zarr v2 and v3." readme = "README.md" requires-python = ">=3.11" license = "MIT" From 3015957a6eb525d5fa51661bbc69a74041ee826b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 28 Jul 2026 15:11:27 +0200 Subject: [PATCH 67/70] style(zarr-metadata): move None to the end of the JSONValue union The unpinned ruff in CI now enforces RUF036. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/src/zarr_metadata/_common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index 13c2457489..f3259f7b73 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -16,11 +16,11 @@ int | float | bool - | None | str | list["JSONValue"] | tuple["JSONValue", ...] - | Mapping[str, "JSONValue"], + | Mapping[str, "JSONValue"] + | None, ) """A recursive type alias for JSON-encodable values. From cb93ef80f3d825446fe4fa97499d0787b410f0a9 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Tue, 28 Jul 2026 16:56:48 +0200 Subject: [PATCH 68/70] feat: add type-safe `get_array` and `get_group` methods to `AsyncGroup` and `Group` (#4128) * chore(deps): bump the actions group across 1 directory with 8 updates (#176) Bumps the actions group with 8 updates in the / directory: | Package | From | To | | --- | --- | --- | | [prefix-dev/setup-pixi](https://github.com/prefix-dev/setup-pixi) | `0.9.5` | `0.9.6` | | [codecov/codecov-action](https://github.com/codecov/codecov-action) | `6.0.0` | `6.0.1` | | [github/issue-metrics](https://github.com/github/issue-metrics) | `4.2.2` | `4.2.7` | | [j178/prek-action](https://github.com/j178/prek-action) | `2.0.3` | `2.0.4` | | [actions/upload-artifact](https://github.com/actions/upload-artifact) | `7.0.0` | `7.0.1` | | [actions/download-artifact](https://github.com/actions/download-artifact) | `7.0.0` | `8.0.1` | | [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) | `1.13.0` | `1.14.0` | | [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) | `0.5.3` | `0.5.6` | Updates `prefix-dev/setup-pixi` from 0.9.5 to 0.9.6 - [Release notes](https://github.com/prefix-dev/setup-pixi/releases) - [Commits](https://github.com/prefix-dev/setup-pixi/compare/1b2de7f3351f171c8b4dfeb558c639cb58ed4ec0...5185adfbffb4bd703da3010310260805d89ebb11) Updates `codecov/codecov-action` from 6.0.0 to 6.0.1 - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/57e3a136b779b570ffcdbf80b3bdc90e7fab3de2...e79a6962e0d4c0c17b229090214935d2e33f8354) Updates `github/issue-metrics` from 4.2.2 to 4.2.7 - [Release notes](https://github.com/github/issue-metrics/releases) - [Commits](https://github.com/github/issue-metrics/compare/c9e9838147fd355dace335ba787f01b6641a400a...1e38d5e62363e14db8019ed7d106b9855bdba6cc) Updates `j178/prek-action` from 2.0.3 to 2.0.4 - [Release notes](https://github.com/j178/prek-action/releases) - [Commits](https://github.com/j178/prek-action/compare/6ad80277337ad479fe43bd70701c3f7f8aa74db3...bdca6f102f98e2b4c7029491a53dfd366469e33d) Updates `actions/upload-artifact` from 7.0.0 to 7.0.1 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v7...043fb46d1a93c77aae656e7c1c64a875d1fc6a0a) Updates `actions/download-artifact` from 7.0.0 to 8.0.1 - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v7...3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c) Updates `pypa/gh-action-pypi-publish` from 1.13.0 to 1.14.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.13.0...cef221092ed1bacb1cc03d23a2d87d1d172e277b) Updates `zizmorcore/zizmor-action` from 0.5.3 to 0.5.6 - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/b1d7e1fb5de872772f31590499237e7cce841e8e...5f14fd08f7cf1cb1609c1e344975f152c7ee938d) --- updated-dependencies: - dependency-name: prefix-dev/setup-pixi dependency-version: 0.9.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: codecov/codecov-action dependency-version: 6.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: github/issue-metrics dependency-version: 4.2.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: j178/prek-action dependency-version: 2.0.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/upload-artifact dependency-version: 7.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/download-artifact dependency-version: 8.0.1 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: pypa/gh-action-pypi-publish dependency-version: 1.14.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: zizmorcore/zizmor-action dependency-version: 0.5.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feat: add type-safe get_array and get_group methods to AsyncGroup and Group Add `get_array` and `get_group` methods that return the child node at a given path or raise: `ArrayNotFoundError`/`GroupNotFoundError` when no node exists, and `ContainsGroupError`/`ContainsArrayError` when the node is the wrong kind. Paths mirror `getitem` semantics, so nested paths like "subgroup/subarray" work on both the plain and consolidated-metadata lookup routes. Existing tests that fetched a child via `getitem` and then manually narrowed the type with `isinstance` asserts or a walrus expression now use the new methods instead. Assisted-by: ClaudeCode:claude-fable-5 * fix: pass pre-formatted messages to error constructors in get_array/get_group The multi-argument template form of BaseZarrError.__init__ is documented as deprecated; build the message string at the raise site instead, matching every other call site in the codebase. Assisted-by: ClaudeCode:claude-fable-5 * docs: add changelog entry for get_array/get_group Assisted-by: ClaudeCode:claude-fable-5 * docs: demonstrate get_array/get_group in the groups user guide Assisted-by: ClaudeCode:claude-fable-5 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- changes/4128.feature.md | 1 + docs/user-guide/groups.md | 23 ++++ src/zarr/core/group.py | 133 +++++++++++++++++++++++ tests/test_group.py | 81 +++++++++++++- tests/test_metadata/test_consolidated.py | 9 +- tests/test_store/test_zip.py | 5 +- 6 files changed, 239 insertions(+), 13 deletions(-) create mode 100644 changes/4128.feature.md diff --git a/changes/4128.feature.md b/changes/4128.feature.md new file mode 100644 index 0000000000..c62a615ac2 --- /dev/null +++ b/changes/4128.feature.md @@ -0,0 +1 @@ +Added `Group.get_array`, `Group.get_group`, `AsyncGroup.get_array`, and `AsyncGroup.get_group`: type-safe accessors that return the child array or group at a given path, raising `ArrayNotFoundError` / `GroupNotFoundError` if no node exists there, and `ContainsGroupError` / `ContainsArrayError` if the node is not of the requested kind. Unlike `Group.__getitem__`, which returns `Array | Group`, these methods have precise return types. Nested paths like `"subgroup/subarray"` are supported. diff --git a/docs/user-guide/groups.md b/docs/user-guide/groups.md index 337ad39554..7429a03847 100644 --- a/docs/user-guide/groups.md +++ b/docs/user-guide/groups.md @@ -51,6 +51,29 @@ print(root['foo/bar']) print(root['foo/bar/baz']) ``` +Accessing a member with `[]` returns either an [`zarr.Array`][] or a [`zarr.Group`][], depending on +what is stored at the given path. When you expect a node of a particular kind, use +[`zarr.Group.get_array`][] or [`zarr.Group.get_group`][] instead. These methods accept the same +paths as `[]`, but they have precise return types and raise an error if no node exists at the +given path, or if the node is not of the expected kind: + +```python exec="true" session="groups" source="above" result="ansi" +print(root.get_group('foo')) +``` + +```python exec="true" session="groups" source="above" result="ansi" +print(root.get_array('foo/bar/baz')) +``` + +```python exec="true" session="groups" source="above" result="ansi" +from zarr.errors import ContainsGroupError + +try: + root.get_array('foo') +except ContainsGroupError as e: + print(e) +``` + The [`zarr.Group.tree`][] method can be used to print a tree representation of the hierarchy, e.g.: diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 0aaf89234e..922eaf1498 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -51,6 +51,7 @@ from zarr.core.metadata.io import save_metadata from zarr.core.sync import SyncMixin, sync from zarr.errors import ( + ArrayNotFoundError, ContainsArrayError, ContainsGroupError, GroupNotFoundError, @@ -820,6 +821,70 @@ async def get[DefaultT]( except KeyError: return default + async def get_array(self, path: str) -> AnyAsyncArray: + """Obtain an array member of this group, raising if it is absent or not an array. + + Parameters + ---------- + path : str + Path of the array relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subarray`. + + Returns + ------- + AsyncArray + The array at the given path. + + Raises + ------ + ArrayNotFoundError + If no node exists at the given path. + ContainsGroupError + If the node at the given path is a group rather than an array. + """ + store_path = self.store_path / path + try: + node = await self.getitem(path) + except KeyError as e: + msg = f"No array found in store {store_path.store!r} at path {store_path.path!r}" + raise ArrayNotFoundError(msg) from e + if isinstance(node, AsyncGroup): + msg = f"A group exists in store {store_path.store!r} at path {store_path.path!r}." + raise ContainsGroupError(msg) + return node + + async def get_group(self, path: str) -> AsyncGroup: + """Obtain a group member of this group, raising if it is absent or not a group. + + Parameters + ---------- + path : str + Path of the group relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subsubgroup`. + + Returns + ------- + AsyncGroup + The group at the given path. + + Raises + ------ + GroupNotFoundError + If no node exists at the given path. + ContainsArrayError + If the node at the given path is an array rather than a group. + """ + store_path = self.store_path / path + try: + node = await self.getitem(path) + except KeyError as e: + msg = f"No group found in store {store_path.store!r} at path {store_path.path!r}" + raise GroupNotFoundError(msg) from e + if isinstance(node, AsyncArray): + msg = f"An array exists in store {store_path.store!r} at path {store_path.path!r}." + raise ContainsArrayError(msg) + return node + async def _save_metadata(self, ensure_parents: bool = False) -> None: await save_metadata(self.store_path, self.metadata, ensure_parents=ensure_parents) @@ -1880,6 +1945,74 @@ def get[DefaultT]( except KeyError: return default + def get_array(self, path: str) -> AnyArray: + """Obtain an array member of this group, raising if it is absent or not an array. + + Parameters + ---------- + path : str + Path of the array relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subarray`. + + Returns + ------- + Array + The array at the given path. + + Raises + ------ + ArrayNotFoundError + If no node exists at the given path. + ContainsGroupError + If the node at the given path is a group rather than an array. + + Examples + -------- + ```python + import zarr + from zarr.core.group import Group + group = Group.from_store(zarr.storage.MemoryStore()) + group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") + group.get_array("subarray") + # + ``` + """ + return Array(self._sync(self._async_group.get_array(path))) + + def get_group(self, path: str) -> Group: + """Obtain a group member of this group, raising if it is absent or not a group. + + Parameters + ---------- + path : str + Path of the group relative to this group. May contain `/` to reference + a member of a subgroup, e.g. `subgroup/subsubgroup`. + + Returns + ------- + Group + The group at the given path. + + Raises + ------ + GroupNotFoundError + If no node exists at the given path. + ContainsArrayError + If the node at the given path is an array rather than a group. + + Examples + -------- + ```python + import zarr + from zarr.core.group import Group + group = Group.from_store(zarr.storage.MemoryStore()) + group.create_group(name="subgroup") + group.get_group("subgroup") + # + ``` + """ + return Group(self._sync(self._async_group.get_group(path))) + def __delitem__(self, key: str) -> None: """Delete a group member. diff --git a/tests/test_group.py b/tests/test_group.py index 1acd5551ca..29377a5392 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -41,8 +41,10 @@ from zarr.core.metadata.v3 import ArrayV3Metadata from zarr.core.sync import _collect_aiterator, sync from zarr.errors import ( + ArrayNotFoundError, ContainsArrayError, ContainsGroupError, + GroupNotFoundError, MetadataValidationError, ZarrUserWarning, ) @@ -101,7 +103,7 @@ async def test_create_creates_parents(store: Store, zarr_format: ZarrFormat) -> root = await zarr.api.asynchronous.open_group( store=store, ) - agroup = await root.getitem("a") + agroup = await root.get_group("a") assert agroup.attrs == {"key": "value"} # create a child node with a couple intermediates @@ -446,6 +448,77 @@ def test_group_get_with_default(store: Store, zarr_format: ZarrFormat) -> None: assert result.attrs["foo"] == "bar" +def test_group_get_array(store: Store, zarr_format: ZarrFormat) -> None: + """ + `Group.get_array` returns the array at the given path, for both direct child names + and nested paths, and the result is statically typed as an Array. + """ + group = Group.from_store(store, zarr_format=zarr_format) + subgroup = group.create_group(name="subgroup") + subarray = group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") + subsubarray = subgroup.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") + + observed = group.get_array("subarray") + assert isinstance(observed, Array) + assert observed == subarray + assert group.get_array("subgroup/subarray") == subsubarray + + +def test_group_get_array_missing(store: Store, zarr_format: ZarrFormat) -> None: + """ + `Group.get_array` raises `ArrayNotFoundError` when no node exists at the given path. + """ + group = Group.from_store(store, zarr_format=zarr_format) + with pytest.raises(ArrayNotFoundError, match="No array found in store"): + group.get_array("missing") + + +def test_group_get_array_wrong_node_type(store: Store, zarr_format: ZarrFormat) -> None: + """ + `Group.get_array` raises `ContainsGroupError` when the node at the given path is a + group rather than an array. + """ + group = Group.from_store(store, zarr_format=zarr_format) + group.create_group(name="subgroup") + with pytest.raises(ContainsGroupError, match="A group exists in store"): + group.get_array("subgroup") + + +def test_group_get_group(store: Store, zarr_format: ZarrFormat) -> None: + """ + `Group.get_group` returns the group at the given path, for both direct child names + and nested paths, and the result is statically typed as a Group. + """ + group = Group.from_store(store, zarr_format=zarr_format) + subgroup = group.create_group(name="subgroup") + subsubgroup = subgroup.create_group(name="subsubgroup") + + observed = group.get_group("subgroup") + assert isinstance(observed, Group) + assert observed == subgroup + assert group.get_group("subgroup/subsubgroup") == subsubgroup + + +def test_group_get_group_missing(store: Store, zarr_format: ZarrFormat) -> None: + """ + `Group.get_group` raises `GroupNotFoundError` when no node exists at the given path. + """ + group = Group.from_store(store, zarr_format=zarr_format) + with pytest.raises(GroupNotFoundError, match="No group found in store"): + group.get_group("missing") + + +def test_group_get_group_wrong_node_type(store: Store, zarr_format: ZarrFormat) -> None: + """ + `Group.get_group` raises `ContainsArrayError` when the node at the given path is an + array rather than a group. + """ + group = Group.from_store(store, zarr_format=zarr_format) + group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") + with pytest.raises(ContainsArrayError, match="An array exists in store"): + group.get_group("subarray") + + @pytest.mark.parametrize("consolidated", [True, False]) def test_group_delitem(store: Store, zarr_format: ZarrFormat, consolidated: bool) -> None: """ @@ -1469,7 +1542,7 @@ async def test_group_getitem_consolidated(self, store: Store) -> None: # On disk, we've consolidated all the metadata in the root zarr.json group = await zarr.api.asynchronous.open(store=store) - rg0 = await group.getitem("g0") + rg0 = await group.get_group("g0") expected = ConsolidatedMetadata( metadata={ @@ -1490,10 +1563,10 @@ async def test_group_getitem_consolidated(self, store: Store) -> None: ) assert rg0.metadata.consolidated_metadata == expected - rg1 = await rg0.getitem("g1") + rg1 = await rg0.get_group("g1") assert rg1.metadata.consolidated_metadata == expected.metadata["g1"].consolidated_metadata - rg2 = await rg1.getitem("g2") + rg2 = await rg1.get_group("g2") assert rg2.metadata.consolidated_metadata == ConsolidatedMetadata(metadata={}) async def test_group_delitem_consolidated(self, store: Store) -> None: diff --git a/tests/test_metadata/test_consolidated.py b/tests/test_metadata/test_consolidated.py index 3596d2bcaa..e6087435fe 100644 --- a/tests/test_metadata/test_consolidated.py +++ b/tests/test_metadata/test_consolidated.py @@ -111,12 +111,10 @@ async def test_getitem_consolidated_empty_leaf_group( group = await zarr.api.asynchronous.open_consolidated( store=memory_store, zarr_format=zarr_format ) - raw = await group.getitem("raw") - assert isinstance(raw, zarr.AsyncGroup) + raw = await group.get_group("raw") assert raw.metadata.consolidated_metadata is not None - varm = await raw.getitem("varm") - assert isinstance(varm, zarr.AsyncGroup) + varm = await raw.get_group("varm") assert varm.metadata.consolidated_metadata == ConsolidatedMetadata(metadata={}) async def test_open_consolidated_false_raises(self) -> None: @@ -770,8 +768,7 @@ async def test_absolute_path_for_subgroup(self, memory_store: zarr.storage.Memor await zarr.api.asynchronous.consolidate_metadata(memory_store) group = await zarr.api.asynchronous.open_group(store=memory_store) - subgroup = await group.getitem("/a") - assert isinstance(subgroup, AsyncGroup) + subgroup = await group.get_group("/a") members = [x async for x in subgroup.keys()] # noqa: SIM118 assert members == ["b"] diff --git a/tests/test_store/test_zip.py b/tests/test_store/test_zip.py index 0d8dadd18a..32b18c5273 100644 --- a/tests/test_store/test_zip.py +++ b/tests/test_store/test_zip.py @@ -22,7 +22,6 @@ import zarr from zarr import create_array from zarr.core.buffer import Buffer, cpu, default_buffer_prototype -from zarr.core.group import Group from zarr.core.sync import sync from zarr.storage import ZipStore from zarr.testing.store import StoreTests @@ -141,13 +140,13 @@ def test_externally_zipped_store(self, tmp_path: Path) -> None: zarr_path = tmp_path / "foo.zarr" root = zarr.open_group(store=zarr_path, mode="w") root.require_group("foo") - assert isinstance(foo := root["foo"], Group) # noqa: RUF018 + foo = root.get_group("foo") foo["bar"] = np.array([1]) shutil.make_archive(str(zarr_path), "zip", zarr_path) zip_path = tmp_path / "foo.zarr.zip" zipped = zarr.open_group(ZipStore(zip_path, mode="r"), mode="r") assert list(zipped.keys()) == list(root.keys()) - assert isinstance(group := zipped["foo"], Group) + group = zipped.get_group("foo") assert list(group.keys()) == list(group.keys()) async def test_list_without_explicit_open(self, tmp_path: Path) -> None: From f8cd675d0f66ec6ac31f88620352c5d8dde6876e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 28 Jul 2026 20:29:20 +0200 Subject: [PATCH 69/70] feat(zarr-metadata): export store-key aliases and type to_key_value with them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The six store-key Literal aliases were private and unused: only reachable via underscore modules, and absent from every signature. Export them from zarr_metadata.model beside their constants (matching the package's name/constant pairing everywhere else), and key each to_key_value return mapping by them so the store keys a model can emit are visible in its signature. from_key_value keeps Mapping[str, bytes] input on purpose — it accepts whole store mappings. A pair test guards export and value drift, and the removal note now states the version-placement and JSON-suffix conventions explicitly. Assisted-by: ClaudeCode:claude-fable-5 --- .../zarr-metadata/changes/4119.feature.md | 7 +++++++ .../zarr-metadata/changes/4119.removal.md | 9 ++++++++ .../src/zarr_metadata/model/__init__.py | 12 +++++++++++ .../src/zarr_metadata/model/_array.py | 12 ++++++++--- .../src/zarr_metadata/model/_group.py | 17 +++++++++++---- .../zarr-metadata/tests/model/test_array.py | 21 ++++++++++++++++++- 6 files changed, 70 insertions(+), 8 deletions(-) diff --git a/packages/zarr-metadata/changes/4119.feature.md b/packages/zarr-metadata/changes/4119.feature.md index 97e5891c34..b9d0bb508c 100644 --- a/packages/zarr-metadata/changes/4119.feature.md +++ b/packages/zarr-metadata/changes/4119.feature.md @@ -78,6 +78,13 @@ emits none), while an explicit empty `.zattrs` is `{}` and round-trips as a file. Previously `to_key_value` always emitted `.zattrs`, silently adding a file to stores that never had one. +The store-key `Literal` aliases (`ZarrV2ArrayMetadataStoreKey`, +`ZarrV2AttributesStoreKey`, ...) are exported from `zarr_metadata.model` +alongside their constants, and each `to_key_value` return type is keyed by +them, so the set of store keys a model can emit is visible in its signature. +`from_key_value` deliberately keeps `Mapping[str, bytes]` input: it accepts +any string-keyed store mapping and ignores unrelated keys. + `to_json` returns a document that shares no mutable state with the model: every value that can hold a mutable container (attributes, configurations, extra fields, v2 codec configurations, fill values, consolidated entries) is diff --git a/packages/zarr-metadata/changes/4119.removal.md b/packages/zarr-metadata/changes/4119.removal.md index 0c90f59131..1caee21999 100644 --- a/packages/zarr-metadata/changes/4119.removal.md +++ b/packages/zarr-metadata/changes/4119.removal.md @@ -21,3 +21,12 @@ entity names are reserved for the `zarr_metadata.model` dataclasses: The old names are removed, not aliased. The `zarr_metadata.pydantic` field types take the bare entity names (`ZarrV3ArrayMetadata`, ...), matching the model classes they validate into. + +The conventions, stated once for future additions: CamelCase type names put +the format version first (`ZarrV2ArrayMetadataJSON`, +`ZarrV3ArrayMetadataStoreKey`), while SCREAMING_SNAKE constants and +snake_case functions put it last (`ARRAY_METADATA_STORE_KEY_V2`, +`validate_array_metadata_v3`). The `JSON` suffix marks a raw-document type +whose bare name is taken by (or reserved for) a `zarr_metadata.model` +dataclass; raw field-level types the models hold verbatim +(`ZarrV2CodecMetadata`, `ZarrV3ExtensionField`) keep their bare names. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index f4c7300ccc..e726c54d3e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -17,8 +17,11 @@ ATTRIBUTES_STORE_KEY_V2, ZarrV2ArrayMetadata, ZarrV2ArrayMetadataPartial, + ZarrV2ArrayMetadataStoreKey, + ZarrV2AttributesStoreKey, ZarrV3ArrayMetadata, ZarrV3ArrayMetadataPartial, + ZarrV3ArrayMetadataStoreKey, ZarrV3MetadataField, ZarrV3NamedConfig, ) @@ -28,11 +31,14 @@ GROUP_METADATA_STORE_KEY_V2, GROUP_METADATA_STORE_KEY_V3, ZarrV2ConsolidatedMetadata, + ZarrV2ConsolidatedMetadataStoreKey, ZarrV2GroupMetadata, ZarrV2GroupMetadataPartial, + ZarrV2GroupMetadataStoreKey, ZarrV3ConsolidatedMetadata, ZarrV3GroupMetadata, ZarrV3GroupMetadataPartial, + ZarrV3GroupMetadataStoreKey, ) from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ( @@ -89,14 +95,20 @@ "ValidationProblem", "ZarrV2ArrayMetadata", "ZarrV2ArrayMetadataPartial", + "ZarrV2ArrayMetadataStoreKey", + "ZarrV2AttributesStoreKey", "ZarrV2ConsolidatedMetadata", + "ZarrV2ConsolidatedMetadataStoreKey", "ZarrV2GroupMetadata", "ZarrV2GroupMetadataPartial", + "ZarrV2GroupMetadataStoreKey", "ZarrV3ArrayMetadata", "ZarrV3ArrayMetadataPartial", + "ZarrV3ArrayMetadataStoreKey", "ZarrV3ConsolidatedMetadata", "ZarrV3GroupMetadata", "ZarrV3GroupMetadataPartial", + "ZarrV3GroupMetadataStoreKey", "ZarrV3MetadataField", "ZarrV3NamedConfig", "is_array_metadata_v2", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 595eefc184..c4c967f891 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -321,7 +321,9 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3ArrayMetadata: return cls.from_json(load_store_json(mapping, ARRAY_METADATA_STORE_KEY_V3)) - def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + def to_key_value( + self, *, indent: int | str | None = None + ) -> Mapping[ZarrV3ArrayMetadataStoreKey, bytes]: return {ARRAY_METADATA_STORE_KEY_V3: dump_store_json(self.to_json(), indent=indent)} @@ -481,12 +483,16 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ArrayMetadata: return cls.from_json({**zarray, "attributes": zattrs}) return cls.from_json(zarray) - def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + def to_key_value( + self, *, indent: int | str | None = None + ) -> Mapping[ZarrV2ArrayMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]: # Attributes live only in the sibling `.zattrs` file; the `.zarray` # document must exclude them. The `.zattrs` key is present exactly # when attributes are set (even empty) — UNSET emits no file. zarray = {k: v for k, v in self.to_json().items() if k != "attributes"} - out = {ARRAY_METADATA_STORE_KEY_V2: dump_store_json(zarray, indent=indent)} + out: dict[ZarrV2ArrayMetadataStoreKey | ZarrV2AttributesStoreKey, bytes] = { + ARRAY_METADATA_STORE_KEY_V2: dump_store_json(zarray, indent=indent) + } if self.attributes is not UNSET: out[ATTRIBUTES_STORE_KEY_V2] = dump_store_json(self.attributes, indent=indent) return out diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 4633699f7c..d576833c26 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -31,6 +31,7 @@ if TYPE_CHECKING: from zarr_metadata._common import JSONValue + from zarr_metadata.model._array import ZarrV2AttributesStoreKey from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON from zarr_metadata.v3.array import ZarrV3ExtensionField from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON @@ -186,7 +187,9 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3GroupMetadata: return cls.from_json(load_store_json(mapping, GROUP_METADATA_STORE_KEY_V3)) - def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + def to_key_value( + self, *, indent: int | str | None = None + ) -> Mapping[ZarrV3GroupMetadataStoreKey, bytes]: return {GROUP_METADATA_STORE_KEY_V3: dump_store_json(self.to_json(), indent=indent)} @@ -338,12 +341,16 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2GroupMetadata: return cls.from_json({**zgroup, "attributes": zattrs}) return cls.from_json(zgroup) - def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + def to_key_value( + self, *, indent: int | str | None = None + ) -> Mapping[ZarrV2GroupMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]: # Attributes live only in the sibling `.zattrs` file; the `.zgroup` # document must exclude them. The `.zattrs` key is present exactly # when attributes are set (even empty) — UNSET emits no file. zgroup = {k: v for k, v in self.to_json().items() if k != "attributes"} - out = {GROUP_METADATA_STORE_KEY_V2: dump_store_json(zgroup, indent=indent)} + out: dict[ZarrV2GroupMetadataStoreKey | ZarrV2AttributesStoreKey, bytes] = { + GROUP_METADATA_STORE_KEY_V2: dump_store_json(zgroup, indent=indent) + } if self.attributes is not UNSET: out[ATTRIBUTES_STORE_KEY_V2] = dump_store_json(self.attributes, indent=indent) return out @@ -429,5 +436,7 @@ def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata: def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ConsolidatedMetadata: return cls.from_json(load_store_json(mapping, CONSOLIDATED_METADATA_STORE_KEY_V2)) - def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: + def to_key_value( + self, *, indent: int | str | None = None + ) -> Mapping[ZarrV2ConsolidatedMetadataStoreKey, bytes]: return {CONSOLIDATED_METADATA_STORE_KEY_V2: dump_store_json(self.to_json(), indent=indent)} diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index e7ae0cb752..467ef1e2ad 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -5,7 +5,7 @@ import json from collections import UserDict from collections.abc import Callable -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, get_args import pytest from typing_extensions import Unpack @@ -64,6 +64,25 @@ def test_guards_exported_from_package() -> None: assert hasattr(zarr_metadata.model, name) +def test_store_key_pairs_exported_from_package() -> None: + """Each store-key constant is exported together with its Literal type + alias, and the pair cannot drift apart.""" + import zarr_metadata.model as m + + pairs = [ + ("ARRAY_METADATA_STORE_KEY_V2", "ZarrV2ArrayMetadataStoreKey"), + ("ARRAY_METADATA_STORE_KEY_V3", "ZarrV3ArrayMetadataStoreKey"), + ("ATTRIBUTES_STORE_KEY_V2", "ZarrV2AttributesStoreKey"), + ("GROUP_METADATA_STORE_KEY_V2", "ZarrV2GroupMetadataStoreKey"), + ("GROUP_METADATA_STORE_KEY_V3", "ZarrV3GroupMetadataStoreKey"), + ("CONSOLIDATED_METADATA_STORE_KEY_V2", "ZarrV2ConsolidatedMetadataStoreKey"), + ] + for const_name, alias_name in pairs: + assert const_name in m.__all__ + assert alias_name in m.__all__ + assert (getattr(m, const_name),) == get_args(getattr(m, alias_name)) + + def test_validation_diagnostics_exported_from_package() -> None: """The validation-diagnostic types and validators are exported from the package.""" import zarr_metadata.model From 41a528f1771cf7f319b43f4a6215a190120bf1ec Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 28 Jul 2026 20:39:49 +0200 Subject: [PATCH 70/70] refactor(zarr-metadata): make every public type name parse against a naming grammar Three grammars now cover the public surface, enforced by a conformance test that walks every public module's __all__: - core document/model names: ZarrV{2,3} + entity + optional role suffix (JSON / JSONPartial / Partial / StoreKey) - extension-entity names: registered entity + exactly one role suffix (CodecMetadata, DataTypeName, FillValue, ...) - a closed standalone-vocabulary allowlist for role-less scalar and diagnostic types, with a staleness guard The three .z-file document types were the only names that fit no grammar and are renamed: ZArrayMetadata -> ZarrV2ZArrayJSON, ZGroupMetadata -> ZarrV2ZGroupJSON, ZAttrsMetadata -> ZarrV2ZAttrsJSON. The leading V2 of V2ChunkKeyEncodingMetadata is that encoding's registered entity name, not a format version; its module docstring now says so. Assisted-by: ClaudeCode:claude-fable-5 --- .../zarr-metadata/changes/4119.removal.md | 9 ++ .../src/zarr_metadata/__init__.py | 12 +- .../src/zarr_metadata/v2/__init__.py | 12 +- .../src/zarr_metadata/v2/array.py | 8 +- .../src/zarr_metadata/v2/attributes.py | 6 +- .../src/zarr_metadata/v2/consolidated.py | 14 +-- .../src/zarr_metadata/v2/group.py | 10 +- .../zarr_metadata/v3/chunk_key_encoding/v2.py | 6 + .../zarr-metadata/tests/test_public_api.py | 111 +++++++++++++++++- .../tests/v2/array/test_fixtures.py | 6 +- .../tests/v2/group/test_fixtures.py | 6 +- 11 files changed, 160 insertions(+), 40 deletions(-) diff --git a/packages/zarr-metadata/changes/4119.removal.md b/packages/zarr-metadata/changes/4119.removal.md index 1caee21999..2a9a6f84c4 100644 --- a/packages/zarr-metadata/changes/4119.removal.md +++ b/packages/zarr-metadata/changes/4119.removal.md @@ -17,6 +17,9 @@ entity names are reserved for the `zarr_metadata.model` dataclasses: - `DataTypeMetadataV2` → `ZarrV2DataTypeMetadata` - `ArrayOrderV2` → `ZarrV2ArrayOrder` - `ArrayDimensionSeparatorV2` → `ZarrV2ArrayDimensionSeparator` +- `ZArrayMetadata` → `ZarrV2ZArrayJSON` (the strict on-disk `.zarray` document) +- `ZGroupMetadata` → `ZarrV2ZGroupJSON` (the strict on-disk `.zgroup` document) +- `ZAttrsMetadata` → `ZarrV2ZAttrsJSON` (the `.zattrs` document) The old names are removed, not aliased. The `zarr_metadata.pydantic` field types take the bare entity names (`ZarrV3ArrayMetadata`, ...), matching the @@ -30,3 +33,9 @@ snake_case functions put it last (`ARRAY_METADATA_STORE_KEY_V2`, whose bare name is taken by (or reserved for) a `zarr_metadata.model` dataclass; raw field-level types the models hold verbatim (`ZarrV2CodecMetadata`, `ZarrV3ExtensionField`) keep their bare names. +Extension-entity types put the registered entity name first and end in +exactly one role suffix (`BloscCodecMetadata`, `Uint8DataTypeName`) — the +`V2` in `V2ChunkKeyEncodingMetadata` is that encoding's entity name, not a +format version, which is always spelled `ZarrV2`/`ZarrV3`. Every public +type name is checked against this grammar by +`tests/test_public_api.py::test_public_type_names_comply_with_naming_grammar`. diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index ab7bce2ff2..b5e52e976d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -22,20 +22,20 @@ from zarr_metadata.v2.array import ( ARRAY_DIMENSION_SEPARATOR_V2, ARRAY_ORDER_V2, - ZArrayMetadata, ZarrV2ArrayDimensionSeparator, ZarrV2ArrayMetadataJSON, ZarrV2ArrayMetadataJSONPartial, ZarrV2ArrayOrder, ZarrV2DataTypeMetadata, + ZarrV2ZArrayJSON, ) -from zarr_metadata.v2.attributes import ZAttrsMetadata +from zarr_metadata.v2.attributes import ZarrV2ZAttrsJSON from zarr_metadata.v2.codec import ZarrV2CodecMetadata from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON from zarr_metadata.v2.group import ( ZarrV2GroupMetadataJSON, ZarrV2GroupMetadataJSONPartial, - ZGroupMetadata, + ZarrV2ZGroupJSON, ) from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3.array import ( @@ -338,9 +338,6 @@ "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingSeparator", "ValidationProblem", - "ZArrayMetadata", - "ZAttrsMetadata", - "ZGroupMetadata", "ZarrV2ArrayDimensionSeparator", "ZarrV2ArrayMetadata", "ZarrV2ArrayMetadataJSON", @@ -355,6 +352,9 @@ "ZarrV2GroupMetadataJSON", "ZarrV2GroupMetadataJSONPartial", "ZarrV2GroupMetadataPartial", + "ZarrV2ZArrayJSON", + "ZarrV2ZAttrsJSON", + "ZarrV2ZGroupJSON", "ZarrV3ArrayMetadata", "ZarrV3ArrayMetadataJSON", "ZarrV3ArrayMetadataJSONPartial", diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py index 62dd8168ea..b9001d168e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py @@ -1,21 +1,18 @@ """Zarr v2 metadata types.""" from zarr_metadata.v2.array import ( - ZArrayMetadata, ZarrV2ArrayDimensionSeparator, ZarrV2ArrayMetadataJSON, ZarrV2ArrayOrder, ZarrV2DataTypeMetadata, + ZarrV2ZArrayJSON, ) -from zarr_metadata.v2.attributes import ZAttrsMetadata +from zarr_metadata.v2.attributes import ZarrV2ZAttrsJSON from zarr_metadata.v2.codec import ZarrV2CodecMetadata from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON -from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON, ZGroupMetadata +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON, ZarrV2ZGroupJSON __all__ = [ - "ZArrayMetadata", - "ZAttrsMetadata", - "ZGroupMetadata", "ZarrV2ArrayDimensionSeparator", "ZarrV2ArrayMetadataJSON", "ZarrV2ArrayOrder", @@ -23,4 +20,7 @@ "ZarrV2ConsolidatedMetadataJSON", "ZarrV2DataTypeMetadata", "ZarrV2GroupMetadataJSON", + "ZarrV2ZArrayJSON", + "ZarrV2ZAttrsJSON", + "ZarrV2ZGroupJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/array.py b/packages/zarr-metadata/src/zarr_metadata/v2/array.py index 96c8b75036..84b6446bcb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/array.py @@ -55,13 +55,13 @@ """Tuple of permitted values for the `dimension_separator` field of v2 array metadata.""" -class ZArrayMetadata(TypedDict): +class ZarrV2ZArrayJSON(TypedDict): """ On-disk `.zarray` file content. Strict shape of the JSON document persisted at `/.zarray` for a v2 array. User attributes live in a sibling `.zattrs` file and are - NOT part of this type; see `ZAttrsMetadata`. + NOT part of this type; see `ZarrV2ZAttrsJSON`. See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ @@ -87,7 +87,7 @@ class ZarrV2ArrayMetadataJSON(TypedDict): `attributes` field so a single TypedDict represents the complete in-memory state of a v2 array node. Consumers that read or write a real `.zarray` file should split / merge `attributes` accordingly, - or use `ZArrayMetadata` (strict on-disk) plus `ZAttrsMetadata` directly. + or use `ZarrV2ZArrayJSON` (strict on-disk) plus `ZarrV2ZAttrsJSON` directly. See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ @@ -152,10 +152,10 @@ class ZarrV2ArrayMetadataJSONPartial(TypedDict, total=False): __all__ = [ "ARRAY_DIMENSION_SEPARATOR_V2", "ARRAY_ORDER_V2", - "ZArrayMetadata", "ZarrV2ArrayDimensionSeparator", "ZarrV2ArrayMetadataJSON", "ZarrV2ArrayMetadataJSONPartial", "ZarrV2ArrayOrder", "ZarrV2DataTypeMetadata", + "ZarrV2ZArrayJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py b/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py index 18b8ded9da..f7cc31babe 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py @@ -7,16 +7,16 @@ from zarr_metadata._common import JSONValue -ZAttrsMetadata = Mapping[str, JSONValue] +ZarrV2ZAttrsJSON = Mapping[str, JSONValue] """On-disk `.zattrs` file content. A JSON object holding user-defined attributes for a v2 array or group. Spec-defined keys for arrays / groups live in sibling `.zarray` / `.zgroup` -files (modeled by `ZArrayMetadata` / `ZGroupMetadata`). This type does not +files (modeled by `ZarrV2ZArrayJSON` / `ZarrV2ZGroupJSON`). This type does not constrain the keys or values of the attributes mapping. """ __all__ = [ - "ZAttrsMetadata", + "ZarrV2ZAttrsJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py index 8c077d8291..6b586bb92e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py @@ -10,9 +10,9 @@ from typing_extensions import TypedDict -from zarr_metadata.v2.array import ZArrayMetadata -from zarr_metadata.v2.attributes import ZAttrsMetadata -from zarr_metadata.v2.group import ZGroupMetadata +from zarr_metadata.v2.array import ZarrV2ZArrayJSON +from zarr_metadata.v2.attributes import ZarrV2ZAttrsJSON +from zarr_metadata.v2.group import ZarrV2ZGroupJSON class ZarrV2ConsolidatedMetadataJSON(TypedDict): @@ -24,9 +24,9 @@ class ZarrV2ConsolidatedMetadataJSON(TypedDict): that path. The keys include the filename suffix, not just the node path; the value's shape is determined by which file the key points at: - - `/.zarray` -> `ZArrayMetadata` - - `/.zgroup` -> `ZGroupMetadata` - - `/.zattrs` -> `ZAttrsMetadata` + - `/.zarray` -> `ZarrV2ZArrayJSON` + - `/.zgroup` -> `ZarrV2ZGroupJSON` + - `/.zattrs` -> `ZarrV2ZAttrsJSON` The TypedDict cannot discriminate the value shape on the key suffix at the type level; consumers should narrow at runtime by inspecting @@ -34,7 +34,7 @@ class ZarrV2ConsolidatedMetadataJSON(TypedDict): """ zarr_consolidated_format: int - metadata: Mapping[str, ZArrayMetadata | ZGroupMetadata | ZAttrsMetadata] + metadata: Mapping[str, ZarrV2ZArrayJSON | ZarrV2ZGroupJSON | ZarrV2ZAttrsJSON] __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/group.py b/packages/zarr-metadata/src/zarr_metadata/v2/group.py index 00639b998e..50f2482e6f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/group.py @@ -11,14 +11,14 @@ from zarr_metadata._common import JSONValue -class ZGroupMetadata(TypedDict): +class ZarrV2ZGroupJSON(TypedDict): """ On-disk `.zgroup` file content. Strict shape of the JSON document persisted at `/.zgroup` for a v2 group. The spec defines exactly one field. User attributes live in a sibling `.zattrs` file and are NOT part of this type; see - `ZAttrsMetadata`. + `ZarrV2ZAttrsJSON`. See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ @@ -34,8 +34,8 @@ class ZarrV2GroupMetadataJSON(TypedDict): and `.zattrs` (user attributes). On disk these are persisted as two separate files; this type folds them so a single TypedDict represents the complete in-memory state of a v2 group node. Consumers that read - or write the real on-disk files should use `ZGroupMetadata` (strict - `.zgroup`) plus `ZAttrsMetadata` directly. + or write the real on-disk files should use `ZarrV2ZGroupJSON` (strict + `.zgroup`) plus `ZarrV2ZAttrsJSON` directly. See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ @@ -75,7 +75,7 @@ class ZarrV2GroupMetadataJSONPartial(TypedDict, total=False): __all__ = [ - "ZGroupMetadata", "ZarrV2GroupMetadataJSON", "ZarrV2GroupMetadataJSONPartial", + "ZarrV2ZGroupJSON", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index fef5793626..e2783d296d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -4,6 +4,12 @@ Intended only to allow existing v2 arrays to be converted to v3 without having to rename chunks. Not recommended for new arrays. +Naming note: these are Zarr **v3** types. The leading `V2` in +`V2ChunkKeyEncodingMetadata` (and friends) is the encoding's registered +*entity name* (`"v2"`), not the format-version marker that `ZarrV2...` +names carry — this package's version-prefixed names always spell it +`ZarrV2` / `ZarrV3`. + See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-key-encoding """ diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index de8e271d28..e65c680fd1 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -1,5 +1,7 @@ """Test that the curated front-door names are accessible from the top-level zarr_metadata package.""" +import importlib +import pkgutil import re from typing import get_args @@ -23,12 +25,12 @@ def _group_rank(s: str) -> int: # Category A — metadata-document types "ZarrV2ArrayMetadataJSON", "ZarrV2ArrayMetadataJSONPartial", - "ZArrayMetadata", + "ZarrV2ZArrayJSON", "ZarrV2GroupMetadataJSON", "ZarrV2GroupMetadataJSONPartial", - "ZGroupMetadata", + "ZarrV2ZGroupJSON", "ZarrV2ConsolidatedMetadataJSON", - "ZAttrsMetadata", + "ZarrV2ZAttrsJSON", "ZarrV2CodecMetadata", "ZarrV3ArrayMetadataJSON", "ZarrV3ArrayMetadataJSONPartial", @@ -217,6 +219,109 @@ def test_all_is_grouped_and_unique() -> None: assert len(zm.__all__) == len(set(zm.__all__)) +# --- naming grammar --------------------------------------------------------- + +# Core document/model names: the format version comes first (`ZarrV2` / +# `ZarrV3`), then the CamelCase entity, then an optional role suffix +# (`JSON`, `JSONPartial`, `Partial`, `StoreKey`) — validated loosely here +# because `JSON` decomposes into single-letter words under any strict +# word-splitting regex. +_CORE_NAME = re.compile(r"^ZarrV[23](?:[A-Z][a-z0-9]*)+$") + +# Zarr v3 extension-entity names: the registered entity comes first (`Blosc`, +# `Uint8`, ... — `V2` here is the *entity name* of the v2-compatibility chunk +# key encoding, not a format-version marker, which is always spelled +# `ZarrV2`/`ZarrV3`), followed by exactly one role suffix. +_EXTENSION_ROLES = ( + "CodecConfiguration", + "CodecMetadata", + "CodecName", + "CodecObject", + "ChunkGridConfiguration", + "ChunkGridMetadata", + "ChunkGridName", + "ChunkGridObject", + "ChunkKeyEncodingConfiguration", + "ChunkKeyEncodingMetadata", + "ChunkKeyEncodingName", + "ChunkKeyEncodingObject", + "ChunkKeyEncodingSeparator", + "DataTypeName", + "FillValue", + "Configuration", + "Component", +) +_EXTENSION_NAME = re.compile(r"^(?:[A-Z][a-z0-9]*)+?(?:" + "|".join(_EXTENSION_ROLES) + r")$") + +# Standalone vocabulary: scalar Literal aliases, structural helper shapes, and +# the validation diagnostics. Closed by hand — a new name belongs here only if +# it is genuinely role-less; anything document- or entity-shaped must fit the +# grammars above instead. +_STANDALONE_VOCAB = frozenset( + { + "Base64Bytes", + "BloscCName", + "BloscShuffle", + "CastOutOfRangeMode", + "CastRoundingMode", + "Endianness", + "HexFloat16", + "HexFloat32", + "HexFloat64", + "JSONValue", + "MetadataValidationError", + "NumpyDatetime64", + "NumpyTimeUnit", + "NumpyTimedelta64", + "ProblemKind", + "RectilinearDimSpec", + "ScalarMap", + "ScalarMapEntry", + "ShardingIndexLocation", + "Struct", + "StructField", + "ValidationProblem", + } +) + + +def _public_type_names() -> set[tuple[str, str]]: + """Every (module, CamelCase name) pair exported via a public `__all__`.""" + module_names = {"zarr_metadata"} + for info in pkgutil.walk_packages(zm.__path__, prefix="zarr_metadata."): + if not any(part.startswith("_") for part in info.name.split(".")[1:]): + module_names.add(info.name) + out: set[tuple[str, str]] = set() + for module_name in module_names: + module = importlib.import_module(module_name) + for name in getattr(module, "__all__", ()): + if name.startswith("_") or name.isupper() or name.islower(): + continue + out.add((module_name, name)) + return out + + +def test_public_type_names_comply_with_naming_grammar() -> None: + """Every public type name parses against the package naming grammar: + version-first core names, entity-plus-role extension names, or the closed + standalone vocabulary.""" + exported = _public_type_names() + violations = [ + f"{module}.{name}" + for module, name in sorted(exported) + if name not in _STANDALONE_VOCAB + and not _CORE_NAME.match(name) + and not _EXTENSION_NAME.match(name) + ] + assert not violations, f"names outside the naming grammar: {violations}" + + +def test_standalone_vocab_is_not_stale() -> None: + """Every allowlisted vocabulary name is still actually exported.""" + exported_names = {name for _, name in _public_type_names()} + assert exported_names >= _STANDALONE_VOCAB + + def test_promoted_pairs_drift() -> None: pairs = [ (zm.ENDIANNESS, zm.Endianness), diff --git a/packages/zarr-metadata/tests/v2/array/test_fixtures.py b/packages/zarr-metadata/tests/v2/array/test_fixtures.py index 1576aae8db..578d3647ab 100644 --- a/packages/zarr-metadata/tests/v2/array/test_fixtures.py +++ b/packages/zarr-metadata/tests/v2/array/test_fixtures.py @@ -1,7 +1,7 @@ """Decode v2 array metadata fixtures via pydantic. Each `*.json` file in this directory is a representative on-disk -`.zarray` that should validate cleanly as `ZArrayMetadata` (the strict +`.zarray` that should validate cleanly as `ZarrV2ZArrayJSON` (the strict on-disk shape). User attributes live in sibling `.zattrs` files and are not part of these fixtures. @@ -17,11 +17,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v2.array import ZArrayMetadata +from zarr_metadata.v2.array import ZarrV2ZArrayJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(ZArrayMetadata) +ADAPTER = TypeAdapter(ZarrV2ZArrayJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem) diff --git a/packages/zarr-metadata/tests/v2/group/test_fixtures.py b/packages/zarr-metadata/tests/v2/group/test_fixtures.py index 1ad88a577b..29652185b2 100644 --- a/packages/zarr-metadata/tests/v2/group/test_fixtures.py +++ b/packages/zarr-metadata/tests/v2/group/test_fixtures.py @@ -1,7 +1,7 @@ """Decode v2 group metadata fixtures via pydantic. Each `*.json` file in this directory is a representative on-disk -`.zgroup` that should validate cleanly as `ZGroupMetadata` (the strict +`.zgroup` that should validate cleanly as `ZarrV2ZGroupJSON` (the strict on-disk shape). User attributes live in sibling `.zattrs` files and are not part of these fixtures. """ @@ -14,11 +14,11 @@ import pytest from pydantic import TypeAdapter -from zarr_metadata.v2.group import ZGroupMetadata +from zarr_metadata.v2.group import ZarrV2ZGroupJSON FIXTURES_DIR = Path(__file__).parent FIXTURES = sorted(FIXTURES_DIR.glob("*.json")) -ADAPTER = TypeAdapter(ZGroupMetadata) +ADAPTER = TypeAdapter(ZarrV2ZGroupJSON) @pytest.mark.parametrize("fixture", FIXTURES, ids=lambda p: p.stem)