diff --git a/DATASET_CATALOG_USAGE.md b/DATASET_CATALOG_USAGE.md index c277271..14db65b 100644 --- a/DATASET_CATALOG_USAGE.md +++ b/DATASET_CATALOG_USAGE.md @@ -85,7 +85,7 @@ datasets = list_available_datasets(catalog) # } # Resolve a specific dataset to its CID -cid = resolve_dataset_cid_from_stac( +resolved = resolve_dataset_cid_from_stac( catalog=catalog, collection="ecmwf_era5", dataset="temperature_2m", @@ -93,7 +93,7 @@ cid = resolve_dataset_cid_from_stac( organization="ecmwf", ) -print(f"Dataset CID: {cid}") +print(f"Dataset CID: {resolved.cid} (variant: {resolved.variant})") ``` ## STAC Catalog Structure @@ -317,7 +317,7 @@ def resolve_dataset_cid_from_stac( collection: str, dataset: str, variant: Optional[str] = None -) -> str +) -> ResolvedDataset ``` **Parameters:** @@ -327,7 +327,7 @@ def resolve_dataset_cid_from_stac( - `variant`: Optional variant name **Returns:** -- `str`: IPFS CID (without "ipfs://" prefix) +- `ResolvedDataset`: IPFS CID (without "ipfs://" prefix) and selected variant **Raises:** - `ValueError`: If collection, dataset, or variant is not found @@ -340,14 +340,14 @@ from dclimate_client_py import load_stac_catalog, resolve_dataset_cid_from_stac catalog = load_stac_catalog("https://ipfs-gateway.dclimate.net") # Resolve dataset to CID -cid = resolve_dataset_cid_from_stac( +resolved = resolve_dataset_cid_from_stac( catalog=catalog, collection="ifs", dataset="temperature", variant="single" ) -print(f"Dataset CID: {cid}") +print(f"Dataset CID: {resolved.cid} (variant: {resolved.variant})") ``` ### `get_root_catalog_cid()` @@ -362,7 +362,7 @@ def get_root_catalog_cid() -> str - `str`: The IPFS CID of the latest root STAC catalog **Raises:** -- `requests.HTTPError`: If the API request fails +- `httpx.HTTPError`: If the API request fails - `KeyError`: If response doesn't contain expected 'cid' field **Example:** diff --git a/README.md b/README.md index b8ba98c..dd8ff99 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

# dClimate-Client-Py -[![codecov](https://codecov.io/gh/dClimate/dClimate-Zarr-Client/graph/badge.svg?token=AovaMO6DX5)](https://codecov.io/gh/dClimate/dClimate-Zarr-Client) +[![codecov](https://codecov.io/gh/dClimate/dclimate-client-py/graph/badge.svg)](https://codecov.io/gh/dClimate/dclimate-client-py) Retrieve dClimate GIS zarr datasets stored on IPFS @@ -226,11 +226,11 @@ Various exceptions to be raised for bad or invalid user input. --- -### geo_utils.py +### geotemporal_data.py -Functions to manipulate `xarray` datasets. Contains polygon, rectangle, circle and point spatial -subsetting options, as well as temporal subsetting. Also allows for both spatial and temporal -aggregations. +`GeotemporalData`, a wrapper around `xarray` datasets. Contains polygon, rectangle, circle and +point spatial subsetting options, as well as temporal subsetting. Also allows for both spatial +and temporal aggregations. --- diff --git a/dclimate_client_py/__init__.py b/dclimate_client_py/__init__.py index c2b0305..ed2dd9a 100644 --- a/dclimate_client_py/__init__.py +++ b/dclimate_client_py/__init__.py @@ -1,10 +1,6 @@ # public API -from .client import ( - load_s3, - geo_temporal_query, -) -from .dclimate_client import dClimateClient -from .geotemporal_data import GeotemporalData +from importlib import import_module + from .encryption_codec import ( EncryptionCodec, ) @@ -16,11 +12,8 @@ SpatialExtent, TemporalExtent, ) -from .stac_catalog import ( - load_stac_catalog, - list_available_datasets, -) from .stac_server import ( + ResolvedDataset, resolve_cid_from_stac_server, list_available_datasets_from_stac_server, STAC_SERVER_URL, @@ -43,6 +36,31 @@ X402NotInstalledError, ) +_LAZY_IMPORTS = { + "load_s3": (".client", "load_s3"), + "geo_temporal_query": (".client", "geo_temporal_query"), + "dClimateClient": (".dclimate_client", "dClimateClient"), + "GeotemporalData": (".geotemporal_data", "GeotemporalData"), + "load_stac_catalog": (".stac_catalog", "load_stac_catalog"), + "list_available_datasets": (".stac_catalog", "list_available_datasets"), +} + + +def __getattr__(name: str): + try: + module_name, attribute_name = _LAZY_IMPORTS[name] + except KeyError: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None + + value = getattr(import_module(module_name, __name__), attribute_name) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(_LAZY_IMPORTS)) + + __all__ = [ "dClimateClient", "load_s3", @@ -57,6 +75,7 @@ "TemporalExtent", "load_stac_catalog", "list_available_datasets", + "ResolvedDataset", "resolve_cid_from_stac_server", "list_available_datasets_from_stac_server", "STAC_SERVER_URL", diff --git a/dclimate_client_py/client.py b/dclimate_client_py/client.py index 093ea97..e100c25 100644 --- a/dclimate_client_py/client.py +++ b/dclimate_client_py/client.py @@ -36,21 +36,21 @@ def load_s3( def geo_temporal_query( dataset_name: str, source: typing.Literal["s3"] = "s3", - bucket_name: str = None, - var_name: str = None, + bucket_name: typing.Optional[str] = None, + var_name: typing.Optional[str] = None, gateway_uri_stem: str | None = None, rpc_uri_stem: str | None = None, - forecast_reference_time: str = None, - point_kwargs: dict = None, - circle_kwargs: dict = None, - rectangle_kwargs: dict = None, - polygon_kwargs: dict = None, - multiple_points_kwargs: dict = None, + forecast_reference_time: typing.Optional[str] = None, + point_kwargs: typing.Optional[dict] = None, + circle_kwargs: typing.Optional[dict] = None, + rectangle_kwargs: typing.Optional[dict] = None, + polygon_kwargs: typing.Optional[dict] = None, + multiple_points_kwargs: typing.Optional[dict] = None, bounds=None, - bounds_options: dict = None, - spatial_agg_kwargs: dict = None, - temporal_agg_kwargs: dict = None, - rolling_agg_kwargs: dict = None, + bounds_options: typing.Optional[dict] = None, + spatial_agg_kwargs: typing.Optional[dict] = None, + temporal_agg_kwargs: typing.Optional[dict] = None, + rolling_agg_kwargs: typing.Optional[dict] = None, time_range: typing.Optional[typing.List[datetime.datetime]] = None, # as_of: typing.Optional[datetime.datetime] = None, # Removed as_of point_limit: int = DEFAULT_POINT_LIMIT, diff --git a/dclimate_client_py/concatenate.py b/dclimate_client_py/concatenate.py index 949fdda..c8fbf1e 100644 --- a/dclimate_client_py/concatenate.py +++ b/dclimate_client_py/concatenate.py @@ -17,7 +17,6 @@ def find_split_index( - combined_coords: typing.Any, next_coords: typing.Any, last_coord_value: typing.Any, ) -> int: @@ -27,7 +26,6 @@ def find_split_index( This prevents duplicate data when concatenating datasets. Args: - combined_coords: Coordinates from the combined dataset (for reference) next_coords: Coordinates from the next variant to concatenate last_coord_value: The last coordinate value from the combined dataset @@ -109,10 +107,11 @@ async def concatenate_datasets( Implements smart concatenation logic: 1. Start with first dataset (highest priority) 2. For each subsequent dataset: - - Find the last coordinate value in combined dataset - - Find split index in next dataset where coords > last coord + - Find split index in next dataset where coords > the last coord + accepted so far - Slice next dataset to only include new data - - Concatenate sliced dataset + 3. Concatenate all accepted slices with a single ``xr.concat`` call + (an iterative per-dataset concat is O(n^2) time and ~2x peak memory) Args: datasets: List of xarray datasets to concatenate (in priority order) @@ -144,6 +143,9 @@ async def concatenate_datasets( # Start with the first dataset (highest priority) combined = datasets[0] + datasets_to_concat = [combined] + last_coord_value = combined[dimension].values[-1] + total_coord_count = len(combined[dimension]) logger.debug( f"Starting with dataset 1/{len(datasets)}, " f"{dimension} range: {combined[dimension].values[0]} to {combined[dimension].values[-1]}" @@ -151,9 +153,6 @@ async def concatenate_datasets( # Concatenate each subsequent dataset for i, next_ds in enumerate(datasets[1:], start=2): - # Get the last coordinate value from combined dataset - last_coord_value = combined[dimension].values[-1] - # Get coordinates from next dataset next_coords = next_ds[dimension].values @@ -165,7 +164,6 @@ async def concatenate_datasets( # Find where to split the next dataset try: split_index = find_split_index( - combined[dimension].values, next_coords, last_coord_value, ) @@ -177,18 +175,16 @@ async def concatenate_datasets( # Slice the next dataset to only include new data sliced_next = next_ds.isel({dimension: slice(split_index, None)}) + datasets_to_concat.append(sliced_next) + last_coord_value = sliced_next[dimension].values[-1] + total_coord_count += len(sliced_next[dimension]) logger.debug( f"Sliced dataset {i} to {len(sliced_next[dimension])} new coords" ) - # Concatenate with combined dataset - combined = xr.concat( - [combined, sliced_next], - dim=dimension, - ) logger.debug( - f"After concatenating dataset {i}, total {dimension} coords: {len(combined[dimension])}" + f"After concatenating dataset {i}, total {dimension} coords: {total_coord_count}" ) except NoDataFoundError as e: @@ -196,6 +192,9 @@ async def concatenate_datasets( # Continue to next dataset continue + if len(datasets_to_concat) > 1: + combined = xr.concat(datasets_to_concat, dim=dimension) + logger.info( f"Concatenation complete. Final dataset has {len(combined[dimension])} " f"{dimension} coordinates ranging from {combined[dimension].values[0]} " diff --git a/dclimate_client_py/dclimate_client.py b/dclimate_client_py/dclimate_client.py index 88cc27b..909f608 100644 --- a/dclimate_client_py/dclimate_client.py +++ b/dclimate_client_py/dclimate_client.py @@ -5,13 +5,16 @@ internally, abstracting away KuboCAS lifecycle management. """ +import asyncio import typing from collections.abc import Mapping -import requests +if typing.TYPE_CHECKING: + import pystac + +import httpx import xarray as xr from py_hamt import KuboCAS -import pystac # Import here to avoid circular imports from .ipfs_retrieval import _load_dataset_from_ipfs_cid @@ -20,12 +23,8 @@ from .geotemporal_data import GeotemporalData from .datasets import DatasetMetadata from .dclimate_zarr_errors import InvalidSelectionError -from .stac_catalog import ( - load_stac_catalog, - resolve_dataset_cid_from_stac, - list_available_datasets, -) from .stac_server import ( + ResolvedDataset, resolve_cid_from_stac_server, list_available_datasets_from_stac_server, ) @@ -37,6 +36,8 @@ SirenRegion, ) +DEFAULT_PUBLIC_GATEWAY = "https://ipfs-gateway.dclimate.net" + class dClimateClient: """ @@ -49,10 +50,26 @@ class dClimateClient: ---------- gateway_base_url : str, optional IPFS HTTP Gateway base URL (e.g., "https://ipfs.io" or "http://localhost:8080"). - If None, uses KuboCAS defaults or environment variables. + If None, KuboCAS uses its own defaults while STAC-catalog fallback reads + use ``DEFAULT_PUBLIC_GATEWAY``. rpc_base_url : str, optional IPFS RPC API base URL (e.g., "http://localhost:5001"). If None, uses KuboCAS defaults or environment variables. + concurrency : int, optional + Maximum number of concurrent Kubo gateway and RPC requests. + headers : dict[str, str], optional + Default headers for the internally-created HTTP client. + auth : tuple[str, str], optional + Authentication tuple (username, password) for the internally-created client. + max_retries : int, optional + Maximum number of retries for retryable gateway requests. + initial_delay : float, optional + Initial retry delay in seconds. + backoff_factor : float, optional + Multiplier used for exponential retry backoff. + client_factory : Callable[[], httpx.AsyncClient], optional + Create a separate, fully configured HTTP client for each event loop. + Cannot be combined with ``headers`` or ``auth``. Examples -------- @@ -85,15 +102,37 @@ class dClimateClient: def __init__( self, - gateway_base_url: typing.Optional[str] = "https://ipfs-gateway.dclimate.net", - rpc_base_url: typing.Optional[str] = "https://ipfs-gateway.dclimate.net", + gateway_base_url: typing.Optional[str] = DEFAULT_PUBLIC_GATEWAY, + rpc_base_url: typing.Optional[str] = DEFAULT_PUBLIC_GATEWAY, stac_server_url: typing.Optional[str] = "https://api.stac.dclimate.net", siren: typing.Optional[SirenOptions] = None, - ): + *, + concurrency: typing.Optional[int] = None, + headers: typing.Optional[dict[str, str]] = None, + auth: typing.Optional[tuple[str, str]] = None, + max_retries: typing.Optional[int] = None, + initial_delay: typing.Optional[float] = None, + backoff_factor: typing.Optional[float] = None, + client_factory: typing.Optional[typing.Callable[[], httpx.AsyncClient]] = None, + ) -> None: + if client_factory is not None and (headers is not None or auth is not None): + raise ValueError("client_factory cannot be combined with headers or auth") + self._gateway_base_url = gateway_base_url + self._catalog_gateway_base_url = ( + gateway_base_url if gateway_base_url is not None else DEFAULT_PUBLIC_GATEWAY + ) self._rpc_base_url = rpc_base_url self._stac_server_url = stac_server_url - self._stac_catalog: typing.Optional[pystac.Catalog] = None + self._concurrency = concurrency + self._headers = headers + self._auth = auth + self._max_retries = max_retries + self._initial_delay = initial_delay + self._backoff_factor = backoff_factor + self._client_factory = client_factory + self._stac_catalog: typing.Optional["pystac.Catalog"] = None + self._stac_catalog_lock = asyncio.Lock() self._kubo_cas: typing.Optional[KuboCAS] = None # Note: STAC catalog is loaded lazily (only if STAC server fails) @@ -105,22 +144,79 @@ def __init__( async def __aenter__(self) -> "dClimateClient": """Initialize KuboCAS when entering async context.""" # Create KuboCAS with configured endpoints - self._kubo_cas = KuboCAS( - gateway_base_url=self._gateway_base_url, - rpc_base_url=self._rpc_base_url, + kubo_kwargs: dict[str, typing.Any] = { + "gateway_base_url": self._gateway_base_url, + "rpc_base_url": self._rpc_base_url, + } + optional_kubo_kwargs = { + "concurrency": self._concurrency, + "headers": self._headers, + "auth": self._auth, + "max_retries": self._max_retries, + "initial_delay": self._initial_delay, + "backoff_factor": self._backoff_factor, + "client_factory": self._client_factory, + } + kubo_kwargs.update( + { + key: value + for key, value in optional_kubo_kwargs.items() + if value is not None + } ) + self._kubo_cas = KuboCAS(**kubo_kwargs) # Enter the KuboCAS context manager await self._kubo_cas.__aenter__() return self async def __aexit__(self, exc_type, exc_val, exc_tb): """Clean up KuboCAS when exiting async context.""" - if self._siren_client is not None: - await self._siren_client.aclose() - if self._kubo_cas: - await self._kubo_cas.__aexit__(exc_type, exc_val, exc_tb) + incoming_cancellation = isinstance(exc_val, asyncio.CancelledError) + siren_error: BaseException | None = None + try: + if self._siren_client is not None: + await self._siren_client.aclose() + except BaseException as error: + siren_error = error + + try: + if self._kubo_cas is not None: + await self._kubo_cas.__aexit__(exc_type, exc_val, exc_tb) + except BaseException as kubo_error: + if incoming_cancellation and not isinstance( + kubo_error, asyncio.CancelledError + ): + # Preserve cancellation from the context body. An ordinary + # cleanup failure must not replace task cancellation, but + # remains inspectable through the exception context chain. + if siren_error is not None: + kubo_error.__context__ = siren_error + exc_val.__context__ = kubo_error + return False + if siren_error is None: + raise + # Both cleanups failed. Follow the AsyncExitStack convention (the + # later error propagates with the earlier as __context__), except + # that a cancellation always outranks an ordinary error. + if isinstance(siren_error, asyncio.CancelledError) and not isinstance( + kubo_error, asyncio.CancelledError + ): + raise siren_error + kubo_error.__context__ = siren_error + raise + finally: self._kubo_cas = None + if siren_error is not None: + if incoming_cancellation and not isinstance( + siren_error, asyncio.CancelledError + ): + exc_val.__context__ = siren_error + else: + raise siren_error + + return False + @staticmethod def _apply_zarr_group_metadata(ds: xr.Dataset, metadata: DatasetMetadata) -> None: loaded_zarr_group = ds.attrs.get("_ipfs_zarr_group") @@ -194,7 +290,7 @@ async def load_dataset( If dataset cannot be found in STAC catalog InvalidSelectionError If collection parameter is not provided (when not using direct CID) - requests.RequestException + httpx.HTTPError If connection to IPFS gateway fails Examples @@ -220,21 +316,22 @@ async def load_dataset( "Use 'async with dClimateClient() as client:'" ) - resolved_collection = collection - if ( - organization - and collection - and not collection.startswith(f"{organization}_") - ): - resolved_collection = f"{organization}_{collection}" - # Case 1: Direct CID provided - bypass catalog resolution + metadata: DatasetMetadata if cid: - slug_collection = resolved_collection or collection or "unknown" + direct_collection = collection + if ( + organization + and direct_collection + and not direct_collection.startswith(f"{organization}_") + ): + direct_collection = f"{organization}_{direct_collection}" + slug_collection = direct_collection or "unknown" + direct_variant = variant or "unknown" dataset_slug = ( - f"{organization}/{slug_collection}/{dataset}/{variant or 'default'}" + f"{organization}/{slug_collection}/{dataset}/{direct_variant}" if organization - else f"{slug_collection}/{dataset}/{variant or 'default'}" + else f"{slug_collection}/{dataset}/{direct_variant}" ) ds = await _load_dataset_from_ipfs_cid( ipfs_cid=cid, @@ -244,10 +341,10 @@ async def load_dataset( ) # Build metadata for direct CID case - metadata: DatasetMetadata = { - "collection": resolved_collection or "unknown", + metadata = { + "collection": direct_collection or "unknown", "dataset": dataset, - "variant": variant or "unknown", + "variant": direct_variant, "slug": dataset_slug, "cid": cid, "url": None, @@ -255,8 +352,8 @@ async def load_dataset( "source": "direct_cid", "organization": organization or ( - resolved_collection.split("_")[0] - if resolved_collection and "_" in resolved_collection + direct_collection.split("_")[0] + if direct_collection and "_" in direct_collection else None ), } @@ -273,31 +370,49 @@ async def load_dataset( "collection parameter is required. Use client.list_datasets() to see available collections." ) - final_cid = None + resolved_collection = collection + if organization and not collection.startswith(f"{organization}_"): + resolved_collection = f"{organization}_{collection}" + + resolved: typing.Optional[ResolvedDataset] = None # Try STAC server first (faster, avoids loading IPFS catalog) if self._stac_server_url: try: - final_cid = resolve_cid_from_stac_server( + resolved = await asyncio.to_thread( + resolve_cid_from_stac_server, collection=resolved_collection, dataset=dataset, variant=variant, server_url=self._stac_server_url, ) - except (requests.RequestException, ValueError): + except (httpx.HTTPError, ValueError): # Fall back when server lookup fails or returns no usable match. pass # Fallback: Resolve via STAC catalog from IPFS - if final_cid is None: + if resolved is None: + from .stac_catalog import ( + list_available_datasets, + load_stac_catalog, + resolve_dataset_cid_from_stac, + ) + # Lazy load STAC catalog if self._stac_catalog is None: - self._stac_catalog = load_stac_catalog( - gateway_url=self._gateway_base_url - ) + async with self._stac_catalog_lock: + if self._stac_catalog is None: + self._stac_catalog = await asyncio.to_thread( + load_stac_catalog, + gateway_url=self._catalog_gateway_base_url, + headers=self._headers, + auth=self._auth, + ) if not organization and resolved_collection: - available = list_available_datasets(self._stac_catalog) + available = await asyncio.to_thread( + list_available_datasets, self._stac_catalog + ) if resolved_collection not in available: prefixed_matches = [ coll_id @@ -307,7 +422,8 @@ async def load_dataset( if len(prefixed_matches) == 1: resolved_collection = prefixed_matches[0] - final_cid = resolve_dataset_cid_from_stac( + resolved = await asyncio.to_thread( + resolve_dataset_cid_from_stac, catalog=self._stac_catalog, collection=resolved_collection, dataset=dataset, @@ -315,24 +431,26 @@ async def load_dataset( organization=organization, ) + assert resolved is not None + ds = await _load_dataset_from_ipfs_cid( - ipfs_cid=final_cid, + ipfs_cid=resolved.cid, kubo_cas=self._kubo_cas, zarr_group=zarr_group, shard_read_mode=shard_read_mode, ) # Build metadata for STAC case - metadata: DatasetMetadata = { + metadata = { "collection": resolved_collection, "dataset": dataset, - "variant": variant or "default", + "variant": resolved.variant, "slug": ( - f"{organization}/{resolved_collection or collection}/{dataset}/{variant or 'default'}" + f"{organization}/{resolved_collection or collection}/{dataset}/{resolved.variant}" if organization - else f"{resolved_collection or collection}/{dataset}/{variant or 'default'}" + else f"{resolved_collection or collection}/{dataset}/{resolved.variant}" ), - "cid": final_cid, + "cid": resolved.cid, "url": None, "timestamp": None, "source": "stac", @@ -418,10 +536,16 @@ def list_datasets(self) -> typing.Dict[str, typing.Dict[str, typing.Any]]: ... } + Notes + ----- + This is a synchronous method: on the IPFS-catalog fallback path it + performs blocking network I/O and will stall a running event loop. + Inside async code prefer ``await client.alist_datasets()``. + Examples -------- >>> async with dClimateClient() as client: - ... datasets = client.list_datasets() + ... datasets = await client.alist_datasets() ... print(datasets["ecmwf_ifs"]["types"]) ['temperature', 'precipitation', 'wind_u', 'wind_v', ...] """ @@ -432,15 +556,50 @@ def list_datasets(self) -> typing.Dict[str, typing.Dict[str, typing.Any]]: if self._stac_server_url: try: return list_available_datasets_from_stac_server(self._stac_server_url) - except (requests.RequestException, ValueError): + except (httpx.HTTPError, ValueError): pass # Fallback: walk the IPFS-hosted catalog. + from .stac_catalog import load_stac_catalog, list_available_datasets + if self._stac_catalog is None: - self._stac_catalog = load_stac_catalog(gateway_url=self._gateway_base_url) + self._stac_catalog = load_stac_catalog( + gateway_url=self._catalog_gateway_base_url, + headers=self._headers, + auth=self._auth, + ) return list_available_datasets(self._stac_catalog) + async def alist_datasets(self) -> typing.Dict[str, typing.Dict[str, typing.Any]]: + """Async variant of :meth:`list_datasets`. + + Runs the blocking STAC/catalog work in a thread so the event loop + (and py-hamt's concurrent chunk fetches) never stall, and shares the + catalog lazy-init lock with :meth:`load_dataset`. + """ + if self._stac_server_url: + try: + return await asyncio.to_thread( + list_available_datasets_from_stac_server, self._stac_server_url + ) + except (httpx.HTTPError, ValueError): + pass + + from .stac_catalog import load_stac_catalog, list_available_datasets + + if self._stac_catalog is None: + async with self._stac_catalog_lock: + if self._stac_catalog is None: + self._stac_catalog = await asyncio.to_thread( + load_stac_catalog, + gateway_url=self._catalog_gateway_base_url, + headers=self._headers, + auth=self._auth, + ) + + return await asyncio.to_thread(list_available_datasets, self._stac_catalog) + # ------------------------------------------------------------------ # Siren REST API methods # ------------------------------------------------------------------ diff --git a/dclimate_client_py/encryption_codec.py b/dclimate_client_py/encryption_codec.py index 9ce5854..ea326cb 100644 --- a/dclimate_client_py/encryption_codec.py +++ b/dclimate_client_py/encryption_codec.py @@ -1,11 +1,13 @@ import asyncio -from typing import Self +from typing import Self, cast from zarr.abc.codec import BytesBytesCodec from zarr.core.buffer import Buffer from zarr.core.common import JSON from Crypto.Cipher import ChaCha20_Poly1305 from Crypto.Random import get_random_bytes +_THREAD_OFFLOAD_THRESHOLD = 128 * 1024 + class EncryptionCodec(BytesBytesCodec): """A Zarr v3 codec implementing XChaCha20-Poly1305 encryption.""" @@ -41,8 +43,8 @@ def from_dict(cls, data: dict[str, JSON]) -> Self: Returns: Self: An instance of EncryptionCodec. """ - configuration = data.get("configuration", {}) - header = configuration.get("header", "dclimate-Zarr") + configuration = cast(dict[str, JSON], data.get("configuration", {})) + header = cast(str, configuration.get("header", "dclimate-Zarr")) return cls(header=header) def to_dict(self) -> dict[str, JSON]: @@ -71,7 +73,10 @@ def decrypt(): cipher.update(self._encoded_header) return cipher.decrypt_and_verify(ciphertext, tag) - plaintext = await asyncio.to_thread(decrypt) + if len(buf) < _THREAD_OFFLOAD_THRESHOLD: + plaintext = decrypt() + else: + plaintext = await asyncio.to_thread(decrypt) return chunk_spec.prototype.buffer.from_bytes(plaintext) async def _encode_single(self, chunk_bytes: Buffer, chunk_spec) -> Buffer: @@ -93,7 +98,10 @@ def encrypt(): ciphertext, tag = cipher.encrypt_and_digest(raw) return nonce + tag + ciphertext - encoded = await asyncio.to_thread(encrypt) + if len(raw) < _THREAD_OFFLOAD_THRESHOLD: + encoded = encrypt() + else: + encoded = await asyncio.to_thread(encrypt) return chunk_spec.prototype.buffer.from_bytes(encoded) def compute_encoded_size(self, input_byte_length: int, chunk_spec) -> int: diff --git a/dclimate_client_py/geotemporal_data.py b/dclimate_client_py/geotemporal_data.py index 0de3e30..392fee4 100644 --- a/dclimate_client_py/geotemporal_data.py +++ b/dclimate_client_py/geotemporal_data.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datetime import functools import math @@ -6,13 +8,15 @@ import typing from collections.abc import Mapping -import geopandas as gpd import pandas as pd import numpy as np from shapely.ops import unary_union import xarray as xr from xarray.core.variable import MissingDimensionsError +if typing.TYPE_CHECKING: + import geopandas as gpd + from .dclimate_zarr_errors import ( InvalidForecastRequestError, ) @@ -36,7 +40,12 @@ class GeotemporalData: data variable. """ - def __init__(self, data: xr.Dataset, dataset_name: str, data_var: str = None): + def __init__( + self, + data: xr.Dataset, + dataset_name: str, + data_var: typing.Optional[str] = None, + ): self.data = data self.dataset_name = dataset_name self._data_var = data_var @@ -105,7 +114,7 @@ def check_dataset_size(self, point_limit: int = DEFAULT_POINT_LIMIT): SelectionTooLargeError When dataset size limit is violated """ - num_points = functools.reduce(operator.mul, self.data.sizes.values()) + num_points = functools.reduce(operator.mul, self.data.sizes.values(), 1) if num_points > point_limit: raise errors.SelectionTooLargeError( f"Selection of {num_points} data points is more than limit of {point_limit}" @@ -122,7 +131,9 @@ def check_has_data(self): if self.data_var.isnull().all(): raise errors.NoDataFoundError("Selection is empty or all NA") - def forecast(self, forecast_reference_time: datetime.datetime) -> "GeotemporalData": + def forecast( + self, forecast_reference_time: typing.Union[str, datetime.datetime] + ) -> "GeotemporalData": """ Filter a 4D forecast dataset to a 3D dataset ready for analysis @@ -203,7 +214,7 @@ def point( data = self.data.sel(selection, method="nearest") else: try: - data = self.data.sel(selection, method="nearest", tolerance=10e-5) + data = self.data.sel(selection, method="nearest", tolerance=1e-5) except KeyError: raise errors.NoDataFoundError( "User requested not to snap_to_grid, but exact coord not in dataset" @@ -217,16 +228,32 @@ def points( epsg_crs: int, snap_to_grid: bool = True, ) -> "GeotemporalData": - mask = list(gpd.geoseries.GeoSeries(points_mask).set_crs(epsg_crs).to_crs(4326)) - lats, lons = [point.y for point in mask], [point.x for point in mask] - lats, lons = xr.DataArray(lats, dims="point"), xr.DataArray(lons, dims="point") + import geopandas as gpd + + series = gpd.GeoSeries(points_mask).set_crs(epsg_crs).to_crs(4326) + if series.isna().any(): + # GeoSeries.y maps missing geometries to NaN, which .sel would + # silently snap to an arbitrary grid cell. + raise errors.InvalidSelectionError( + "points_mask contains missing geometries" + ) + if series.is_empty.any(): + raise errors.InvalidSelectionError("points_mask contains empty geometries") + lat_values = series.y.to_numpy() + lon_values = series.x.to_numpy() + if not np.isfinite(lat_values).all() or not np.isfinite(lon_values).all(): + raise errors.InvalidSelectionError( + "points_mask contains non-finite coordinates" + ) + lats = xr.DataArray(lat_values, dims="point") + lons = xr.DataArray(lon_values, dims="point") if snap_to_grid: data = self.data.sel(latitude=lats, longitude=lons, method="nearest") else: try: data = self.data.sel( - latitude=lats, longitude=lons, method="nearest", tolerance=10e-5 + latitude=lats, longitude=lons, method="nearest", tolerance=1e-5 ) except KeyError: raise errors.NoDataFoundError( @@ -244,7 +271,7 @@ def circle( lat: float, lon: float, radius: float, - ) -> xr.Dataset: + ) -> "GeotemporalData": """Reduces dataset to points within radius of given center coordinates Parameters @@ -259,11 +286,32 @@ def circle( Returns ------- - GeotempoeralData + GeotemporalData New dataset """ - distances = _haversine(lat, lon, self.data["latitude"], self.data["longitude"]) - data = self.data.where(distances < radius, drop=True) + latitudes = self.data["latitude"] + longitudes = self.data["longitude"] + data = self.data + + # Pre-crop only for well-behaved grids; exotic layouts (non-index, + # non-monotonic, NaN coords, 0-360 longitudes) skip the crop and get + # the original full-grid haversine mask below. + bounds = None + if _sliceable_coordinate(self.data, "latitude") and _sliceable_coordinate( + self.data, "longitude" + ): + bounds = _circle_bounding_box(lat, lon, radius, latitudes, longitudes) + if bounds is not None: + min_lat, min_lon, max_lat, max_lon = bounds + data = data.sel( + { + "latitude": _coordinate_slice(latitudes, min_lat, max_lat), + "longitude": _coordinate_slice(longitudes, min_lon, max_lon), + } + ) + + distances = _haversine(lat, lon, data["latitude"], data["longitude"]) + data = data.where(distances < radius, drop=True) return self._new(data) def rectangle( @@ -296,7 +344,10 @@ def rectangle( Returns ------- GeotemporalData - New dataset + New dataset. For regular (1-D monotonic) grids this is a + zero-copy view of the parent dataset, and integer data + variables keep their dtype (the old mask-based selection + upcast them to float64). """ try: latitudes = self.data[latitude_key] @@ -306,13 +357,34 @@ def rectangle( "Latitude/longitude coordinates were not found in the dataset." ) from exc - data = self.data.where( - (latitudes >= min_lat) - & (latitudes <= max_lat) - & (longitudes >= min_lon) - & (longitudes <= max_lon), - drop=True, + if not ( + _sliceable_coordinate(self.data, latitude_key) + and _sliceable_coordinate(self.data, longitude_key) + ): + # Legacy mask path for exotic layouts: non-index or + # non-monotonic coordinates, NaNs, curvilinear grids. + data = self.data.where( + (latitudes >= min_lat) + & (latitudes <= max_lat) + & (longitudes >= min_lon) + & (longitudes <= max_lon), + drop=True, + ) + return self._new(data) + + data = self.data.sel( + { + latitude_key: _coordinate_slice(latitudes, min_lat, max_lat), + longitude_key: _coordinate_slice(longitudes, min_lon, max_lon), + } ) + if data[latitude_key].size == 0 or data[longitude_key].size == 0: + data = data.isel( + { + latitudes.dims[0]: slice(0, 0), + longitudes.dims[0]: slice(0, 0), + } + ) return self._new(data) def polygons( @@ -338,25 +410,45 @@ def polygons( GeotemporalData New dataset """ + try: + import rioxarray + except ImportError as exc: + raise ImportError( + "GeotemporalData.polygons() requires rioxarray to be installed" + ) from exc + + import geopandas as gpd + + # Normalize the mask before comparing areas or selecting a fallback + # point. Dataset coordinates are WGS84; using projected coordinates + # here would otherwise compare square metres with square degrees and + # pass metre-valued centroids to latitude/longitude selection. + mask = gpd.GeoSeries(polygons_mask, crs=epsg_crs).to_crs(4326) + normalized_polygons = mask.array + # If the polygon(s) are collectively smaller than the size of one grid cell, # clipping will return no data In this case return data from the grid cell nearest # to the center of the polygon - if self.data.attrs["spatial resolution"] ** 2 > polygons_mask.union_all().area: - return self.reduce_polygon_to_point(polygons_mask) + if ( + self.data.attrs["spatial resolution"] ** 2 + > normalized_polygons.union_all().area + ): + return self.reduce_polygon_to_point(normalized_polygons) # return clipped data as normal if the polygons are large enough - self.data.rio.set_spatial_dims( - x_dim="longitude", y_dim="latitude", inplace=True + spatial_data = self.data.rio.set_spatial_dims( + x_dim="longitude", y_dim="latitude", inplace=False ) - self.data.rio.write_crs("epsg:4326", inplace=True) - mask = gpd.geoseries.GeoSeries(polygons_mask).set_crs(epsg_crs).to_crs(4326) + spatial_data = spatial_data.rio.write_crs("epsg:4326") min_lon, min_lat, max_lon, max_lat = mask.total_bounds - box_ds = self.rectangle(min_lat, min_lon, max_lat, max_lon).data + box_ds = ( + self._new(spatial_data).rectangle(min_lat, min_lon, max_lat, max_lon).data + ) self._new(box_ds).check_dataset_size(point_limit=point_limit) try: shaped_ds = box_ds.rio.clip(mask, 4326, drop=True) - except errors.NoDataInBounds: - return self.data.reduce_polygon_to_point(polygons_mask) + except rioxarray.exceptions.NoDataInBounds: + return self.reduce_polygon_to_point(normalized_polygons) data_var = list(shaped_ds.data_vars)[0] if "grid_mapping" in shaped_ds[data_var].attrs: @@ -527,7 +619,7 @@ def temporal_aggregation( "day": f"{time_unit}D", "week": f"{time_unit}W", "month": f"{time_unit}ME", - "quarter": f"{time_unit}Q", + "quarter": f"{time_unit}QE", "year": f"{time_unit}YE", } # Resample by the specified time period and aggregate by the specified method @@ -563,9 +655,10 @@ def rolling_aggregation( _check_input_parameters(agg_method=agg_method) # Aggregate by the specified method over the specified rolling window length rolled = self.data.rolling(time=window_size) - rolled_agg = getattr(rolled, agg_method)(keep_attrs=True).dropna("time") - # remove NAs at beginning/end of array where window size is not large enough to - # compute a value + # Trim incomplete leading windows while preserving spatial NAs. + rolled_agg = getattr(rolled, agg_method)(keep_attrs=True).isel( + time=slice(window_size - 1, None) + ) return self._new(rolled_agg) @@ -582,10 +675,12 @@ def to_netcdf(self, *args, **kwargs): If no arguments are passed, a bytes object is returned. """ + ds = self.data.copy() + try: - if self.data.update_in_progress and not self.data.update_is_append_only: - update_date_range = self.data.attrs["update_date_range"] - self.data.attrs["updating date range"] = ( + if ds.update_in_progress and not ds.update_is_append_only: + update_date_range = ds.attrs["update_date_range"] + ds.attrs["updating date range"] = ( f"{update_date_range[0]}-{update_date_range[1]}" ) except AttributeError: @@ -599,10 +694,15 @@ def to_netcdf(self, *args, **kwargs): "finalization date", "update_date_range", ]: - if bad_key in self.data.attrs: - del self.data.attrs[bad_key] + if bad_key in ds.attrs: + del ds.attrs[bad_key] - return self.data.to_netcdf(*args, **kwargs) + serialized = ds.to_netcdf(*args, **kwargs) + # xarray 2025.09+ returns a memoryview for in-memory serialization, + # while this public wrapper has always promised bytes. + if isinstance(serialized, memoryview): + return serialized.tobytes() + return serialized def as_dict(self) -> dict: """Prepares dict containing metadata and values from dataset @@ -614,7 +714,8 @@ def as_dict(self) -> dict: """ vals = self.data_var.values ret_dict = {} - dimensions = [] + dimensions: list[typing.Any] = [] + missing_value: typing.Any = None ret_dict["units"] = self.data_var.attrs.get("units", "unknown") if "time" in self.data: ret_dict["times"] = ( @@ -629,13 +730,17 @@ def as_dict(self) -> dict: ) ret_dict["point_coords_order"] = ["latitude", "longitude"] dimensions.insert(0, "point") - ret_dict["data"] = np.where(~np.isfinite(vals), None, vals).T.tolist() + ret_dict["data"] = np.where( + ~np.isfinite(vals), missing_value, vals + ).T.tolist() else: for dim in self.data_var.dims: if dim != "time": ret_dict[f"{dim}s"] = self.data[dim].values.flatten().tolist() dimensions.append(dim) - ret_dict["data"] = np.where(~np.isfinite(vals), None, vals).tolist() + ret_dict["data"] = np.where( + ~np.isfinite(vals), missing_value, vals + ).tolist() ret_dict["dimensions_order"] = dimensions try: if self.data.update_in_progress and not self.data.update_is_append_only: @@ -646,20 +751,68 @@ def as_dict(self) -> dict: def query( self, - forecast_reference_time: datetime.datetime = None, - point_kwargs: dict = None, - circle_kwargs: dict = None, - rectangle_kwargs: dict = None, - polygon_kwargs: dict = None, - multiple_points_kwargs: dict = None, - bounds: BoundsSelection = None, - bounds_options: dict = None, - spatial_agg_kwargs: dict = None, - temporal_agg_kwargs: dict = None, - rolling_agg_kwargs: dict = None, + forecast_reference_time: typing.Union[str, datetime.datetime, None] = None, + point_kwargs: typing.Optional[dict] = None, + circle_kwargs: typing.Optional[dict] = None, + rectangle_kwargs: typing.Optional[dict] = None, + polygon_kwargs: typing.Optional[dict] = None, + multiple_points_kwargs: typing.Optional[dict] = None, + bounds: typing.Optional[BoundsSelection] = None, + bounds_options: typing.Optional[dict] = None, + spatial_agg_kwargs: typing.Optional[dict] = None, + temporal_agg_kwargs: typing.Optional[dict] = None, + rolling_agg_kwargs: typing.Optional[dict] = None, time_range: typing.Optional[typing.List[datetime.datetime]] = None, point_limit: int = DEFAULT_POINT_LIMIT, ) -> "GeotemporalData": + if point_kwargs is not None: + missing = [ + key + for key in ("latitude", "longitude") + if key not in point_kwargs or point_kwargs[key] is None + ] + if missing: + raise errors.InvalidSelectionError( + f"point_kwargs missing required key(s): {', '.join(missing)}" + ) + + if circle_kwargs is not None: + missing = [] + if "lat" in circle_kwargs: + if circle_kwargs["lat"] is None: + missing.append("lat") + elif circle_kwargs.get("center_lat") is None: + missing.append("center_lat" if "center_lat" in circle_kwargs else "lat") + if "lon" in circle_kwargs: + if circle_kwargs["lon"] is None: + missing.append("lon") + elif circle_kwargs.get("center_lon") is None: + missing.append("center_lon" if "center_lon" in circle_kwargs else "lon") + if circle_kwargs.get("radius") is None: + missing.append("radius") + if missing: + raise errors.InvalidSelectionError( + f"circle_kwargs missing required key(s): {', '.join(missing)}" + ) + + if rectangle_kwargs is not None: + missing = [ + key + for key in ("min_lat", "min_lon", "max_lat", "max_lon") + if key not in rectangle_kwargs or rectangle_kwargs[key] is None + ] + if missing: + raise errors.InvalidSelectionError( + f"rectangle_kwargs missing required key(s): {', '.join(missing)}" + ) + + if polygon_kwargs == {}: + raise errors.InvalidSelectionError("polygon_kwargs must not be empty") + if multiple_points_kwargs == {}: + raise errors.InvalidSelectionError( + "multiple_points_kwargs must not be empty" + ) + # Filter data down temporally, then spatially, and check that the size of # resulting dataset fits within the limit. While a user can get the entire DS by # providing no filters, this will almost certainly cause the size checks to fail @@ -669,9 +822,17 @@ def query( if point_kwargs: data = data.point(**point_kwargs) elif circle_kwargs: - lat = circle_kwargs.get("lat", circle_kwargs.get("center_lat")) - lon = circle_kwargs.get("lon", circle_kwargs.get("center_lon")) - radius = circle_kwargs.get("radius") + lat = ( + circle_kwargs["lat"] + if "lat" in circle_kwargs + else circle_kwargs["center_lat"] + ) + lon = ( + circle_kwargs["lon"] + if "lon" in circle_kwargs + else circle_kwargs["center_lon"] + ) + radius = circle_kwargs["radius"] data = data.circle(lat=lat, lon=lon, radius=radius) elif rectangle_kwargs: data = data.rectangle(**rectangle_kwargs) @@ -876,12 +1037,97 @@ def _normalize_time_range_selection( return start, end +def _sliceable_coordinate(data: xr.Dataset, key: str) -> bool: + """True when ``key`` is a 1-D, NaN-free, monotonic dimension index. + + Only such coordinates can be selected with ``.sel(slice)``; anything + else (non-index coords, curvilinear grids, unsorted or NaN-holding + coords) must go through the mask-based fallback paths. + """ + coord = data[key] + if coord.ndim != 1 or coord.dims[0] != key or key not in data.xindexes: + return False + index = data.indexes[key] + if index.hasnans: + return False + return bool(index.is_monotonic_increasing or index.is_monotonic_decreasing) + + +def _coordinate_slice( + coordinate: xr.DataArray, minimum: float, maximum: float +) -> slice: + if coordinate.size == 0: + return slice(minimum, maximum) + + dimension = coordinate.dims[0] + first = coordinate.isel({dimension: 0}).values.item() + last = coordinate.isel({dimension: -1}).values.item() + if first <= last: + return slice(minimum, maximum) + return slice(maximum, minimum) + + +def _circle_bounding_box( + lat: float, + lon: float, + radius: float, + latitudes: xr.DataArray, + longitudes: xr.DataArray, +) -> typing.Optional[tuple[float, float, float, float]]: + values = (lat, lon, radius) + if not all(isinstance(value, numbers.Real) for value in values): + return None + if not all(math.isfinite(value) for value in values) or radius < 0: + return None + if not (-90 <= lat <= 90 and -180 <= lon <= 180): + return None + if latitudes.size == 0 or longitudes.size == 0: + return None + + latitude_dimension = latitudes.dims[0] + longitude_dimension = longitudes.dims[0] + coordinate_endpoints: tuple[typing.Any, ...] = ( + latitudes.isel({latitude_dimension: 0}).values.item(), + latitudes.isel({latitude_dimension: -1}).values.item(), + longitudes.isel({longitude_dimension: 0}).values.item(), + longitudes.isel({longitude_dimension: -1}).values.item(), + ) + if not all( + isinstance(value, numbers.Real) and math.isfinite(value) + for value in coordinate_endpoints + ): + return None + if not all(-90 <= value <= 90 for value in coordinate_endpoints[:2]): + return None + if not all(-180 <= value <= 180 for value in coordinate_endpoints[2:]): + return None + + angular_radius = radius / 6371 + angular_radius_degrees = math.degrees(angular_radius) + pad = 1e-9 # ~0.1 mm; dwarfs haversine float error (~1e-11 degrees) + min_lat = max(-90, lat - angular_radius_degrees - pad) + max_lat = min(90, lat + angular_radius_degrees + pad) + + latitude_radians = math.radians(lat) + if angular_radius >= math.pi / 2 - abs(latitude_radians): + return min_lat, -180, max_lat, 180 + + longitude_radius = math.degrees( + math.asin(math.sin(angular_radius) / math.cos(latitude_radians)) + ) + min_lon = lon - longitude_radius - pad + max_lon = lon + longitude_radius + pad + if min_lon < -180 or max_lon > 180: + return None + return min_lat, min_lon, max_lat, max_lon + + def _haversine( - lat1: typing.Union[np.ndarray, float], - lon1: typing.Union[np.ndarray, float], - lat2: typing.Union[np.ndarray, float], - lon2: typing.Union[np.ndarray, float], -) -> typing.Union[np.ndarray, float]: + lat1: typing.Union[np.ndarray, xr.DataArray, float], + lon1: typing.Union[np.ndarray, xr.DataArray, float], + lat2: typing.Union[np.ndarray, xr.DataArray, float], + lon2: typing.Union[np.ndarray, xr.DataArray, float], +) -> typing.Union[np.ndarray, xr.DataArray, float]: """Calculates arclength distance in km between coordinate pairs, assuming the earth is a perfect sphere diff --git a/dclimate_client_py/ipfs_retrieval.py b/dclimate_client_py/ipfs_retrieval.py index 5c4298c..45fe5a8 100644 --- a/dclimate_client_py/ipfs_retrieval.py +++ b/dclimate_client_py/ipfs_retrieval.py @@ -5,10 +5,13 @@ """ import logging +import socket import time import warnings from typing import Any +import aiohttp +import httpx import xarray as xr from multiformats import CID from opentelemetry import metrics, trace @@ -136,22 +139,31 @@ def _record_span_error(active_span: Span, exc: Exception) -> None: def _is_connection_error(exc: Exception) -> bool: - """Classify gateway and network failures from exception text.""" - text = str(exc).lower() - return any( - token in text - for token in ( - "connection refused", - "connection reset", - "max retries exceeded", - "name or service not known", - "network is unreachable", - "nodename nor servname", - "temporary failure in name resolution", - "timeout", - "timed out", - ) + """Classify gateway and network failures by type, including chained causes. + + Deliberately narrower than ``OSError``: filesystem errors such as + ``FileNotFoundError``/``PermissionError`` must not masquerade as gateway + failures, or the caller would skip the HAMT fallback for them. + """ + connection_error_types = ( + ConnectionError, + TimeoutError, + socket.gaierror, + socket.herror, + httpx.TransportError, + aiohttp.ClientConnectionError, + aiohttp.ServerTimeoutError, ) + # Note: exhausted HTTP-status retries (httpx.HTTPStatusError) are + # deliberately NOT connection errors — they take the HAMT fallback. + current: BaseException | None = exc + seen: set[int] = set() + while current is not None and id(current) not in seen: + seen.add(id(current)) + if isinstance(current, connection_error_types): + return True + current = current.__cause__ or current.__context__ + return False def _normalize_zarr_group(zarr_group: str | None) -> str | None: @@ -207,19 +219,28 @@ def _open_zarr_from_store( """Open a Zarr store, choosing a default group for py-hamt v2 pyramids.""" normalized_group = _normalize_zarr_group(zarr_group) if normalized_group is not None: - return xr.open_zarr(store=store, group=normalized_group), normalized_group + return ( + xr.open_zarr(store=store, group=normalized_group, decode_timedelta=True), + normalized_group, + ) if _store_requires_explicit_zarr_group(store): for candidate_group in _zarr_group_candidates(store): - return xr.open_zarr(store=store, group=candidate_group), candidate_group + return ( + xr.open_zarr(store=store, group=candidate_group, decode_timedelta=True), + candidate_group, + ) try: - return xr.open_zarr(store=store), None + return xr.open_zarr(store=store, decode_timedelta=True), None except ValueError as exc: if not _is_explicit_zarr_group_error(exc): raise for candidate_group in _zarr_group_candidates(store): - return xr.open_zarr(store=store, group=candidate_group), candidate_group + return ( + xr.open_zarr(store=store, group=candidate_group, decode_timedelta=True), + candidate_group, + ) raise diff --git a/dclimate_client_py/py.typed b/dclimate_client_py/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/dclimate_client_py/s3_retrieval.py b/dclimate_client_py/s3_retrieval.py index 11b7ecc..d208d02 100644 --- a/dclimate_client_py/s3_retrieval.py +++ b/dclimate_client_py/s3_retrieval.py @@ -1,14 +1,32 @@ +from __future__ import annotations + from aiobotocore import session from functools import lru_cache import datetime import os -from s3fs import S3FileSystem, S3Map import typing import json import xarray as xr from dclimate_client_py.dclimate_zarr_errors import DatasetNotFoundError +if typing.TYPE_CHECKING: + from s3fs import S3FileSystem + + +def __getattr__(name: str): + if name in {"S3FileSystem", "S3Map"}: + import s3fs + + value = getattr(s3fs, name) + globals()[name] = value + return value + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__() -> list[str]: + return sorted(set(globals()) | {"S3FileSystem", "S3Map"}) + @lru_cache(maxsize=1) def get_aio_session(): @@ -21,15 +39,19 @@ def get_s3_fs() -> S3FileSystem: Returns: S3FileSystem: """ + s3_file_system = globals().get("S3FileSystem") + if s3_file_system is None: + s3_file_system = __getattr__("S3FileSystem") + if "ZARR_AWS_PROFILE_NAME" in os.environ: - return S3FileSystem(session=get_aio_session()) + return s3_file_system(session=get_aio_session()) elif "AWS_ACCESS_KEY_ID" in os.environ and "AWS_SECRET_ACCESS_KEY" in os.environ: - return S3FileSystem( + return s3_file_system( key=os.environ["AWS_ACCESS_KEY_ID"], secret=os.environ["AWS_SECRET_ACCESS_KEY"], ) else: - return S3FileSystem(anon=False) + return s3_file_system(anon=False) def get_dataset_from_s3(dataset_name: str, bucket_name: str) -> xr.Dataset: @@ -43,20 +65,26 @@ def get_dataset_from_s3(dataset_name: str, bucket_name: str) -> xr.Dataset: xr.Dataset: dataset corresponding to key """ try: - s3_map = S3Map( + s3_map_type = globals().get("S3Map") + if s3_map_type is None: + s3_map_type = __getattr__("S3Map") + s3_map = s3_map_type( f"s3://{bucket_name}/datasets/{dataset_name}.zarr", s3=get_s3_fs(), ) - ds = xr.open_zarr(s3_map, chunks=None) + ds = xr.open_zarr(s3_map, chunks=None, decode_timedelta=True) except FileNotFoundError: raise DatasetNotFoundError(f"Invalid dataset name {dataset_name}") - if ds.update_in_progress: - if hasattr(ds, "initial_parse") and ds.initial_parse: + attrs = getattr(ds, "attrs", {}) + if attrs.get("update_in_progress", getattr(ds, "update_in_progress", False)): + if attrs.get("initial_parse", getattr(ds, "initial_parse", False)): raise DatasetNotFoundError( f"Dataset {dataset_name} is undergoing initial parse, retry request later" ) - if ds.update_is_append_only: + if attrs.get( + "update_is_append_only", getattr(ds, "update_is_append_only", False) + ): start, end = ds.attrs["date range"][0], ds.attrs["update_previous_end_date"] else: start, end = ds.attrs["date range"] diff --git a/dclimate_client_py/siren/siren_client.py b/dclimate_client_py/siren/siren_client.py index d18c450..af603a8 100644 --- a/dclimate_client_py/siren/siren_client.py +++ b/dclimate_client_py/siren/siren_client.py @@ -7,6 +7,7 @@ import datetime import inspect import os +from collections.abc import Mapping from typing import Any, Optional, TypeGuard from urllib.parse import quote @@ -200,7 +201,9 @@ def _build_x402_client(x402_client_class: Any, facilitator_url: str | None) -> A return x402_client_class() try: - params = inspect.signature(x402_client_class).parameters + params: Mapping[str, inspect.Parameter] = inspect.signature( + x402_client_class + ).parameters except (TypeError, ValueError): params = {} @@ -401,14 +404,18 @@ async def _get_x402_fetch(self) -> Any: client = x402Client() for network in _network_preferences(self._auth.network): client.register_policy(prefer_network(network)) - register_exact_evm_client(client, self._auth.signer) + register_exact_evm_client( + client, + self._auth.signer, # type: ignore[arg-type] + ) - self._x402_http_client = x402HttpxClient(client, timeout=self._timeout) + http_client = x402HttpxClient(client, timeout=self._timeout) + self._x402_http_client = http_client async def wrapped_fetch( url: str, method: str = "GET", **kwargs: Any ) -> httpx.Response: - return await self._x402_http_client.request(method, url, **kwargs) + return await http_client.request(method, url, **kwargs) self._x402_fetch = wrapped_fetch return self._x402_fetch diff --git a/dclimate_client_py/stac_catalog.py b/dclimate_client_py/stac_catalog.py index a68911b..1146387 100644 --- a/dclimate_client_py/stac_catalog.py +++ b/dclimate_client_py/stac_catalog.py @@ -5,31 +5,72 @@ for discovering and accessing dClimate datasets stored on IPFS. """ -from typing import Optional, Dict, List, Set, Tuple, Any +from os import PathLike, fspath +from typing import Optional, Dict, List, Set, Tuple, Any, cast import logging -import requests +import weakref +from threading import Lock +from urllib.parse import urlsplit + +import httpx import pystac from .datasets import SpatialExtent, TemporalExtent +from .stac_server import ( + ResolvedDataset, + _dataset_and_variant_from_item_id, + _dataset_and_variant_from_known_datasets, +) logger = logging.getLogger(__name__) - - -def get_root_catalog_cid() -> str: +STAC_CATALOG_URL = "https://ipfs-gateway.dclimate.net/stac" +_HTTP_CLIENT: httpx.Client | None = None +_HTTP_CLIENT_LOCK = Lock() + + +def _client() -> httpx.Client: + """Return the process-wide pooled client used for synchronous STAC reads.""" + global _HTTP_CLIENT + if _HTTP_CLIENT is None: + with _HTTP_CLIENT_LOCK: + if _HTTP_CLIENT is None: + _HTTP_CLIENT = httpx.Client(timeout=30, follow_redirects=True) + return _HTTP_CLIENT + + +def get_root_catalog_cid( + catalog_url: str = STAC_CATALOG_URL, + *, + headers: Optional[Dict[str, str]] = None, + auth: Optional[Tuple[str, str]] = None, +) -> str: """ Get the root STAC catalog CID. Fetches the latest catalog CID from the dClimate IPFS gateway API. + Args: + catalog_url: URL of the dClimate STAC root-CID pointer endpoint. + headers: Optional default headers for the request. The pointer endpoint + lives on the IPFS gateway host, so authenticated gateways need the + same credentials here as for ``/ipfs`` reads. + auth: Optional ``(username, password)`` basic-auth pair for the request. + Returns: str: The IPFS CID of the root STAC catalog Raises: - requests.HTTPError: If the API request fails + httpx.HTTPError: If the API request fails KeyError: If the response doesn't contain the expected 'cid' field """ - url = "https://ipfs-gateway.dclimate.net/stac" - response = requests.get(url) + # The pooled client is process-wide and gateway-agnostic, so credentials are + # applied per-request rather than baked into the shared client. + response = _client().get( + catalog_url, + timeout=30, + headers=headers, + auth=auth if auth is not None else httpx.USE_CLIENT_DEFAULT, + ) response.raise_for_status() data = response.json() return data["cid"] @@ -76,7 +117,9 @@ def _resolve_child_by_dclimate_id( """Resolve a catalog child by its dclimate:id extra field.""" for link in parent.get_child_links(): if link.extra_fields.get("dclimate:id") == child_id: - return link.resolve_stac_object(root=parent).target, link + return cast( + pystac.Catalog, link.resolve_stac_object(root=parent).target + ), link return None, None @@ -95,13 +138,16 @@ def _resolve_child_by_collection_slug( if collection_slug not in collections: continue - org_catalog = link.resolve_stac_object(root=parent).target + org_catalog = cast(pystac.Catalog, link.resolve_stac_object(root=parent).target) if org_catalog is None: continue for col_link in org_catalog.get_child_links(): if col_link.extra_fields.get("dclimate:id") == collection_slug: - return col_link.resolve_stac_object(root=org_catalog).target, col_link + return cast( + pystac.Catalog, + col_link.resolve_stac_object(root=org_catalog).target, + ), col_link return None, None @@ -114,21 +160,43 @@ class IPFSStacIO(pystac.StacIO): that are stored on IPFS and referenced using ipfs:// protocol URIs. """ - def __init__(self, gateway_url: str): + def __init__( + self, + gateway_url: str, + *, + headers: Optional[Dict[str, str]] = None, + auth: Optional[Tuple[str, str]] = None, + ): """ Initialize the IPFS STAC I/O handler. Args: gateway_url: Base URL of the IPFS HTTP gateway (e.g., 'https://ipfs-gateway.dclimate.net') + headers: Optional default headers applied to every gateway request. + Required for authenticated gateways so catalog fallback does not + fail with 401. + auth: Optional ``(username, password)`` basic-auth pair applied to + every gateway request. """ self.gateway_url = gateway_url.rstrip("/") + # Per-instance client so each catalog owns its pool lifecycle + # (closed via weakref.finalize when the catalog is collected). + # httpx.Client is thread-safe across asyncio.to_thread workers. + # Credentials are baked into the client so every gateway read carries + # them, mirroring the KuboCAS data path. + self.client = httpx.Client( + timeout=30, + follow_redirects=True, + headers=headers, + auth=auth, + ) - def read_text(self, source: str, *args, **kwargs) -> str: + def read_text(self, source: str | PathLike[str], *args, **kwargs) -> str: """ Read text content from a source URI. If the source starts with 'ipfs://', resolves it via the HTTP gateway. - Otherwise, delegates to the default StacIO implementation. + HTTP(S) sources are fetched directly by the owned HTTP client. Args: source: URI to read from (e.g., 'ipfs://bafkrei...' or 'https://...') @@ -137,19 +205,28 @@ def read_text(self, source: str, *args, **kwargs) -> str: str: The text content Raises: - requests.HTTPError: If the HTTP request fails + httpx.HTTPError: If the HTTP request fails """ - if source.startswith("ipfs://"): - cid = source.replace("ipfs://", "") + source_text = fspath(source) + if source_text.startswith("ipfs://"): + cid = source_text.replace("ipfs://", "") url = f"{self.gateway_url}/ipfs/{cid}" - response = requests.get(url) + response = self.client.get(url, timeout=30) response.raise_for_status() return response.text - # Fall back to default behavior for HTTP/HTTPS URLs - return super().read_text(source, *args, **kwargs) + scheme = urlsplit(source_text).scheme.lower() + if scheme in {"http", "https"}: + response = self.client.get(source_text) + response.raise_for_status() + return response.text + + unsupported_scheme = scheme or "" + raise ValueError( + f"Unsupported STAC source scheme '{unsupported_scheme}': {source_text}" + ) - def write_text(self, dest: str, txt: str, *args, **kwargs) -> None: + def write_text(self, dest: str | PathLike[str], txt: str, *args, **kwargs) -> None: """ Write text content is not supported for IPFS. @@ -158,9 +235,18 @@ def write_text(self, dest: str, txt: str, *args, **kwargs) -> None: """ raise NotImplementedError("Writing to IPFS is not supported via StacIO") + def close(self) -> None: + """Close the pooled HTTP client owned by this I/O handler.""" + self.client.close() + def load_stac_catalog( - gateway_url: str, root_cid: Optional[str] = None + gateway_url: str, + root_cid: Optional[str] = None, + catalog_url: str = STAC_CATALOG_URL, + *, + headers: Optional[Dict[str, str]] = None, + auth: Optional[Tuple[str, str]] = None, ) -> pystac.Catalog: """ Load the dClimate STAC catalog from IPFS. @@ -168,24 +254,38 @@ def load_stac_catalog( Args: gateway_url: Base URL of the IPFS HTTP gateway root_cid: Optional IPFS CID of the root catalog. If None, fetches via get_root_catalog_cid() + catalog_url: Root-CID pointer endpoint used when ``root_cid`` is omitted. + headers: Optional default headers for every gateway/pointer request. + Pass the same credentials used for the KuboCAS data path so catalog + fallback works against authenticated gateways. + auth: Optional ``(username, password)`` basic-auth pair for every + gateway/pointer request. Returns: pystac.Catalog: The loaded STAC catalog with all links and references Raises: - requests.HTTPError: If fetching from IPFS fails + httpx.HTTPError: If fetching from IPFS fails pystac.STACError: If the catalog structure is invalid """ if root_cid is None: - root_cid = get_root_catalog_cid() + root_cid = get_root_catalog_cid(catalog_url, headers=headers, auth=auth) - # Set up custom IPFS I/O handler - stac_io = IPFSStacIO(gateway_url) - pystac.StacIO.set_default(lambda: stac_io) + # Bind the I/O handler to this catalog. Avoid pystac's process-global + # default, because concurrent clients may use different gateways. + stac_io = IPFSStacIO(gateway_url, headers=headers, auth=auth) # Load the root catalog catalog_uri = f"ipfs://{root_cid}" - catalog = pystac.Catalog.from_file(catalog_uri) + try: + catalog = pystac.Catalog.from_file(catalog_uri, stac_io=stac_io) + except BaseException: + stac_io.close() + raise + + # Keep the pool alive for lazy link resolution, then release it when the + # returned catalog is no longer in use. + weakref.finalize(catalog, stac_io.close) return catalog @@ -196,10 +296,12 @@ def resolve_dataset_cid_from_stac( dataset: str, variant: Optional[str] = None, organization: Optional[str] = None, -) -> str: +) -> ResolvedDataset: """ Resolve a dataset to its IPFS CID by querying the STAC catalog. + Changed in 0.6: returns ResolvedDataset. + This function navigates the STAC catalog structure to find the specific dataset variant and extracts the Zarr data CID from the STAC Item's assets. @@ -216,7 +318,7 @@ def resolve_dataset_cid_from_stac( catalog metadata. Returns: - str: The IPFS CID of the Zarr dataset (without 'ipfs://' prefix) + ResolvedDataset: The IPFS CID and selected variant Raises: ValueError: If collection, dataset, or variant is not found in the catalog @@ -240,49 +342,23 @@ def resolve_dataset_cid_from_stac( ) if collection_obj is None and resolved_collection_id != collection: collection_obj, _ = _resolve_child_by_dclimate_id(org_catalog, collection) + if collection_obj is not None: + resolved_collection_id = collection if collection_obj is None: raise ValueError( f"Collection '{collection}' not found under organization '{organization}'" ) else: + for candidate_link in catalog.get_child_links(): + if resolved_collection_id in _extract_collections_from_org_link( + candidate_link + ): + org_link = candidate_link + break # First, try legacy layout where collections hang off the root catalog collection_obj, _ = _resolve_child_by_collection_slug( catalog, resolved_collection_id ) - # # Otherwise, infer the organization by scanning org metadata on the root catalog - # if collection_obj is None: - # for candidate_link in catalog.get_child_links(): - # print(candidate_link) - # org_id = candidate_link.extra_fields.get("dclimate:id") - # print(f"Organization ID: {org_id}") - # if not org_id: - # continue - - # declared_collections = _extract_collections_from_org_link(candidate_link) - # dataset_collections = { - # slug.split("/", 1)[0] - # for slug in candidate_link.extra_fields.get("dclimate:datasets", []) - # if isinstance(slug, str) and "/" in slug - # } - # declared_collections.update(dataset_collections) - - # if resolved_collection_id in declared_collections: - # org_link = candidate_link - # org_catalog = candidate_link.resolve_stac_object(root=catalog).target - # break - - # prefixed = f"{org_id}_{collection}" - # if prefixed in declared_collections: - # resolved_collection_id = prefixed - # org_link = candidate_link - # org_catalog = candidate_link.resolve_stac_object(root=catalog).target - # break - - # if collection_obj is None and org_catalog: - # collection_obj, _ = _resolve_child_by_dclimate_id( - # org_catalog, resolved_collection_id - # ) - if collection_obj is None: org_msg = ( f" under organization '{org_link.extra_fields.get('dclimate:id')}'" @@ -294,15 +370,59 @@ def resolve_dataset_cid_from_stac( ) # Find the item matching dataset and variant + items = list(collection_obj.get_items()) + known_datasets = { + value + for value in collection_obj.extra_fields.get("dclimate:types", []) or [] + if isinstance(value, str) and value + } + if org_link is not None: + known_datasets.update( + _extract_datasets_for_collection(org_link, resolved_collection_id) + ) + known_datasets.update( + property_dataset + for item in items + for property_dataset in [(item.properties or {}).get("dclimate:dataset_id")] + if isinstance(property_dataset, str) and property_dataset + ) + # Metadata can be incomplete and mention only a shorter sibling (for + # example ``precip`` but not ``precip-daily``). The requested name is + # nevertheless a valid parsing hint, matching the STAC-server resolver. + known_datasets.add(dataset) + candidates = [] - for item in collection_obj.get_items(): + selected_item = None + selected_variant = None + for item in items: # Item IDs follow pattern: "{collection_id}-{dataset}" or "-{variant}" - item_id = item.id - prefix = f"{collection_obj.id}-" - remainder = item_id[len(prefix) :] if item_id.startswith(prefix) else item_id - parts = remainder.split("-") - item_dataset = parts[0] if parts else remainder - item_variant = parts[1] if len(parts) > 1 else None + properties = item.properties or {} + property_dataset = properties.get("dclimate:dataset_id") + property_variant = properties.get("dclimate:variant") + if property_dataset: + item_dataset = property_dataset + _, parsed_variant = _dataset_and_variant_from_item_id( + item.id, collection_obj.id, dataset=property_dataset + ) + elif property_variant: + item_dataset, parsed_variant = _dataset_and_variant_from_item_id( + item.id, collection_obj.id, variant=property_variant + ) + if item_dataset is None: + # The id does not encode the property variant (e.g. a bare + # id with properties variant "default") — parse with the + # requested-dataset hint instead. + item_dataset, parsed_variant = _dataset_and_variant_from_item_id( + item.id, collection_obj.id, dataset + ) + else: + item_dataset, parsed_variant = _dataset_and_variant_from_known_datasets( + item.id, collection_obj.id, known_datasets + ) + # A bare item (no variant segment/property) is what the listing APIs + # report as the "default" variant — keep resolve symmetric with list + # and with the STAC-server resolver. + item_variant = property_variant or parsed_variant or "default" if item_dataset != dataset: continue @@ -311,10 +431,8 @@ def resolve_dataset_cid_from_stac( if variant is not None and item_variant == variant: selected_item = item + selected_variant = variant break - else: - selected_item = None - if variant is not None: if not selected_item: raise ValueError( @@ -326,22 +444,24 @@ def resolve_dataset_cid_from_stac( f"Dataset '{dataset}' not found in collection '{collection_obj.id}'" ) # If multiple variants exist and none specified, pick a sensible default - preferred_order = ["default", "final", "finalized", "latest", None] - selected_item = candidates[0][1] + preferred_order = ["default", "final", "finalized", "latest"] + selected_variant, selected_item = candidates[0] for preferred in preferred_order: for cand_variant, cand_item in candidates: if cand_variant == preferred: selected_item = cand_item + selected_variant = cand_variant break else: continue break + assert selected_variant is not None if "data" in selected_item.assets: href = selected_item.assets["data"].href if href.startswith("ipfs://"): - return href.replace("ipfs://", "") - return href + href = href.replace("ipfs://", "") + return ResolvedDataset(href, selected_variant) raise ValueError(f"Item '{selected_item.id}' does not have a 'data' asset") @@ -407,7 +527,9 @@ def list_available_datasets(catalog: pystac.Catalog) -> Dict[str, Dict[str, Any] if is_org: org_id = child_id org_title = link.title or org_id - org_catalog = link.resolve_stac_object(root=catalog).target + org_catalog = cast( + pystac.Catalog, link.resolve_stac_object(root=catalog).target + ) # Map collection -> category (historical/forecast/etc.) collection_categories: Dict[str, str] = {} @@ -452,19 +574,69 @@ def list_available_datasets(catalog: pystac.Catalog) -> Dict[str, Dict[str, Any] # Resolve items to extract per-variant extents try: - col_catalog = col_link.resolve_stac_object(root=org_catalog).target + col_catalog = cast( + pystac.Catalog, + col_link.resolve_stac_object(root=org_catalog).target, + ) if col_catalog is not None: - prefix = f"{collection_id}-" - for item in col_catalog.get_items(): - item_id = item.id - remainder = ( - item_id[len(prefix) :] - if item_id.startswith(prefix) - else item_id + items = list(col_catalog.get_items()) + known_datasets = set(types) + known_datasets.update( + property_dataset + for item in items + for property_dataset in [ + (item.properties or {}).get("dclimate:dataset_id") + ] + if isinstance(property_dataset, str) and property_dataset + ) + for item in items: + # Prefer explicit dclimate:* properties (like the + # server lister); fall back to the shared + # hyphen-aware id parsing, which is ambiguous for + # hyphenated dataset ids without a dataset hint. + # Bare items are the "default" variant, matching + # the resolvers. + props = item.properties or {} + property_dataset = props.get("dclimate:dataset_id") + property_variant = props.get("dclimate:variant") + if property_dataset: + parsed_dataset, parsed_variant = ( + _dataset_and_variant_from_item_id( + item.id, + collection_id, + dataset=property_dataset, + ) + ) + elif property_variant: + parsed_dataset, parsed_variant = ( + _dataset_and_variant_from_item_id( + item.id, + collection_id, + variant=property_variant, + ) + ) + if parsed_dataset is None: + parsed_dataset, parsed_variant = ( + _dataset_and_variant_from_known_datasets( + item.id, + collection_id, + known_datasets, + ) + ) + else: + parsed_dataset, parsed_variant = ( + _dataset_and_variant_from_known_datasets( + item.id, + collection_id, + known_datasets, + ) + ) + item_dataset = property_dataset or parsed_dataset + if item_dataset is None: + continue + item_variant = ( + property_variant or parsed_variant or "default" ) - parts = remainder.split("-") - item_dataset = parts[0] if parts else remainder - item_variant = parts[1] if len(parts) > 1 else "" cid: Optional[str] = None if "data" in item.assets: @@ -487,6 +659,8 @@ def list_available_datasets(catalog: pystac.Catalog) -> Dict[str, Dict[str, Any] if temporal: variant_entry["temporal_extent"] = temporal entry["variants"].append(variant_entry) + known_datasets.add(item_dataset) + entry["types"] = sorted(known_datasets) except Exception: logger.debug( "Could not resolve items for collection %s", diff --git a/dclimate_client_py/stac_server.py b/dclimate_client_py/stac_server.py index da164de..5a66995 100644 --- a/dclimate_client_py/stac_server.py +++ b/dclimate_client_py/stac_server.py @@ -5,25 +5,125 @@ which is faster than traversing the IPFS-hosted catalog structure. """ -from typing import Any, Dict, Optional, Set -import requests +from collections.abc import Iterator +from json import dumps +from threading import Lock +from typing import Any, Dict, Iterable, NamedTuple, Optional, Set +from urllib.parse import urljoin + +import httpx from .datasets import SpatialExtent, TemporalExtent STAC_SERVER_URL = "https://api.stac.dclimate.net" -def _dataset_id_from_item_id(feature_id: str, collection: str) -> Optional[str]: +_MAX_SEARCH_PAGES = 50 +_HTTP_CLIENT: httpx.Client | None = None +_HTTP_CLIENT_LOCK = Lock() + + +def _client() -> httpx.Client: + """Return the process-wide pooled client used for synchronous STAC calls.""" + global _HTTP_CLIENT + if _HTTP_CLIENT is None: + with _HTTP_CLIENT_LOCK: + if _HTTP_CLIENT is None: + _HTTP_CLIENT = httpx.Client(timeout=30, follow_redirects=True) + return _HTTP_CLIENT + + +class ResolvedDataset(NamedTuple): + cid: str + variant: str + + +def _dataset_and_variant_from_item_id( + feature_id: str, + collection: str, + dataset: Optional[str] = None, + variant: Optional[str] = None, +) -> tuple[Optional[str], Optional[str]]: prefix = f"{collection}-" remainder = ( feature_id[len(prefix) :] if feature_id.startswith(prefix) else feature_id ) - dataset, _, _ = remainder.partition("-") - return dataset or None + if dataset is not None: + if remainder == dataset: + return dataset, None + dataset_prefix = f"{dataset}-" + if remainder.startswith(dataset_prefix): + return dataset, remainder[len(dataset_prefix) :] or None + return None, None + + if variant is not None: + variant_suffix = f"-{variant}" + if remainder.endswith(variant_suffix): + return remainder[: -len(variant_suffix)] or None, variant + return None, None + + parsed_dataset, separator, variant = remainder.partition("-") + return parsed_dataset or None, (variant or None) if separator else None + + +def _dataset_id_from_item_id( + feature_id: str, + collection: str, + dataset: Optional[str] = None, +) -> Optional[str]: + parsed_dataset, _ = _dataset_and_variant_from_item_id( + feature_id, collection, dataset + ) + return parsed_dataset + + +def _dataset_and_variant_from_known_datasets( + feature_id: str, + collection: str, + known_datasets: Iterable[str], +) -> tuple[Optional[str], Optional[str]]: + """Parse an item id using the longest matching known dataset id. + + Hyphens delimit both dataset ids and variants, so an unhinted id such as + ``chirps-precip-daily-final-p05`` is inherently ambiguous. Collection + metadata and explicit sibling-item properties provide the missing hint. + """ + candidates = sorted( + {value for value in known_datasets if isinstance(value, str) and value}, + key=len, + reverse=True, + ) + for candidate in candidates: + parsed_dataset, parsed_variant = _dataset_and_variant_from_item_id( + feature_id, collection, dataset=candidate + ) + if parsed_dataset is not None: + return parsed_dataset, parsed_variant + return _dataset_and_variant_from_item_id(feature_id, collection) + + +def _feature_variant( + feature: Dict[str, Any], collection: str, dataset: Optional[str] = None +) -> Optional[str]: + props = feature.get("properties") or {} + variant = props.get("dclimate:variant") + if variant: + return variant + + feature_id = feature.get("id") + if not isinstance(feature_id, str): + return None + _, parsed_variant = _dataset_and_variant_from_item_id( + feature_id, collection, dataset + ) + return parsed_variant def _feature_matches_dataset( - feature: Dict[str, Any], collection: str, dataset: str + feature: Dict[str, Any], + collection: str, + dataset: str, + known_datasets: Iterable[str] = (), ) -> bool: feature_collection = feature.get("collection") if feature_collection and feature_collection != collection: @@ -37,7 +137,105 @@ def _feature_matches_dataset( feature_id = feature.get("id") if not isinstance(feature_id, str): return False - return _dataset_id_from_item_id(feature_id, collection) == dataset + variant = props.get("dclimate:variant") + if variant: + parsed_dataset, _ = _dataset_and_variant_from_item_id( + feature_id, collection, variant=variant + ) + if parsed_dataset is not None: + return parsed_dataset == dataset + # The id does not encode the property variant (e.g. a bare id with + # properties variant "default") — fall through to dataset matching. + if known_datasets: + parsed_dataset, _ = _dataset_and_variant_from_known_datasets( + feature_id, collection, known_datasets + ) + return parsed_dataset == dataset + return _dataset_id_from_item_id(feature_id, collection, dataset) == dataset + + +def _search_pages( + server_url: str, + body: Dict[str, Any], + timeout: int, +) -> Iterator[Dict[str, Any]]: + """Yield bounded STAC search pages while following ``rel=next`` links.""" + url = f"{server_url.rstrip('/')}/search" + method = "POST" + request_body: Optional[Dict[str, Any]] = body + request_headers: Dict[str, str] = {} + seen: Set[tuple[str, str, str, str]] = set() + + for _ in range(_MAX_SEARCH_PAGES): + page_key = ( + method, + url, + dumps(request_body, sort_keys=True, default=str), + dumps(request_headers, sort_keys=True, default=str), + ) + if page_key in seen: + return + seen.add(page_key) + + if method == "POST": + request_kwargs: Dict[str, Any] = { + "json": request_body, + "timeout": timeout, + } + if request_headers: + request_kwargs["headers"] = request_headers + response = _client().post(url, **request_kwargs) + else: + request_kwargs = {"params": request_body or None, "timeout": timeout} + if request_headers: + request_kwargs["headers"] = request_headers + response = _client().get(url, **request_kwargs) + response.raise_for_status() + page = response.json() + yield page + + if not (page.get("features") or []): + return + next_link = next( + ( + link + for link in page.get("links", []) or [] + if link.get("rel") == "next" and link.get("href") + ), + None, + ) + if next_link is None: + return + + url = urljoin(url, next_link["href"]) + method = str(next_link.get("method", "GET")).upper() + if method not in {"GET", "POST"}: + return + linked_headers = next_link.get("headers") + request_headers = linked_headers if isinstance(linked_headers, dict) else {} + linked_body = next_link.get("body") + if isinstance(linked_body, dict): + # STAC API next-link contract: with "merge": true the linked + # body extends the original request (keeping filters like + # "collections"); otherwise it replaces it wholesale. + if next_link.get("merge"): + request_body = {**body, **linked_body} + else: + request_body = linked_body + elif next_link.get("merge"): + # ``merge: true`` without a link body still carries the original + # search filters to the next request. + request_body = dict(body) + else: + request_body = None + else: + # Reaching the bound with a valid next link means the result is + # incomplete. Surface that explicitly so callers can use their + # catalog fallback instead of accepting truncated search results. + raise ValueError( + f"STAC search reached its page limit of {_MAX_SEARCH_PAGES} " + "while another next link was present" + ) def resolve_cid_from_stac_server( @@ -45,10 +243,13 @@ def resolve_cid_from_stac_server( dataset: str, variant: Optional[str] = None, server_url: str = STAC_SERVER_URL, -) -> str: +) -> ResolvedDataset: """ Resolve dataset CID via STAC server /search API. + Changed in 0.6: returns ResolvedDataset; variant='' is treated as an + explicit (unresolvable) variant rather than no-variant. + Uses the same API format as the frontend (POST /search with collections filter). Args: @@ -58,11 +259,11 @@ def resolve_cid_from_stac_server( server_url: STAC server base URL Returns: - str: The IPFS CID of the Zarr dataset (without 'ipfs://' prefix) + ResolvedDataset: The IPFS CID and selected variant Raises: ValueError: If dataset or variant is not found - requests.HTTPError: If the server request fails + httpx.HTTPError: If the server request fails """ # Search by collection body = { @@ -70,21 +271,39 @@ def resolve_cid_from_stac_server( "collections": [collection], } - response = requests.post(f"{server_url}/search", json=body, timeout=10) - response.raise_for_status() - - features = response.json().get("features", []) - + # An item with no variant segment/property is what the listing API + # reports as the "default" variant — keep resolve symmetric with list. + def _effective_variant(feature: Dict[str, Any]) -> str: + return _feature_variant(feature, collection, dataset) or "default" + + features: list[Dict[str, Any]] = [] + for page in _search_pages(server_url, body, timeout=10): + features.extend(page.get("features", []) or []) + + known_datasets = { + dataset_id + for feature in features + if isinstance(feature, dict) + for dataset_id in [(feature.get("properties") or {}).get("dclimate:dataset_id")] + if isinstance(dataset_id, str) and dataset_id + } + known_datasets.add(dataset) # Filter to the exact dataset. A prefix match would conflate datasets such - # as precipitation_total and precipitation_total_land. - matches = [f for f in features if _feature_matches_dataset(f, collection, dataset)] + # as ``precip`` and a known hyphenated dataset ``precip-daily``. + matches = [ + feature + for feature in features + if _feature_matches_dataset( + feature, collection, dataset, known_datasets=known_datasets + ) + ] if not matches: raise ValueError(f"No items found for {collection}/{dataset}") # Select by variant or use default preference - if variant: + if variant is not None: item = next( - (f for f in matches if f["properties"].get("dclimate:variant") == variant), + (f for f in matches if _effective_variant(f) == variant), None, ) if not item: @@ -96,11 +315,7 @@ def resolve_cid_from_stac_server( item = matches[0] for preferred in ["default", "final", "finalized", "latest"]: found = next( - ( - f - for f in matches - if f["properties"].get("dclimate:variant") == preferred - ), + (f for f in matches if _effective_variant(f) == preferred), None, ) if found: @@ -108,11 +323,12 @@ def resolve_cid_from_stac_server( break # Extract CID from asset + selected_variant = variant if variant is not None else _effective_variant(item) href = item.get("assets", {}).get("data", {}).get("href", "") if href.startswith("ipfs://"): - return href.replace("ipfs://", "") + return ResolvedDataset(href.replace("ipfs://", ""), selected_variant) if href: - return href + return ResolvedDataset(href, selected_variant) raise ValueError(f"Item '{item['id']}' has no data asset") @@ -130,7 +346,7 @@ def list_available_datasets_from_stac_server( List all datasets/variants by querying a STAC API server directly. Fast path that mirrors ``list_available_datasets`` (the IPFS walker) without - traversing the IPFS-hosted catalog tree. Two requests: + traversing the IPFS-hosted catalog tree. It queries: 1. ``GET /collections`` — collection ids, titles 2. ``POST /search`` — items, with dataset/variant/CID in properties @@ -148,21 +364,11 @@ def list_available_datasets_from_stac_server( ``dclimate:observation`` properties — only when every item in the collection agrees, to avoid picking a misleading value when items disagree. - - The fixed ``limit: 1000`` covers today's catalog (~45 items) by a wide - margin. If the catalog grows past that, switch to following the STAC - ``next`` link instead of a single request. + - Search pagination is bounded to avoid looping on malformed ``next`` links. """ - collections_resp = requests.get(f"{server_url}/collections", timeout=10) + collections_resp = _client().get(f"{server_url}/collections", timeout=10) collections_resp.raise_for_status() - search_resp = requests.post( - f"{server_url}/search", - json={"limit": 1000}, - timeout=15, - ) - search_resp.raise_for_status() - collections_body = collections_resp.json() - search_body = search_resp.json() # Accumulator per collection. Built up from /collections then enriched by # the /search response. Collections that have no items end up filtered out @@ -181,9 +387,40 @@ def list_available_datasets_from_stac_server( "organization": organization, "observations": set(), "datasets": {}, # dataset_name -> { variant_name -> variant_entry } + "known_datasets": set(), } - for feature in search_body.get("features", []) or []: + summaries = coll.get("summaries") or {} + declared_datasets = coll.get("dclimate:types") or summaries.get( + "dclimate:dataset_id", [] + ) + if isinstance(declared_datasets, list): + accumulators[coll_id]["known_datasets"].update( + value for value in declared_datasets if isinstance(value, str) and value + ) + + search_features = [ + feature + for page in _search_pages(server_url, {"limit": 100}, timeout=15) + for feature in page.get("features", []) or [] + ] + dataset_hints: Dict[str, Set[str]] = {} + for feature in search_features: + collection_id = feature.get("collection") + props = feature.get("properties") or {} + dataset_id = props.get("dclimate:dataset_id") + if ( + isinstance(collection_id, str) + and isinstance(dataset_id, str) + and dataset_id + ): + dataset_hints.setdefault(collection_id, set()).add(dataset_id) + + for collection_id, hints in dataset_hints.items(): + if collection_id in accumulators: + accumulators[collection_id]["known_datasets"].update(hints) + + for feature in search_features: feature_id = feature.get("id", "") collection_id = feature.get("collection") if not collection_id and isinstance(feature_id, str) and "-" in feature_id: @@ -202,6 +439,7 @@ def list_available_datasets_from_stac_server( "organization": organization, "observations": set(), "datasets": {}, + "known_datasets": set(dataset_hints.get(collection_id, set())), } accumulators[collection_id] = entry @@ -212,17 +450,37 @@ def list_available_datasets_from_stac_server( # Prefer explicit dclimate:* properties; fall back to id-parsing for # items that pre-date the property convention. - id_parts = feature_id.split("-") if isinstance(feature_id, str) else [] - dataset_name = props.get("dclimate:dataset_id") or ( - id_parts[1] if len(id_parts) >= 2 else None - ) - variant_name = props.get("dclimate:variant") or ( - "-".join(id_parts[2:]) if len(id_parts) >= 3 else "default" - ) + property_dataset = props.get("dclimate:dataset_id") + property_variant = props.get("dclimate:variant") + if not isinstance(feature_id, str): + parsed_dataset, parsed_variant = None, None + elif property_dataset: + parsed_dataset, parsed_variant = _dataset_and_variant_from_item_id( + feature_id, collection_id, dataset=property_dataset + ) + elif property_variant: + parsed_dataset, parsed_variant = _dataset_and_variant_from_item_id( + feature_id, collection_id, variant=property_variant + ) + if parsed_dataset is None: + parsed_dataset, _ = _dataset_and_variant_from_known_datasets( + feature_id, collection_id, entry["known_datasets"] + ) + parsed_variant = property_variant + else: + parsed_dataset, parsed_variant = _dataset_and_variant_from_known_datasets( + feature_id, collection_id, entry["known_datasets"] + ) + dataset_name = property_dataset or parsed_dataset + variant_name = props.get("dclimate:variant") or parsed_variant or "default" if not dataset_name: continue cid = _strip_ipfs_scheme(props.get("dclimate:latest_dataset_cid")) + if not cid: + cid = _strip_ipfs_scheme( + (feature.get("assets") or {}).get("data", {}).get("href") + ) variant_entry: Dict[str, Any] = { "dataset": dataset_name, diff --git a/pyproject.toml b/pyproject.toml index fc0d750..32cba39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "pdm.backend" [project] name = "dclimate-client-py" -version = "0.5.11" # Set a static version or handle it in versioning strategy +version = "0.6.0" # Set a static version or handle it in versioning strategy description = "Python client library for accessing dClimate weather and climate data" readme = "README.md" license = {text = "MIT"} @@ -29,24 +29,28 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] dependencies = [ - "aiobotocore", + "aiobotocore>=2.13.0", + "aiohttp>=3.9.0", "xarray>=2025.3.0", - "rioxarray", + "rioxarray>=0.15.0", "zarr>=3.0.8", + # numcodecs 0.16's zarr3 compatibility module requires zarr >=3.1.3, + # while py-hamt 3.x currently constrains zarr below 3.1. + "numcodecs[crc32c]>=0.14,<0.16", "numpy>=2.1.3", - "py_hamt>=3.4.1", - "requests", - "pyarrow", - "geopandas", - "pandas", - "s3fs", - "shapely", - "scipy", + "py_hamt>=3.5.0", + "multiformats>=0.3.1", + "geopandas>=1.0.0", + "pandas>=2.2.0", + "s3fs>=2024.6.0", + "shapely>=2.0.0", + # xarray's in-memory to_netcdf() (GeotemporalData.to_netcdf with no args) + # requires the scipy netCDF backend + "scipy>=1.12.0", "pycryptodome>=3.21.0", "pystac>=1.10.0", "httpx>=0.27.0", "x402[evm,httpx]>=2.1.0", - "python-dotenv>=1.0.0", "opentelemetry-api>=1.30.0", ] @@ -54,12 +58,28 @@ dependencies = [ Homepage = "https://dclimate.net/" [project.optional-dependencies] -testing = ["pytest", "pytest-cov", "pytest-mock", "pytest-asyncio>=1.3.0"] +testing = [ + "mypy>=1.14", + "pytest", + "pytest-cov", + "pytest-mock", + "pytest-asyncio>=1.3.0", +] dev = ["pre-commit>=4.1.0", "ruff>=0.9.5"] +examples = ["python-dotenv>=1.0.0"] [tool.pytest.ini_options] +testpaths = ["tests"] +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" markers = [ "ipfs: marks tests requiring a running IPFS daemon and network access", + "ipfs_rpc: marks tests requiring a writable IPFS RPC endpoint", + "stac_pointer: marks tests requiring the dClimate STAC CID endpoint", "integration: marks integration tests (often slower, may require external services like IPFS)", # Add other custom marks here if you create more ] + +[tool.mypy] +python_version = "3.12" +ignore_missing_imports = true diff --git a/scripts/benchmark_gateway.py b/scripts/benchmark_gateway.py new file mode 100644 index 0000000..24a8e85 --- /dev/null +++ b/scripts/benchmark_gateway.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Benchmark bounded dClimate gateway reads using the py-hamt #58 methodology. + +The benchmark compares HTTP/2 with HTTP/1.1 while holding Kubo request +concurrency and the dataset sample constant. The companion infrastructure +change is to raise nginx's ``keepalive_requests`` from its default of 1000 on +the dClimate gateway: large dataset opens otherwise cause GOAWAY churn. +Benchmark HTTP/2 on and off both before and after that gateway change. +""" + +import argparse +import json +import statistics +import sys +import time +from typing import Any + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--http2", + action=argparse.BooleanOptionalAction, + default=True, + help="enable HTTP/2 for gateway requests (default: enabled)", + ) + parser.add_argument( + "--concurrency", + type=int, + default=32, + help="maximum concurrent Kubo gateway requests (default: 32)", + ) + parser.add_argument( + "--repetitions", + type=int, + default=3, + help="number of timed dataset opens (default: 3)", + ) + parser.add_argument("--collection", default="cpc-precip-conus") + parser.add_argument("--dataset", default="precip") + parser.add_argument("--variant", default=None) + parser.add_argument("--gateway", default="https://ipfs-gateway.dclimate.net") + parser.add_argument("--stac-server", default="https://api.stac.dclimate.net") + return parser + + +async def _benchmark(args: argparse.Namespace) -> dict[str, Any]: + import httpx + + from dclimate_client_py import dClimateClient + + elapsed_times: list[float] = [] + bytes_read: list[int] = [] + + for repetition in range(1, args.repetitions + 1): + started = time.perf_counter() + async with dClimateClient( + gateway_base_url=args.gateway, + rpc_base_url=args.gateway, + stac_server_url=args.stac_server, + concurrency=args.concurrency, + client_factory=lambda: httpx.AsyncClient( + http2=args.http2, + timeout=60.0, + follow_redirects=True, + ), + ) as client: + dataset, _metadata = await client.load_dataset( + collection=args.collection, + dataset=args.dataset, + variant=args.variant, + return_xarray=True, + ) + bounded_indexers = { + dimension: slice(0, min(size, 8)) + for dimension, size in dataset.sizes.items() + } + sample = dataset.isel(bounded_indexers).load() + sample_bytes = sample.nbytes + elapsed = time.perf_counter() - started + elapsed_times.append(elapsed) + bytes_read.append(sample_bytes) + print( + f"repetition {repetition}/{args.repetitions}: " + f"{elapsed:.3f} s ({sample_bytes} sample bytes)", + file=sys.stderr, + ) + + median = statistics.median(elapsed_times) + print(f"median: {median:.3f} s", file=sys.stderr) + return { + "http2": args.http2, + "concurrency": args.concurrency, + "repetitions": args.repetitions, + "collection": args.collection, + "dataset": args.dataset, + "variant": args.variant, + "gateway": args.gateway, + "stac_server": args.stac_server, + "times_seconds": elapsed_times, + "median_seconds": median, + "sample_bytes": bytes_read, + } + + +def main() -> None: + parser = _parser() + args = parser.parse_args() + if args.concurrency <= 0: + parser.error("--concurrency must be positive") + if args.repetitions <= 0: + parser.error("--repetitions must be positive") + + import asyncio + + try: + result = asyncio.run(_benchmark(args)) + except Exception as exc: # noqa: BLE001 - CLI failures should be one concise line + print(f"failure: {type(exc).__name__}: {exc}", file=sys.stderr) + raise SystemExit(1) from None + else: + print(json.dumps(result, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/test_stac_integration.py b/scripts/stac_integration.py similarity index 98% rename from test_stac_integration.py rename to scripts/stac_integration.py index 5d3e021..538ad3f 100644 --- a/test_stac_integration.py +++ b/scripts/stac_integration.py @@ -3,7 +3,7 @@ Test script for STAC integration This script tests the STAC catalog integration with the dClimate client. -Run with: python test_stac_integration.py +Run with: uv run python scripts/stac_integration.py """ import asyncio diff --git a/tests/conftest.py b/tests/conftest.py index 9466a41..c91e9de 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,14 +3,36 @@ import pathlib import geopandas as gpd -import os import numpy as np import pytest import xarray as xr -import requests # Import requests here for the check +import httpx import zarr import zarr.storage +from tests.ipfs_config import IPFS_GATEWAY_URL, IPFS_RPC_URL, STAC_CATALOG_URL + + +@pytest.fixture +def install_httpx_mock(monkeypatch): + """Inject a pooled MockTransport client through a module's client accessor.""" + clients: list[httpx.Client] = [] + + def install(module, handler): + client = httpx.Client( + transport=httpx.MockTransport(handler), + timeout=30, + follow_redirects=True, + ) + clients.append(client) + monkeypatch.setattr(module, "_client", lambda: client) + return client + + yield install + + for client in clients: + client.close() + def pytest_addoption(parser): """Add custom pytest command line options.""" @@ -28,16 +50,46 @@ def pytest_configure(config): "markers", "integration: mark test as integration test requiring external services", ) + config.addinivalue_line( + "markers", "ipfs_rpc: mark test as requiring a writable IPFS RPC endpoint" + ) + config.addinivalue_line( + "markers", "stac_pointer: mark test as requiring the STAC CID endpoint" + ) def pytest_collection_modifyitems(config, items): - """Skip integration tests unless --run-integration is passed.""" - if config.getoption("--run-integration"): - return + """Gate tests that require external integration services.""" skip_integration = pytest.mark.skip(reason="need --run-integration option to run") + ipfs_items = [item for item in items if "ipfs" in item.keywords] + skip_ipfs = None + if ipfs_items: + if not is_ipfs_running(IPFS_GATEWAY_URL): + skip_ipfs = pytest.mark.skip( + reason=f"IPFS gateway not responding at {IPFS_GATEWAY_URL}" + ) + ipfs_rpc_items = [item for item in items if "ipfs_rpc" in item.keywords] + skip_ipfs_rpc = None + if ipfs_rpc_items and not is_ipfs_rpc_running(IPFS_RPC_URL): + skip_ipfs_rpc = pytest.mark.skip( + reason=f"IPFS RPC endpoint not responding at {IPFS_RPC_URL}" + ) + stac_pointer_items = [item for item in items if "stac_pointer" in item.keywords] + skip_stac_pointer = None + if stac_pointer_items and not is_stac_pointer_running(STAC_CATALOG_URL): + skip_stac_pointer = pytest.mark.skip( + reason=f"STAC catalog pointer not responding at {STAC_CATALOG_URL}" + ) + for item in items: - if "integration" in item.keywords: + if "integration" in item.keywords and not config.getoption("--run-integration"): item.add_marker(skip_integration) + if "ipfs" in item.keywords and skip_ipfs is not None: + item.add_marker(skip_ipfs) + if "ipfs_rpc" in item.keywords and skip_ipfs_rpc is not None: + item.add_marker(skip_ipfs_rpc) + if "stac_pointer" in item.keywords and skip_stac_pointer is not None: + item.add_marker(skip_stac_pointer) HERE = pathlib.Path(__file__).parent @@ -49,7 +101,7 @@ def pytest_collection_modifyitems(config, items): def input_ds(): # Keeping local fixtures for tests that don't need IPFS loading (like test_geotemporal_data) with zarr.storage.ZipStore(ETC / "retrieval_test.zip", mode="r") as in_zarr: - return xr.open_zarr(in_zarr, chunks=None).compute() + return xr.open_zarr(in_zarr, chunks=None, decode_timedelta=True).compute() @pytest.fixture @@ -58,7 +110,7 @@ def forecast_ds(): with zarr.storage.ZipStore( ETC / "forecast_retrieval_test.zip", mode="r" ) as in_zarr: - return xr.open_zarr(in_zarr, chunks=None).compute() + return xr.open_zarr(in_zarr, chunks=None, decode_timedelta=True).compute() @pytest.fixture @@ -130,18 +182,6 @@ def single_var_dataset(): return make_dataset(vars=1) -# --- Add IPFS Connection Check Fixture --- -# We need the check_ipfs_connection fixture available for multiple test files -# Let's reuse the one from test_integration_ipfs.py - - -@pytest.fixture(scope="session") # Changed scope to session for efficiency -def ipfs_gateway_url(): - """Returns the IPFS gateway URL to check.""" - # Prioritize environment variable, then default from py-hamt's IPFSStore - return os.environ.get("IPFS_GATEWAY_URI_STEM", "http://127.0.0.1:8080") - - def is_ipfs_running(gateway_url: str) -> bool: """Check if IPFS daemon Gateway is responsive.""" @@ -151,8 +191,10 @@ def is_ipfs_running(gateway_url: str) -> bool: # Use a known immutable CID (e.g., the empty directory CID) # Let's try a known immutable path: "Hello from IPFS Gateway Checker" known_cid = "bafybeifx7yeb55armcsxwwitkymga5xf53dxiarykms3ygqic223w5sk3m" # Example file - response = requests.head( - f"{gateway_url.rstrip('/')}/ipfs/{known_cid}", timeout=5 + response = httpx.head( + f"{gateway_url.rstrip('/')}/ipfs/{known_cid}", + timeout=5, + follow_redirects=True, ) # Allow 200 OK or 404 Not Found (if CID isn't locally available but gateway is up) # Avoid checking strict 200 as CID might not be pinned locally but gateway is running @@ -166,29 +208,37 @@ def is_ipfs_running(gateway_url: str) -> bool: f"IPFS Gateway check failed (Status: {response.status_code}) at {gateway_url}" ) return False - except requests.exceptions.ConnectionError: + except httpx.ConnectError: print(f"IPFS Gateway connection failed at {gateway_url}") return False - except requests.exceptions.Timeout: + except httpx.TimeoutException: print(f"IPFS Gateway check timed out at {gateway_url}") return False - except requests.exceptions.RequestException as e: + except httpx.HTTPError as e: print(f"IPFS Gateway check failed with unexpected error: {e}") return False -# Apply autouse=True to run this check once for the session for all tests -@pytest.fixture(scope="session", autouse=True) -def check_ipfs_connection(ipfs_gateway_url): - """Skips tests if IPFS daemon Gateway is not accessible.""" - if not is_ipfs_running(ipfs_gateway_url): - pytest.skip( - f"IPFS daemon Gateway not responding at {ipfs_gateway_url}. Skipping integration tests." - ) - else: - print( - f"IPFS daemon Gateway responding at {ipfs_gateway_url}. Proceeding with integration tests." - ) +def is_ipfs_rpc_running(rpc_url: str) -> bool: + """Check whether the writable Kubo RPC API is responsive.""" + try: + response = httpx.post(f"{rpc_url}/api/v0/id", timeout=5, follow_redirects=True) + response.raise_for_status() + payload = response.json() + return isinstance(payload, dict) and bool(payload.get("ID")) + except (httpx.HTTPError, ValueError): + return False + + +def is_stac_pointer_running(catalog_url: str) -> bool: + """Check whether the STAC pointer returns a non-empty root CID.""" + try: + response = httpx.get(catalog_url, timeout=5, follow_redirects=True) + response.raise_for_status() + payload = response.json() + return isinstance(payload, dict) and bool(payload.get("cid")) + except (httpx.HTTPError, ValueError): + return False # Define known dataset IDs accessible via STAC for tests diff --git a/tests/ipfs_config.py b/tests/ipfs_config.py new file mode 100644 index 0000000..81856ea --- /dev/null +++ b/tests/ipfs_config.py @@ -0,0 +1,18 @@ +"""Shared endpoint settings for IPFS integration tests.""" + +from __future__ import annotations + +import os + + +IPFS_GATEWAY_URL = ( + os.environ.get("IPFS_GATEWAY_URI_STEM") + or os.environ.get("DCLIMATE_IPFS_GATEWAY") + or "http://127.0.0.1:8080" +).rstrip("/") + +IPFS_RPC_URL = os.environ.get("IPFS_RPC_URI_STEM", "http://127.0.0.1:5001").rstrip("/") + +STAC_CATALOG_URL = os.environ.get( + "DCLIMATE_STAC_CATALOG_URL", "https://ipfs-gateway.dclimate.net/stac" +) diff --git a/tests/test_client.py b/tests/test_client.py index 8e2ed0c..0b6b8bf 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -21,11 +21,6 @@ KNOWN_STAC_DATE_END, ) -# --- Test Markers --- -# pytestmark = pytest.mark.client # Mark tests specific to the client module -# Apply IPFS check fixture to relevant tests/module if not session-wide autouse -pytestmark = pytest.mark.usefixtures("check_ipfs_connection") - # Keep S3 sample path if needed for S3 tests SAMPLE_ZARRS = pathlib.Path(__file__).parent / "etc" / "sample_zarrs" diff --git a/tests/test_debug.py b/tests/test_debug.py deleted file mode 100644 index d836e9e..0000000 --- a/tests/test_debug.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Debug test for loading noaa_gfs:temperature_max_forecast:default dataset.""" - -import pytest -from dclimate_client_py.dclimate_client import dClimateClient - - -@pytest.mark.asyncio -async def test_load_noaa_gfs_temperature_max_forecast(): - """Test loading noaa_gfs:temperature_max_forecast:default dataset.""" - async with dClimateClient() as client: - print("\n--- Attempting to load dataset ---") - print("Collection: noaa_gfs") - print("Dataset: temperature_max_forecast") - print("Variant: default") - - try: - dataset, metadata = await client.load_dataset( - collection="noaa_gfs", - dataset="temperature_max_forecast", - variant="default", - return_xarray=True, - ) - - print("\n--- Dataset loaded successfully ---") - print(f"Dataset type: {type(dataset)}") - print(f"Dataset: {dataset}") - print(f"\nMetadata: {metadata}") - - if hasattr(dataset, "data_vars"): - print(f"\nData variables: {list(dataset.data_vars)}") - if hasattr(dataset, "coords"): - print(f"Coordinates: {list(dataset.coords)}") - if hasattr(dataset, "dims"): - print(f"Dimensions: {dict(dataset.dims)}") - - except Exception as e: - print("\n--- Error loading dataset ---") - print(f"Error type: {type(e).__name__}") - print(f"Error message: {e}") - raise - - -@pytest.mark.asyncio -async def test_load_noaa_gfs_with_xarray_false(): - """Test loading noaa_gfs dataset with return_xarray=False.""" - async with dClimateClient() as client: - print("\n--- Attempting to load dataset (return_xarray=False) ---") - - try: - dataset, metadata = await client.load_dataset( - collection="noaa_gfs", - dataset="temperature_max_forecast", - variant="default", - return_xarray=False, - ) - - print("\n--- Dataset loaded successfully ---") - print(f"Dataset type: {type(dataset)}") - print(f"Dataset: {dataset}") - print(f"\nMetadata: {metadata}") - - if hasattr(dataset, "data"): - print(f"\nUnderlying data type: {type(dataset.data)}") - print( - f"Data vars: {list(dataset.data.data_vars) if hasattr(dataset.data, 'data_vars') else 'N/A'}" - ) - - except Exception as e: - print("\n--- Error loading dataset ---") - print(f"Error type: {type(e).__name__}") - print(f"Error message: {e}") - raise - - -@pytest.mark.asyncio -async def test_list_available_datasets(): - """List available datasets to verify noaa_gfs exists.""" - async with dClimateClient() as client: - print("\n--- Checking catalog for noaa_gfs datasets ---") - - try: - # Try to access catalog if available - if hasattr(client, "catalog") or hasattr(client, "get_catalog"): - catalog = ( - await client.get_catalog() - if hasattr(client, "get_catalog") - else client.catalog - ) - print(f"Catalog: {catalog}") - else: - print("No catalog method available on client") - - # Try searching for noaa_gfs - if hasattr(client, "search"): - results = await client.search("noaa_gfs") - print(f"Search results for 'noaa_gfs': {results}") - - except Exception as e: - print(f"Error accessing catalog: {type(e).__name__} - {e}") diff --git a/tests/test_ipfs_retrieval.py b/tests/test_ipfs_retrieval.py index 1197fd0..8b59c3c 100644 --- a/tests/test_ipfs_retrieval.py +++ b/tests/test_ipfs_retrieval.py @@ -1,5 +1,6 @@ import warnings +import httpx import pytest import xarray as xr @@ -28,25 +29,54 @@ def _v2_top_level_groups(self): return {"1", "0"} -@pytest.fixture(autouse=True) -def check_ipfs_connection(): - return None +def _chained(wrapper: Exception, cause: Exception) -> Exception: + wrapper.__cause__ = cause + return wrapper @pytest.mark.parametrize( - "message", + "error", [ - "Connection refused", - "Max retries exceeded", - "Name or service not known", - "network is unreachable", - "nodename nor servname provided", - "temporary failure in name resolution", - "timed out opening sharded store", + ConnectionError("connection refused"), + TimeoutError("timed out opening sharded store"), + httpx.ConnectTimeout("connect timed out"), + httpx.ReadTimeout("gateway timed out"), + httpx.ConnectError("connection refused"), + httpx.ConnectError("max retries exceeded"), + _chained(RuntimeError("wrapped"), httpx.ReadTimeout("gateway timed out")), ], ) -def test_is_connection_error_classifies_gateway_failures(message): - assert ipfs_retrieval._is_connection_error(RuntimeError(message)) +def test_is_connection_error_classifies_gateway_failures(error): + assert ipfs_retrieval._is_connection_error(error) + + +@pytest.mark.parametrize( + "error", + [ + FileNotFoundError("no such shard file"), + PermissionError("permission denied"), + IsADirectoryError("is a directory"), + ValueError("not a sharded zarr store"), + httpx.HTTPStatusError( + "500 Server Error: Internal Server Error", + request=httpx.Request("GET", "https://gateway.example"), + response=httpx.Response(500), + ), + _chained( + RuntimeError("retries exhausted on 500s"), + httpx.HTTPStatusError( + "too many 500 responses", + request=httpx.Request("GET", "https://gateway.example"), + response=httpx.Response(500), + ), + ), + _chained(RuntimeError("wrapped"), FileNotFoundError("missing metadata")), + ], +) +def test_is_connection_error_rejects_non_network_failures(error): + # Filesystem/parse errors must not classify as gateway failures, or the + # caller would skip the HAMT fallback for them. + assert not ipfs_retrieval._is_connection_error(error) @pytest.mark.asyncio @@ -77,7 +107,8 @@ async def sharded_open(**kwargs): opened_groups = [] - def open_zarr(*, store, group=None): + def open_zarr(*, store, group=None, decode_timedelta=False): + assert decode_timedelta is True opened_groups.append(group) return xr.Dataset() @@ -102,7 +133,8 @@ async def sharded_open(**kwargs): opened_groups = [] - def open_zarr(*, store, group=None): + def open_zarr(*, store, group=None, decode_timedelta=False): + assert decode_timedelta is True opened_groups.append(group) return xr.Dataset() @@ -127,7 +159,8 @@ async def sharded_open(**kwargs): async def hamt_build(**kwargs): raise AssertionError("HAMT fallback should not be attempted") - def open_zarr(*, store, group=None): + def open_zarr(*, store, group=None, decode_timedelta=False): + assert decode_timedelta is True raise ValueError("explicit Zarr group required") monkeypatch.setattr(ipfs_retrieval.ShardedZarrStore, "open", sharded_open) @@ -155,7 +188,7 @@ async def sharded_open(**kwargs): monkeypatch.setattr( ipfs_retrieval.xr, "open_zarr", - lambda *, store, group=None: xr.Dataset(), + lambda *, store, group=None, decode_timedelta=False: xr.Dataset(), ) with warnings.catch_warnings(record=True) as caught_warnings: @@ -184,7 +217,8 @@ async def hamt_build(**kwargs): opened_groups = [] - def open_zarr(*, store, group=None): + def open_zarr(*, store, group=None, decode_timedelta=False): + assert decode_timedelta is True opened_groups.append(group) return xr.Dataset() diff --git a/tests/test_list_datasets_parity.py b/tests/test_list_datasets_parity.py index 517147d..6603474 100644 --- a/tests/test_list_datasets_parity.py +++ b/tests/test_list_datasets_parity.py @@ -22,17 +22,16 @@ pytest tests/test_list_datasets_parity.py --run-integration -The autouse ``check_ipfs_connection`` fixture in ``conftest.py`` points at a -local IPFS daemon by default; override via the ``IPFS_GATEWAY_URI_STEM`` -environment variable to point at the public gateway. +The ``ipfs`` marker gate checks a local IPFS daemon by default; override +``IPFS_GATEWAY_URI_STEM`` to point at the public gateway. """ import json import os from typing import Any, Dict +import httpx import pytest -import requests from dclimate_client_py.stac_catalog import ( load_stac_catalog, @@ -42,25 +41,33 @@ list_available_datasets_from_stac_server, STAC_SERVER_URL, ) +from tests.ipfs_config import IPFS_GATEWAY_URL, STAC_CATALOG_URL -pytestmark = pytest.mark.integration +pytestmark = [ + pytest.mark.integration, + pytest.mark.ipfs, + pytest.mark.stac_pointer, +] STAC_URL = os.environ.get("STAC_SERVER_URL", STAC_SERVER_URL) -PUBLIC_IPFS_GATEWAY = os.environ.get( - "DCLIMATE_IPFS_GATEWAY", "https://ipfs-gateway.dclimate.net" -) +PUBLIC_IPFS_GATEWAY = IPFS_GATEWAY_URL def _probe(url: str, *, post: bool = False, timeout: float = 10.0) -> bool: try: if post: - resp = requests.post(url, json={"limit": 1}, timeout=timeout) + resp = httpx.post( + url, + json={"limit": 1}, + timeout=timeout, + follow_redirects=True, + ) else: - resp = requests.get(url, timeout=timeout) - return resp.ok - except (requests.ConnectionError, requests.Timeout, requests.RequestException): + resp = httpx.get(url, timeout=timeout, follow_redirects=True) + return resp.is_success + except httpx.HTTPError: return False @@ -73,10 +80,13 @@ def stac_catalog() -> Dict[str, Dict[str, Any]]: @pytest.fixture(scope="module") def ipfs_catalog() -> Dict[str, Dict[str, Any]]: - if not _probe(f"{PUBLIC_IPFS_GATEWAY}/stac"): - pytest.skip(f"IPFS gateway unreachable at {PUBLIC_IPFS_GATEWAY}") + if not _probe(STAC_CATALOG_URL): + pytest.skip(f"STAC catalog pointer unreachable at {STAC_CATALOG_URL}") try: - catalog = load_stac_catalog(gateway_url=PUBLIC_IPFS_GATEWAY) + catalog = load_stac_catalog( + gateway_url=PUBLIC_IPFS_GATEWAY, + catalog_url=STAC_CATALOG_URL, + ) except Exception as exc: # noqa: BLE001 — surface any pystac/network error as a skip pytest.skip(f"IPFS catalog load failed: {exc}") return list_available_datasets(catalog) diff --git a/tests/test_pytest_infra.py b/tests/test_pytest_infra.py new file mode 100644 index 0000000..5544cd3 --- /dev/null +++ b/tests/test_pytest_infra.py @@ -0,0 +1,175 @@ +"""Regression tests for the pytest configuration and test-suite boundaries.""" + +from __future__ import annotations + +import ast +import os +from pathlib import Path +import re +import subprocess + +import pytest + +import tests.conftest as suite_conftest + + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +META_TEST_ENV = "DCLIMATE_META_TEST" +SUMMARY_COUNT = re.compile(r"(?P\d+) (?Ppassed|skipped)\b") + +pytestmark = pytest.mark.skipif( + os.environ.get(META_TEST_ENV) == "1", + reason="pytest infrastructure meta-tests do not run in child pytest processes", +) + + +def _run_pytest(*args: str) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + env["IPFS_GATEWAY_URI_STEM"] = "http://127.0.0.1:9" + env[META_TEST_ENV] = "1" + return subprocess.run( + ["uv", "run", "pytest", *args, "-p", "no:cacheprovider"], + cwd=PROJECT_ROOT, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=120, + check=False, + ) + + +def _summary_counts(output: str) -> tuple[int, int]: + summary_line = next( + (line for line in reversed(output.splitlines()) if SUMMARY_COUNT.search(line)), + None, + ) + assert summary_line is not None, f"pytest summary line not found:\n{output}" + + counts = { + match.group("outcome"): int(match.group("count")) + for match in SUMMARY_COUNT.finditer(summary_line) + } + return counts.get("passed", 0), counts.get("skipped", 0) + + +def test_offline_unit_tests_run_without_ipfs_gateway(): + result = _run_pytest( + "tests/test_zarr_metadata.py", + "tests/test_siren.py", + "-q", + ) + assert result.returncode == 0, result.stdout + passed, skipped = _summary_counts(result.stdout) + + assert passed > 0 and skipped == 0, ( + "offline unit tests must run without an IPFS gateway; " + f"observed passed={passed}, skipped={skipped}\n{result.stdout}" + ) + + +def test_unmarked_async_tests_execute(tmp_path): + # An async test WITHOUT @pytest.mark.asyncio only executes when the suite + # configures pytest-asyncio (asyncio_mode = "auto"); otherwise it is + # skipped with an "async def functions are not natively supported" warning. + unmarked_async_test = tmp_path / "test_meta_unmarked_async.py" + unmarked_async_test.write_text( + "async def test_unmarked_async_executes():\n assert True\n", + encoding="utf-8", + ) + # -c points the child run at the repo config; without it the temp file's + # rootdir has no pyproject.toml and asyncio_mode would never apply. + result = _run_pytest( + str(unmarked_async_test), + "-c", + str(PROJECT_ROOT / "pyproject.toml"), + "-q", + ) + assert result.returncode == 0, result.stdout + + unsupported_message = "async def functions are not natively supported" + assert unsupported_message not in result.stdout, result.stdout + + passed, skipped = _summary_counts(result.stdout) + assert passed == 1 and skipped == 0, ( + f"unmarked async test was not executed:\n{result.stdout}" + ) + + +def test_root_collection_is_confined_to_tests_directory(): + result = _run_pytest("--collect-only", "-q") + assert result.returncode == 0, result.stdout + + nodeids = [line.strip() for line in result.stdout.splitlines() if "::" in line] + stray_nodeids = [ + nodeid + for nodeid in nodeids + if nodeid.startswith(("test_stac_integration.py", "examples/")) + ] + assert not stray_nodeids, f"pytest collected tests outside tests/: {stray_nodeids}" + + +def test_debug_tests_contain_a_real_assertion(): + debug_test = PROJECT_ROOT / "tests/test_debug.py" + if not debug_test.exists(): + return + + tree = ast.parse(debug_test.read_text(encoding="utf-8"), filename=str(debug_test)) + assertions = [node for node in ast.walk(tree) if isinstance(node, ast.Assert)] + assert assertions, "tests/test_debug.py contains no assert statements" + + +@pytest.mark.parametrize( + ("status", "payload", "expected"), + [ + (200, {"ID": "12D3KooWTest"}, True), + (200, {}, False), + (404, {}, False), + (200, ValueError("invalid JSON"), False), + ], +) +def test_ipfs_rpc_probe_requires_successful_kubo_identity( + monkeypatch, status, payload, expected +): + class Response: + def raise_for_status(self): + if status >= 400: + raise suite_conftest.httpx.HTTPError(f"HTTP {status}") + + def json(self): + if isinstance(payload, Exception): + raise payload + return payload + + monkeypatch.setattr( + suite_conftest.httpx, "post", lambda *args, **kwargs: Response() + ) + + assert suite_conftest.is_ipfs_rpc_running("https://rpc.example") is expected + + +@pytest.mark.parametrize( + ("status", "payload", "expected"), + [ + (200, {"cid": "bafy-root"}, True), + (200, {}, False), + (404, {}, False), + (200, ValueError("invalid JSON"), False), + ], +) +def test_stac_pointer_probe_requires_successful_root_cid( + monkeypatch, status, payload, expected +): + class Response: + def raise_for_status(self): + if status >= 400: + raise suite_conftest.httpx.HTTPError(f"HTTP {status}") + + def json(self): + if isinstance(payload, Exception): + raise payload + return payload + + monkeypatch.setattr(suite_conftest.httpx, "get", lambda *args, **kwargs: Response()) + + assert suite_conftest.is_stac_pointer_running("https://catalog.example") is expected diff --git a/tests/test_review_bugs_client.py b/tests/test_review_bugs_client.py new file mode 100644 index 0000000..e553a7a --- /dev/null +++ b/tests/test_review_bugs_client.py @@ -0,0 +1,123 @@ +import asyncio +from unittest.mock import AsyncMock + +import pytest + +from dclimate_client_py.dclimate_client import dClimateClient + + +def _client_with_mocks( + siren_error: BaseException | None = None, + kubo_error: BaseException | None = None, +) -> tuple[dClimateClient, AsyncMock, AsyncMock]: + client = dClimateClient() + kubo_cas = AsyncMock() + siren_client = AsyncMock() + if siren_error is not None: + siren_client.aclose.side_effect = siren_error + if kubo_error is not None: + kubo_cas.__aexit__.side_effect = kubo_error + client._kubo_cas = kubo_cas + client._siren_client = siren_client + return client, kubo_cas, siren_client + + +@pytest.mark.asyncio +async def test_aexit_forwards_with_block_exception_to_kubo(): + client, kubo_cas, _ = _client_with_mocks() + exc = ValueError("boom") + + await client.__aexit__(ValueError, exc, None) + + kubo_cas.__aexit__.assert_awaited_once_with(ValueError, exc, None) + assert client._kubo_cas is None + + +@pytest.mark.asyncio +async def test_aexit_dual_failure_propagates_later_error_with_context(): + siren_error = RuntimeError("Siren close failed") + kubo_error = ValueError("Kubo close failed") + client, _, _ = _client_with_mocks(siren_error=siren_error, kubo_error=kubo_error) + + with pytest.raises(ValueError, match="Kubo close failed") as excinfo: + await client.__aexit__(None, None, None) + + assert excinfo.value.__context__ is siren_error + assert client._kubo_cas is None + + +@pytest.mark.asyncio +async def test_aexit_cancellation_outranks_ordinary_error(): + # Cancellation from either cleanup must propagate, never be demoted + # to the __cause__/__context__ of an ordinary error. + client, _, _ = _client_with_mocks( + siren_error=RuntimeError("Siren close failed"), + kubo_error=asyncio.CancelledError(), + ) + with pytest.raises(asyncio.CancelledError): + await client.__aexit__(None, None, None) + assert client._kubo_cas is None + + client, _, _ = _client_with_mocks( + siren_error=asyncio.CancelledError(), + kubo_error=RuntimeError("Kubo close failed"), + ) + with pytest.raises(asyncio.CancelledError): + await client.__aexit__(None, None, None) + assert client._kubo_cas is None + + +@pytest.mark.asyncio +async def test_aexit_closes_kubo_when_siren_close_raises(): + client = dClimateClient() + kubo_cas = AsyncMock() + siren_client = AsyncMock() + siren_client.aclose.side_effect = RuntimeError("Siren close failed") + client._kubo_cas = kubo_cas + client._siren_client = siren_client + + with pytest.raises(RuntimeError, match="Siren close failed"): + await client.__aexit__(None, None, None) + + kubo_cas.__aexit__.assert_awaited_once_with(None, None, None) + assert client._kubo_cas is None + + +@pytest.mark.asyncio +async def test_aexit_body_cancellation_outranks_ordinary_cleanup_error(): + siren_error = RuntimeError("Siren close failed") + client, kubo_cas, siren_client = _client_with_mocks(siren_error=siren_error) + + # __aenter__ replaces the mock, so exercise __aexit__ through a minimal + # context wrapper that keeps the prepared cleanup clients. + class Context: + async def __aenter__(self): + return client + + async def __aexit__(self, exc_type, exc_val, exc_tb): + return await client.__aexit__(exc_type, exc_val, exc_tb) + + async def run_context(): + async with Context(): + raise asyncio.CancelledError() + + with pytest.raises(asyncio.CancelledError) as excinfo: + await run_context() + + assert excinfo.value.__context__ is siren_error + siren_client.aclose.assert_awaited_once() + kubo_cas.__aexit__.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_aexit_body_cancellation_keeps_all_cleanup_failures_as_context(): + siren_error = RuntimeError("Siren close failed") + kubo_error = RuntimeError("Kubo close failed") + client, _, _ = _client_with_mocks(siren_error=siren_error, kubo_error=kubo_error) + incoming = asyncio.CancelledError() + + suppress = await client.__aexit__(asyncio.CancelledError, incoming, None) + + assert suppress is False + assert incoming.__context__ is kubo_error + assert kubo_error.__context__ is siren_error diff --git a/tests/test_review_bugs_geotemporal.py b/tests/test_review_bugs_geotemporal.py new file mode 100644 index 0000000..0a8aef3 --- /dev/null +++ b/tests/test_review_bugs_geotemporal.py @@ -0,0 +1,154 @@ +import subprocess +import textwrap +from pathlib import Path + +import geopandas as gpd +import numpy as np +import xarray as xr +from shapely.geometry import box + +from dclimate_client_py.geotemporal_data import GeotemporalData + + +def _polygon_dataset(): + values = np.arange(8, dtype=float).reshape(2, 2, 2) + return xr.Dataset( + {"temperature": (("time", "latitude", "longitude"), values)}, + coords={ + "time": np.array(["2024-01-01", "2024-01-02"], dtype="datetime64[ns]"), + "latitude": [0.0, 1.0], + "longitude": [0.0, 1.0], + }, + attrs={"spatial resolution": 0.5}, + ) + + +def _polygon_mask(): + return gpd.GeoSeries([box(-0.4, -0.4, 1.4, 1.4)], crs=4326).array + + +def _force_clip_no_data(monkeypatch): + import rioxarray # noqa: F401 -- registers the xarray ``rio`` accessor + from rioxarray.exceptions import NoDataInBounds + from rioxarray.raster_dataset import RasterDataset + + def raise_no_data_in_bounds(self, *args, **kwargs): + raise NoDataInBounds("polygon contains no grid-cell centers") + + monkeypatch.setattr(RasterDataset, "clip", raise_no_data_in_bounds) + return NoDataInBounds + + +def test_polygons_registers_rioxarray_accessor_in_fresh_interpreter(): + script = textwrap.dedent( + """ + import sys + + assert "rioxarray" not in sys.modules + + import geopandas as gpd + import numpy as np + import xarray as xr + from shapely.geometry import box + + from dclimate_client_py.geotemporal_data import GeotemporalData + + dataset = xr.Dataset( + {"temperature": (("time", "latitude", "longitude"), np.ones((1, 2, 2)))}, + coords={ + "time": np.array(["2024-01-01"], dtype="datetime64[ns]"), + "latitude": [0.0, 1.0], + "longitude": [0.0, 1.0], + }, + attrs={"spatial resolution": 0.5}, + ) + mask = gpd.GeoSeries([box(-0.4, -0.4, 1.4, 1.4)], crs=4326).array + + result = GeotemporalData(dataset, "the-dataset-name").polygons(mask) + + assert result.data.sizes["latitude"] == 2 + assert result.data.sizes["longitude"] == 2 + """ + ) + repo_root = Path(__file__).resolve().parents[1] + + completed = subprocess.run( + ["uv", "run", "python", "-c", script], + cwd=repo_root, + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 0, completed.stdout + completed.stderr + + +def test_polygons_handles_rioxarray_no_data_in_bounds(monkeypatch): + _force_clip_no_data(monkeypatch) + geotemporal = GeotemporalData(_polygon_dataset(), "the-dataset-name") + mask = _polygon_mask() + expected = geotemporal.reduce_polygon_to_point(mask) + + result = geotemporal.polygons(mask) + + xr.testing.assert_identical(result.data, expected.data) + + +def test_polygons_does_not_mutate_caller_dataset(): + # The pre-fix implementation attached rio spatial dims / CRS to + # self.data with inplace=True, polluting the caller's dataset. + dataset = _polygon_dataset() + original = dataset.copy(deep=True) + geotemporal = GeotemporalData(dataset, "the-dataset-name") + + geotemporal.polygons(_polygon_mask()) + + assert "spatial_ref" not in dataset.coords + assert dataset.attrs == original.attrs + xr.testing.assert_identical(dataset, original) + + +def test_projected_small_polygon_fallback_uses_wgs84_coordinates(): + dataset = xr.Dataset( + {"temperature": (("latitude", "longitude"), [[1.0, 2.0], [3.0, 4.0]])}, + coords={"latitude": [34.0, 36.0], "longitude": [-121.0, -119.0]}, + attrs={"spatial resolution": 1.0}, + ) + projected_mask = gpd.GeoSeries( + [box(-119.85, 35.15, -119.75, 35.25)], crs=4326 + ).to_crs(3857) + + result = GeotemporalData(dataset, "temperature").polygons( + projected_mask.array, epsg_crs=3857 + ) + + assert float(result.data.longitude) == -119.0 + assert float(result.data.latitude) == 36.0 + + +def test_rolling_aggregation_preserves_time_with_all_nan_spatial_cell(): + times = np.arange( + np.datetime64("2024-01-01"), + np.datetime64("2024-01-07"), + dtype="datetime64[D]", + ) + values = np.broadcast_to( + np.arange(1.0, 7.0)[:, np.newaxis, np.newaxis], (6, 2, 2) + ).copy() + values[:, 0, 0] = np.nan + dataset = xr.Dataset( + {"temperature": (("time", "latitude", "longitude"), values)}, + coords={"time": times, "latitude": [10.0, 11.0], "longitude": [20.0, 21.0]}, + ) + + result = GeotemporalData(dataset, "the-dataset-name").rolling_aggregation( + window_size=3, agg_method="mean" + ) + + assert result.data.sizes["time"] == dataset.sizes["time"] - 2 + np.testing.assert_array_equal(result.data.time.values, dataset.time.values[2:]) + np.testing.assert_allclose( + result.data["temperature"].isel(latitude=1, longitude=1).values, + [2.0, 3.0, 4.0, 5.0], + ) + assert result.data["temperature"].isel(latitude=0, longitude=0).isnull().all() diff --git a/tests/test_review_bugs_geotemporal2.py b/tests/test_review_bugs_geotemporal2.py new file mode 100644 index 0000000..062a7dc --- /dev/null +++ b/tests/test_review_bugs_geotemporal2.py @@ -0,0 +1,87 @@ +import copy +import warnings + +import numpy as np +import pytest +import xarray as xr + +from dclimate_client_py import dclimate_zarr_errors as errors +from dclimate_client_py.geotemporal_data import GeotemporalData + + +def test_check_dataset_size_accepts_scalar_dataset(): + dataset = xr.Dataset({"temperature": xr.DataArray(12.5)}) + data = GeotemporalData(dataset, "static-temperature") + + data.check_dataset_size(point_limit=1) + + +def test_to_netcdf_preserves_original_dataset_attributes(): + original_attrs = { + "bbox": [-180.0, -90.0, 180.0, 90.0], + "date range": ["2020010100", "2020010200"], + "tags": ["temperature", "static"], + "finalization date": None, + "update_date_range": ["2020010100", "2020010200"], + } + dataset = xr.Dataset( + {"temperature": ("location", [12.5])}, attrs=copy.deepcopy(original_attrs) + ) + data = GeotemporalData(dataset, "static-temperature") + + serialized = data.to_netcdf() + + assert isinstance(serialized, bytes) + assert dataset.attrs == original_attrs + + +def test_temporal_aggregation_quarter_uses_supported_alias_without_warning(): + times = np.array( + [ + "2024-01-15", + "2024-02-15", + "2024-04-15", + "2024-06-15", + "2024-07-15", + ], + dtype="datetime64[ns]", + ) + dataset = xr.Dataset( + {"temperature": ("time", [1.0, 3.0, 5.0, 7.0, 9.0])}, + coords={"time": times}, + ) + data = GeotemporalData(dataset, "quarterly-temperature") + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = data.temporal_aggregation(time_period="quarter", agg_method="mean") + + xr.testing.assert_allclose( + result.data["temperature"], + xr.DataArray( + [2.0, 6.0, 9.0], + dims="time", + coords={ + "time": np.array( + ["2024-03-31", "2024-06-30", "2024-09-30"], + dtype="datetime64[ns]", + ) + }, + name="temperature", + ), + ) + deprecated_q_warnings = [ + warning + for warning in caught + if issubclass(warning.category, (FutureWarning, DeprecationWarning)) + and "'Q'" in str(warning.message) + ] + assert deprecated_q_warnings == [] + + +def test_check_dataset_size_counts_scalar_dataset_as_one_point(): + dataset = xr.Dataset({"temperature": xr.DataArray(12.5)}) + data = GeotemporalData(dataset, "static-temperature") + + with pytest.raises(errors.SelectionTooLargeError): + data.check_dataset_size(point_limit=0) diff --git a/tests/test_review_bugs_s3.py b/tests/test_review_bugs_s3.py new file mode 100644 index 0000000..0e5015e --- /dev/null +++ b/tests/test_review_bugs_s3.py @@ -0,0 +1,70 @@ +import pytest +import xarray as xr + +import dclimate_client_py.s3_retrieval as s3_retrieval + + +def test_get_dataset_from_s3_accepts_missing_update_in_progress(monkeypatch): + dataset = xr.Dataset({"temperature": ("time", [1.0, 2.0])}) + + monkeypatch.setattr(s3_retrieval, "get_s3_fs", lambda: object()) + monkeypatch.setattr(s3_retrieval, "S3Map", lambda *args, **kwargs: object()) + monkeypatch.setattr(s3_retrieval.xr, "open_zarr", lambda *args, **kwargs: dataset) + + result = s3_retrieval.get_dataset_from_s3("temperature", "test-bucket") + + assert result is dataset + + +def _patch_s3(monkeypatch, dataset): + monkeypatch.setattr(s3_retrieval, "get_s3_fs", lambda: object()) + monkeypatch.setattr(s3_retrieval, "S3Map", lambda *args, **kwargs: object()) + monkeypatch.setattr(s3_retrieval.xr, "open_zarr", lambda *args, **kwargs: dataset) + + +def test_get_dataset_from_s3_initial_parse_raises(monkeypatch): + dataset = xr.Dataset( + {"temperature": ("time", [1.0, 2.0])}, + attrs={"update_in_progress": True, "initial_parse": True}, + ) + _patch_s3(monkeypatch, dataset) + + with pytest.raises(s3_retrieval.DatasetNotFoundError, match="initial parse"): + s3_retrieval.get_dataset_from_s3("temperature", "test-bucket") + + +def test_get_dataset_from_s3_append_only_update_slices_to_previous_end(monkeypatch): + times = xr.date_range("2020-01-01", periods=4, freq="h") + dataset = xr.Dataset( + {"temperature": ("time", [1.0, 2.0, 3.0, 4.0])}, + coords={"time": times}, + attrs={ + "update_in_progress": True, + "update_is_append_only": True, + "date range": ["2020010100", "2020010103"], + "update_previous_end_date": "2020010102", + }, + ) + _patch_s3(monkeypatch, dataset) + + result = s3_retrieval.get_dataset_from_s3("temperature", "test-bucket") + + assert result.sizes["time"] == 3 + + +def test_get_dataset_from_s3_full_update_uses_date_range(monkeypatch): + times = xr.date_range("2020-01-01", periods=4, freq="h") + dataset = xr.Dataset( + {"temperature": ("time", [1.0, 2.0, 3.0, 4.0])}, + coords={"time": times}, + attrs={ + "update_in_progress": True, + "update_is_append_only": False, + "date range": ["2020010100", "2020010101"], + }, + ) + _patch_s3(monkeypatch, dataset) + + result = s3_retrieval.get_dataset_from_s3("temperature", "test-bucket") + + assert result.sizes["time"] == 2 diff --git a/tests/test_review_bugs_stac_server.py b/tests/test_review_bugs_stac_server.py new file mode 100644 index 0000000..844a6e0 --- /dev/null +++ b/tests/test_review_bugs_stac_server.py @@ -0,0 +1,129 @@ +import json + +import httpx +import pytest +import dclimate_client_py.stac_server as stac_server + + +_install_mock_client = None + + +@pytest.fixture(autouse=True) +def _use_managed_httpx_clients(install_httpx_mock): + global _install_mock_client + _install_mock_client = install_httpx_mock + yield + _install_mock_client = None + + +def _mock_search(monkeypatch, features): + def handler(request: httpx.Request) -> httpx.Response: + assert request.url == "https://stac.example/search" + assert json.loads(request.content) == { + "limit": 100, + "collections": ["ecmwf_era5"], + } + assert set(request.extensions["timeout"].values()) == {10} + return httpx.Response(200, json={"features": features}, request=request) + + assert _install_mock_client is not None + _install_mock_client(stac_server, handler) + + +def test_resolve_variant_falls_back_to_variant_encoded_in_item_id(monkeypatch): + _mock_search( + monkeypatch, + [ + { + "id": "ecmwf_era5-temperature-finalized", + "collection": "ecmwf_era5", + "properties": {"dclimate:dataset_id": "temperature"}, + "assets": {"data": {"href": "ipfs://bafy-temperature-finalized"}}, + } + ], + ) + + resolved = stac_server.resolve_cid_from_stac_server( + "ecmwf_era5", + "temperature", + variant="finalized", + server_url="https://stac.example", + ) + + assert resolved.cid == "bafy-temperature-finalized" + assert resolved.variant == "finalized" + + +def test_resolve_feature_without_properties(monkeypatch): + _mock_search( + monkeypatch, + [ + { + "id": "ecmwf_era5-temperature", + "collection": "ecmwf_era5", + "assets": {"data": {"href": "ipfs://bafy-temperature"}}, + } + ], + ) + + resolved = stac_server.resolve_cid_from_stac_server( + "ecmwf_era5", + "temperature", + server_url="https://stac.example", + ) + + assert resolved.cid == "bafy-temperature" + assert resolved.variant == "default" + + +def test_resolve_default_variant_matches_bare_item_id(monkeypatch): + # list_available_datasets_from_stac_server reports items without a + # variant segment as variant "default"; resolve must accept the same + # name so a list -> resolve round-trip works. + _mock_search( + monkeypatch, + [ + { + "id": "ecmwf_era5-temperature", + "collection": "ecmwf_era5", + "assets": {"data": {"href": "ipfs://bafy-temperature"}}, + } + ], + ) + + resolved = stac_server.resolve_cid_from_stac_server( + "ecmwf_era5", + "temperature", + variant="default", + server_url="https://stac.example", + ) + + assert resolved.cid == "bafy-temperature" + assert resolved.variant == "default" + + +def test_resolve_without_variant_prefers_unnamed_item_over_latest(monkeypatch): + _mock_search( + monkeypatch, + [ + { + "id": "ecmwf_era5-temperature-latest", + "collection": "ecmwf_era5", + "assets": {"data": {"href": "ipfs://bafy-temperature-latest"}}, + }, + { + "id": "ecmwf_era5-temperature", + "collection": "ecmwf_era5", + "assets": {"data": {"href": "ipfs://bafy-temperature"}}, + }, + ], + ) + + resolved = stac_server.resolve_cid_from_stac_server( + "ecmwf_era5", + "temperature", + server_url="https://stac.example", + ) + + assert resolved.cid == "bafy-temperature" + assert resolved.variant == "default" diff --git a/tests/test_review_bugs_tolerance.py b/tests/test_review_bugs_tolerance.py new file mode 100644 index 0000000..4d089b6 --- /dev/null +++ b/tests/test_review_bugs_tolerance.py @@ -0,0 +1,38 @@ +import geopandas as gpd +import pytest +from shapely.geometry import Point + +from dclimate_client_py import dclimate_zarr_errors as errors +from dclimate_client_py.geotemporal_data import GeotemporalData + + +def test_point_without_snap_uses_one_e_minus_five_tolerance(dataset): + data = GeotemporalData(dataset, dataset_name="tolerance test") + latitude = float(dataset.latitude.values[0]) + longitude = float(dataset.longitude.values[0]) + + with pytest.raises(errors.NoDataFoundError): + data.point(latitude + 5e-5, longitude, snap_to_grid=False) + + selected = data.point(latitude + 5e-6, longitude, snap_to_grid=False) + assert selected.data.latitude.item() == latitude + assert selected.data.longitude.item() == longitude + + +def test_points_without_snap_uses_one_e_minus_five_tolerance(dataset): + data = GeotemporalData(dataset, dataset_name="tolerance test") + latitude = float(dataset.latitude.values[0]) + longitude = float(dataset.longitude.values[0]) + + off_grid_mask = gpd.GeoSeries( + [Point(longitude, latitude + 5e-5)], crs=4326 + ).geometry.values + with pytest.raises(errors.NoDataFoundError): + data.points(off_grid_mask, epsg_crs=4326, snap_to_grid=False) + + within_tolerance_mask = gpd.GeoSeries( + [Point(longitude, latitude + 5e-6)], crs=4326 + ).geometry.values + selected = data.points(within_tolerance_mask, epsg_crs=4326, snap_to_grid=False) + assert selected.data.latitude.item() == latitude + assert selected.data.longitude.item() == longitude diff --git a/tests/test_review_fu_gateway_none.py b/tests/test_review_fu_gateway_none.py new file mode 100644 index 0000000..4685614 --- /dev/null +++ b/tests/test_review_fu_gateway_none.py @@ -0,0 +1,65 @@ +"""Regression coverage for catalog fallback with a defaulted IPFS gateway.""" + +from __future__ import annotations + +from typing import Any + +import pystac + +from dclimate_client_py import stac_catalog +from dclimate_client_py.dclimate_client import dClimateClient + + +DEFAULT_PUBLIC_GATEWAY = "https://ipfs-gateway.dclimate.net" + + +async def test_alist_datasets_uses_public_gateway_when_gateway_is_none( + monkeypatch, +) -> None: + catalog = pystac.Catalog(id="root", description="Minimal root catalog") + loaded_hrefs: list[str] = [] + constructed_gateways: list[Any] = [] + + monkeypatch.setattr( + stac_catalog, + "get_root_catalog_cid", + lambda catalog_url=stac_catalog.STAC_CATALOG_URL, **_: "bafy-review-root", + ) + + def fake_from_file( + cls: type[pystac.Catalog], + href: str, + stac_io: pystac.StacIO | None = None, + ) -> pystac.Catalog: + loaded_hrefs.append(href) + return catalog + + monkeypatch.setattr( + pystac.Catalog, + "from_file", + classmethod(fake_from_file), + ) + + original_init = stac_catalog.IPFSStacIO.__init__ + + def recording_init( + self: stac_catalog.IPFSStacIO, + gateway_url: str, + **kwargs: Any, + ) -> None: + constructed_gateways.append(gateway_url) + original_init(self, gateway_url, **kwargs) + + monkeypatch.setattr(stac_catalog.IPFSStacIO, "__init__", recording_init) + + client = dClimateClient(gateway_base_url=None, stac_server_url=None) + client._kubo_cas = object() + previous_default_io = pystac.StacIO._default_io + try: + datasets = await client.alist_datasets() + finally: + pystac.StacIO.set_default(previous_default_io) + + assert datasets == {} + assert loaded_hrefs == ["ipfs://bafy-review-root"] + assert constructed_gateways == [DEFAULT_PUBLIC_GATEWAY] diff --git a/tests/test_review_fu_httpx.py b/tests/test_review_fu_httpx.py new file mode 100644 index 0000000..5937336 --- /dev/null +++ b/tests/test_review_fu_httpx.py @@ -0,0 +1,150 @@ +"""Migration pins for consolidating package HTTP on httpx.""" + +from __future__ import annotations + +import ast +import re +import tomllib +from pathlib import Path +from typing import Any + +import httpx +import pystac +import xarray as xr + +import dclimate_client_py.dclimate_client as dclimate_client_module +from dclimate_client_py import stac_catalog +from dclimate_client_py.dclimate_client import dClimateClient +from dclimate_client_py.stac_catalog import IPFSStacIO +from dclimate_client_py.stac_server import ResolvedDataset + + +REPO_ROOT = Path(__file__).resolve().parents[1] +PACKAGE_ROOT = REPO_ROOT / "dclimate_client_py" +FORBIDDEN_HTTP_PACKAGES = {"requests", "urllib3"} + + +def test_package_sources_do_not_import_requests_or_urllib3() -> None: + offenders: list[str] = [] + + for source_path in sorted(PACKAGE_ROOT.rglob("*.py")): + tree = ast.parse(source_path.read_text(), filename=str(source_path)) + imports_forbidden_package = any( + ( + isinstance(node, ast.Import) + and any( + alias.name.split(".", 1)[0] in FORBIDDEN_HTTP_PACKAGES + for alias in node.names + ) + ) + or ( + isinstance(node, ast.ImportFrom) + and node.module is not None + and node.module.split(".", 1)[0] in FORBIDDEN_HTTP_PACKAGES + ) + for node in ast.walk(tree) + ) + if imports_forbidden_package: + offenders.append(str(source_path.relative_to(REPO_ROOT))) + + assert not offenders, ( + "package sources still use legacy HTTP packages:\n" + "\n".join(offenders) + ) + + +def test_project_dependencies_do_not_include_requests_or_urllib3() -> None: + with (REPO_ROOT / "pyproject.toml").open("rb") as pyproject_file: + pyproject = tomllib.load(pyproject_file) + + dependencies = pyproject["project"]["dependencies"] + forbidden_dependencies = [ + dependency + for dependency in dependencies + if re.split(r"[<>=!~;\s\[]", dependency, maxsplit=1)[0] + .lower() + .replace("_", "-") + in FORBIDDEN_HTTP_PACKAGES + ] + + assert not forbidden_dependencies, ( + "[project] dependencies still include legacy HTTP packages: " + + ", ".join(forbidden_dependencies) + ) + + +async def test_load_dataset_falls_back_when_stac_transport_is_unreachable( + monkeypatch, +) -> None: + """Non-regression pin: transport errors must still enter catalog fallback.""" + catalog = pystac.Catalog(id="root", description="Fallback catalog") + loaded_hrefs: list[str] = [] + catalog_resolutions: list[dict[str, Any]] = [] + + monkeypatch.setattr( + stac_catalog, + "get_root_catalog_cid", + lambda catalog_url=stac_catalog.STAC_CATALOG_URL, **_: "bafy-review-root", + ) + + def fake_from_file( + cls: type[pystac.Catalog], + href: str, + stac_io: pystac.StacIO | None = None, + ) -> pystac.Catalog: + loaded_hrefs.append(href) + return catalog + + monkeypatch.setattr( + pystac.Catalog, + "from_file", + classmethod(fake_from_file), + ) + monkeypatch.setattr( + stac_catalog, + "list_available_datasets", + lambda loaded_catalog: {"review_collection": {"types": ["temperature"]}}, + ) + + def resolve_from_catalog(**kwargs: Any) -> ResolvedDataset: + catalog_resolutions.append(kwargs) + return ResolvedDataset("bafy-fallback-dataset", "default") + + monkeypatch.setattr( + stac_catalog, + "resolve_dataset_cid_from_stac", + resolve_from_catalog, + ) + + async def fake_load_dataset(**kwargs: Any) -> xr.Dataset: + return xr.Dataset({"temperature": ("x", [1.0])}) + + monkeypatch.setattr( + dclimate_client_module, + "_load_dataset_from_ipfs_cid", + fake_load_dataset, + ) + + client = dClimateClient( + gateway_base_url="https://gateway.example", + stac_server_url="http://127.0.0.1:9", + ) + client._kubo_cas = object() + previous_default_io = pystac.StacIO._default_io + try: + _, metadata = await client.load_dataset( + dataset="temperature", + collection="review_collection", + return_xarray=True, + ) + finally: + pystac.StacIO.set_default(previous_default_io) + + assert loaded_hrefs == ["ipfs://bafy-review-root"] + assert len(catalog_resolutions) == 1 + assert metadata["cid"] == "bafy-fallback-dataset" + + +def test_ipfs_stac_io_owns_httpx_client() -> None: + stac_io = IPFSStacIO("https://gateway.example") + + assert isinstance(getattr(stac_io, "client", None), httpx.Client) diff --git a/tests/test_review_fu_kubo_knobs.py b/tests/test_review_fu_kubo_knobs.py new file mode 100644 index 0000000..19981ab --- /dev/null +++ b/tests/test_review_fu_kubo_knobs.py @@ -0,0 +1,162 @@ +"""Regression tests for exposing KuboCAS connection configuration.""" + +from __future__ import annotations + +import importlib +import inspect +from pathlib import Path +import subprocess +from typing import Any + +import pytest + +from dclimate_client_py.dclimate_client import dClimateClient + + +dclimate_client_module = importlib.import_module("dclimate_client_py.dclimate_client") +PROJECT_ROOT = Path(__file__).resolve().parents[1] +KUBO_OPTION_NAMES = ( + "concurrency", + "headers", + "auth", + "max_retries", + "initial_delay", + "backoff_factor", + "client_factory", +) + + +def _install_recording_kubo( + monkeypatch: pytest.MonkeyPatch, +) -> list[dict[str, Any]]: + recorded_calls: list[dict[str, Any]] = [] + + class RecordingKuboCAS: + def __init__(self, **kwargs: Any) -> None: + recorded_calls.append(kwargs) + + async def __aenter__(self) -> "RecordingKuboCAS": + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: + return None + + monkeypatch.setattr(dclimate_client_module, "KuboCAS", RecordingKuboCAS) + return recorded_calls + + +async def test_kubo_connection_options_are_forwarded_exactly( + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorded_calls = _install_recording_kubo(monkeypatch) + + async with dClimateClient( + concurrency=64, + headers={"Authorization": "Bearer x"}, + auth=("user", "pass"), + max_retries=5, + initial_delay=0.5, + backoff_factor=3.0, + ): + pass + + assert recorded_calls == [ + { + "gateway_base_url": "https://ipfs-gateway.dclimate.net", + "rpc_base_url": "https://ipfs-gateway.dclimate.net", + "concurrency": 64, + "headers": {"Authorization": "Bearer x"}, + "auth": ("user", "pass"), + "max_retries": 5, + "initial_delay": 0.5, + "backoff_factor": 3.0, + } + ] + + +async def test_default_construction_preserves_kubo_defaults( + monkeypatch: pytest.MonkeyPatch, +) -> None: + signature = inspect.signature(dClimateClient.__init__) + missing_options = set(KUBO_OPTION_NAMES) - set(signature.parameters) + assert not missing_options, f"missing Kubo options: {sorted(missing_options)}" + for option_name in KUBO_OPTION_NAMES: + assert signature.parameters[option_name].default is None + + recorded_calls = _install_recording_kubo(monkeypatch) + async with dClimateClient(): + pass + + assert recorded_calls == [ + { + "gateway_base_url": "https://ipfs-gateway.dclimate.net", + "rpc_base_url": "https://ipfs-gateway.dclimate.net", + } + ] + + +async def test_client_factory_is_forwarded_untouched( + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorded_calls = _install_recording_kubo(monkeypatch) + + def client_factory() -> object: + return object() + + async with dClimateClient(client_factory=client_factory): + pass + + assert recorded_calls == [ + { + "gateway_base_url": "https://ipfs-gateway.dclimate.net", + "rpc_base_url": "https://ipfs-gateway.dclimate.net", + "client_factory": client_factory, + } + ] + + +@pytest.mark.parametrize( + "conflicting_option", + [ + {"headers": {"Authorization": "Bearer x"}}, + {"auth": ("user", "pass")}, + ], +) +def test_client_factory_rejects_headers_and_auth_at_construction( + conflicting_option: dict[str, Any], +) -> None: + def client_factory() -> object: + return object() + + with pytest.raises(ValueError, match="client_factory"): + dClimateClient(client_factory=client_factory, **conflicting_option) + + +@pytest.mark.parametrize( + "standalone_option", + [ + {"client_factory": lambda: object()}, + {"headers": {"Authorization": "Bearer x"}}, + {"auth": ("user", "pass")}, + ], +) +def test_client_factory_headers_and_auth_are_valid_alone( + standalone_option: dict[str, Any], +) -> None: + dClimateClient(**standalone_option) + + +def test_gateway_benchmark_script_exposes_tuning_flags() -> None: + completed = subprocess.run( + ["uv", "run", "python", "scripts/benchmark_gateway.py", "--help"], + cwd=PROJECT_ROOT, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + output = completed.stdout + completed.stderr + assert completed.returncode == 0, output + for flag in ("--http2", "--no-http2", "--concurrency", "--repetitions"): + assert flag in output diff --git a/tests/test_review_fu_kwargs_validation.py b/tests/test_review_fu_kwargs_validation.py new file mode 100644 index 0000000..7969c16 --- /dev/null +++ b/tests/test_review_fu_kwargs_validation.py @@ -0,0 +1,83 @@ +"""Regression tests for query keyword validation at the public boundary.""" + +from __future__ import annotations + +import numpy as np +import pytest +import xarray as xr + +from dclimate_client_py import dclimate_zarr_errors as errors +from dclimate_client_py.geotemporal_data import GeotemporalData + + +@pytest.fixture +def geotemporal_data() -> GeotemporalData: + dataset = xr.Dataset( + { + "temperature": ( + ("latitude", "longitude"), + np.arange(9, dtype=float).reshape(3, 3), + ) + }, + coords={ + "latitude": [39.0, 40.0, 41.0], + "longitude": [-75.0, -74.0, -73.0], + }, + ) + return GeotemporalData(dataset, dataset_name="synthetic") + + +def test_query_rejects_circle_kwargs_missing_lon( + geotemporal_data: GeotemporalData, +) -> None: + with pytest.raises(errors.InvalidSelectionError, match="lon"): + geotemporal_data.query(circle_kwargs={"lat": 40.0}) + + +def test_query_rejects_none_required_circle_value( + geotemporal_data: GeotemporalData, +) -> None: + with pytest.raises(errors.InvalidSelectionError, match="lat"): + geotemporal_data.query( + circle_kwargs={"lat": None, "lon": -74.0, "radius": 10.0} + ) + + +def test_query_rejects_empty_point_kwargs( + geotemporal_data: GeotemporalData, +) -> None: + with pytest.raises(errors.InvalidSelectionError, match="latitude"): + geotemporal_data.query(point_kwargs={}) + + +def test_query_rejects_rectangle_kwargs_missing_max_lon( + geotemporal_data: GeotemporalData, +) -> None: + with pytest.raises(errors.InvalidSelectionError, match="max_lon"): + geotemporal_data.query( + rectangle_kwargs={ + "min_lat": 39.0, + "min_lon": -75.0, + "max_lat": 41.0, + } + ) + + +@pytest.mark.parametrize("keyword", ["polygon_kwargs", "multiple_points_kwargs"]) +def test_query_rejects_empty_selection_kwargs( + geotemporal_data: GeotemporalData, + keyword: str, +) -> None: + with pytest.raises(errors.InvalidSelectionError, match=keyword): + geotemporal_data.query(**{keyword: {}}) + + +def test_query_accepts_complete_circle_kwargs( + geotemporal_data: GeotemporalData, +) -> None: + selected = geotemporal_data.query( + circle_kwargs={"lat": 40.0, "lon": -74.0, "radius": 10.0} + ) + + assert selected.data.sizes == {"latitude": 1, "longitude": 1} + assert selected.data["temperature"].item() == 4.0 diff --git a/tests/test_review_fu_typing.py b/tests/test_review_fu_typing.py new file mode 100644 index 0000000..2b90119 --- /dev/null +++ b/tests/test_review_fu_typing.py @@ -0,0 +1,88 @@ +"""Regression tests for the package's PEP 561 typing contract.""" + +from __future__ import annotations + +import os +from pathlib import Path +import re +import subprocess +import zipfile + +import pytest + + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +TYPECHECK_META_ENV = "DCLIMATE_TYPECHECK_META_TEST" + +pytestmark = pytest.mark.skipif( + os.environ.get(TYPECHECK_META_ENV) == "1", + reason="typing meta-tests do not run recursively", +) + + +def test_package_passes_mypy() -> None: + env = os.environ.copy() + env[TYPECHECK_META_ENV] = "1" + env["UV_CACHE_DIR"] = str(PROJECT_ROOT / ".uv-cache") + result = subprocess.run( + [ + "uv", + "run", + "mypy", + "dclimate_client_py", + ], + cwd=PROJECT_ROOT, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=180, + check=False, + ) + + assert result.returncode == 0, ( + f"mypy exited with status {result.returncode}:\n{result.stdout}" + ) + source_count_match = re.search( + r"no issues found in (\d+) source files", result.stdout, re.IGNORECASE + ) + assert source_count_match is not None, ( + f"mypy did not report its checked source-file count:\n{result.stdout}" + ) + assert int(source_count_match.group(1)) >= 16, ( + "mypy checked fewer than the expected 16 package source files:\n" + f"{result.stdout}" + ) + + +def test_package_has_pep561_marker() -> None: + marker = PROJECT_ROOT / "dclimate_client_py" / "py.typed" + + assert marker.is_file(), ( + "dclimate_client_py/py.typed must exist so the package ships its type " + "information in built distributions" + ) + + +def test_built_wheel_contains_pep561_marker(tmp_path: Path) -> None: + env = os.environ.copy() + env[TYPECHECK_META_ENV] = "1" + env["UV_CACHE_DIR"] = str(PROJECT_ROOT / ".uv-cache") + result = subprocess.run( + ["uv", "build", "--wheel", "--out-dir", str(tmp_path)], + cwd=PROJECT_ROOT, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=300, + check=False, + ) + + assert result.returncode == 0, ( + f"uv build exited with status {result.returncode}:\n{result.stdout}" + ) + wheels = list(tmp_path.glob("*.whl")) + assert len(wheels) == 1, f"expected one wheel, found {wheels!r}" + with zipfile.ZipFile(wheels[0]) as wheel: + assert "dclimate_client_py/py.typed" in wheel.namelist() diff --git a/tests/test_review_fu_variant.py b/tests/test_review_fu_variant.py new file mode 100644 index 0000000..761405d --- /dev/null +++ b/tests/test_review_fu_variant.py @@ -0,0 +1,392 @@ +from __future__ import annotations + +from datetime import datetime, timezone +import json +from typing import Any + +import httpx +import pystac +import pytest +import xarray as xr + +import dclimate_client_py.dclimate_client as dclimate_client_module +from dclimate_client_py import stac_catalog, stac_server +from dclimate_client_py.dclimate_client import dClimateClient + + +COLLECTION = "example_collection" +DATASET = "temperature" + + +_install_mock_client = None + + +@pytest.fixture(autouse=True) +def _use_managed_httpx_clients(install_httpx_mock): + global _install_mock_client + _install_mock_client = install_httpx_mock + yield + _install_mock_client = None + + +class _Response: + def __init__(self, payload: dict[str, Any]) -> None: + self._payload = payload + + def json(self) -> dict[str, Any]: + return self._payload + + def raise_for_status(self) -> None: + return None + + +def _install_post(monkeypatch, post) -> None: + def handler(request: httpx.Request) -> httpx.Response: + response = post( + str(request.url), + json=json.loads(request.content), + timeout=request.extensions["timeout"]["read"], + ) + return httpx.Response(200, json=response._payload, request=request) + + assert _install_mock_client is not None + _install_mock_client(stac_server, handler) + + +def _feature(variant: str, cid: str) -> dict[str, Any]: + return { + "id": f"{COLLECTION}-{DATASET}-{variant}", + "collection": COLLECTION, + "properties": { + "dclimate:dataset_id": DATASET, + "dclimate:variant": variant, + }, + "assets": {"data": {"href": f"ipfs://{cid}"}}, + } + + +def _item(variant: str, cid: str) -> pystac.Item: + item = pystac.Item( + id=f"{COLLECTION}-{DATASET}-{variant}", + geometry=None, + bbox=None, + datetime=datetime(2024, 1, 1, tzinfo=timezone.utc), + properties={ + "dclimate:dataset_id": DATASET, + "dclimate:variant": variant, + }, + ) + item.add_asset("data", pystac.Asset(href=f"ipfs://{cid}")) + return item + + +def _catalog_with_items(*items: pystac.Item) -> pystac.Catalog: + root = pystac.Catalog(id="root", description="Root") + organization = pystac.Catalog(id="example", description="Organization") + collection = pystac.Collection( + id=COLLECTION, + description="Example collection", + extent=pystac.Extent( + pystac.SpatialExtent([[-180.0, -90.0, 180.0, 90.0]]), + pystac.TemporalExtent([[items[0].datetime, None]]), + ), + ) + for item in items: + collection.add_item(item) + organization.add_child(collection) + root.add_child(organization) + + root.get_child_links()[0].extra_fields["dclimate:id"] = "example" + organization.get_child_links()[0].extra_fields["dclimate:id"] = COLLECTION + return root + + +async def _stub_dataset_loader(**kwargs: Any) -> xr.Dataset: + return xr.Dataset({"temperature": ("x", [1.0])}) + + +def _cid(result: Any) -> str: + """Accept today's str and the post-fix ResolvedDataset in behavior tests.""" + return getattr(result, "cid", result) + + +async def test_load_dataset_reports_variant_selected_by_stac_server(monkeypatch): + _install_post( + monkeypatch, + lambda *args, **kwargs: _Response( + { + "features": [ + _feature("latest", "bafy-latest"), + _feature("final", "bafy-final"), + ] + } + ), + ) + monkeypatch.setattr( + dclimate_client_module, + "_load_dataset_from_ipfs_cid", + _stub_dataset_loader, + ) + client = dClimateClient(stac_server_url="https://stac.example") + client._kubo_cas = object() + + _, metadata = await client.load_dataset( + dataset=DATASET, + collection=COLLECTION, + variant=None, + return_xarray=True, + ) + + assert metadata["variant"] == "final" + assert metadata["slug"].endswith("/final") + + +async def test_load_dataset_reports_variant_selected_by_stac_catalog(monkeypatch): + monkeypatch.setattr( + dclimate_client_module, + "_load_dataset_from_ipfs_cid", + _stub_dataset_loader, + ) + client = dClimateClient(stac_server_url=None) + client._kubo_cas = object() + client._stac_catalog = _catalog_with_items( + _item("latest", "bafy-latest"), + _item("final", "bafy-final"), + ) + + _, metadata = await client.load_dataset( + dataset=DATASET, + collection=COLLECTION, + organization="example", + variant=None, + return_xarray=True, + ) + + assert metadata["variant"] == "final" + assert metadata["slug"].endswith("/final") + + +async def test_direct_cid_without_variant_uses_unknown_consistently(monkeypatch): + monkeypatch.setattr( + dclimate_client_module, + "_load_dataset_from_ipfs_cid", + _stub_dataset_loader, + ) + client = dClimateClient(stac_server_url=None) + client._kubo_cas = object() + + _, metadata = await client.load_dataset( + dataset=DATASET, + collection=COLLECTION, + cid="bafy-direct", + variant=None, + return_xarray=True, + ) + + assert metadata["variant"] == "unknown" + assert metadata["slug"].endswith("/unknown") + + +async def test_explicit_variant_is_preserved_in_loaded_metadata(monkeypatch): + _install_post( + monkeypatch, + lambda *args, **kwargs: _Response( + {"features": [_feature("latest", "bafy-latest")]} + ), + ) + monkeypatch.setattr( + dclimate_client_module, + "_load_dataset_from_ipfs_cid", + _stub_dataset_loader, + ) + client = dClimateClient(stac_server_url="https://stac.example") + client._kubo_cas = object() + + _, metadata = await client.load_dataset( + dataset=DATASET, + collection=COLLECTION, + variant="latest", + return_xarray=True, + ) + + assert metadata["variant"] == "latest" + assert metadata["slug"].endswith("/latest") + + +def test_no_variant_search_paginates_to_preferred_default(monkeypatch): + first_page_with_default = { + "features": [ + { + "id": f"{COLLECTION}-{DATASET}", + "collection": COLLECTION, + "properties": {"dclimate:dataset_id": DATASET}, + "assets": {"data": {"href": "ipfs://bafy-default"}}, + } + ], + "links": [ + { + "rel": "next", + "href": "https://stac.example/search", + "method": "POST", + "body": {"token": "page-2"}, + } + ], + } + default_pages = [first_page_with_default, {"features": []}] + default_page_calls = 0 + + def post_default_page(url, *, json, timeout): + nonlocal default_page_calls + default_page_calls += 1 + return _Response(default_pages[default_page_calls - 1]) + + _install_post(monkeypatch, post_default_page) + page_one_default = stac_server.resolve_cid_from_stac_server( + COLLECTION, + DATASET, + server_url="https://stac.example", + ) + assert _cid(page_one_default) == "bafy-default" + assert page_one_default.variant == "default" + assert default_page_calls == 2 + + pages = [ + { + "features": [_feature("latest", "bafy-latest")], + "links": [ + { + "rel": "next", + "href": "https://stac.example/search", + "method": "POST", + "body": {"token": "page-2"}, + } + ], + }, + { + "features": [ + { + "id": f"{COLLECTION}-{DATASET}", + "collection": COLLECTION, + "properties": {"dclimate:dataset_id": DATASET}, + "assets": {"data": {"href": "ipfs://bafy-default"}}, + } + ] + }, + ] + calls: list[dict[str, Any] | None] = [] + + def post(url, *, json, timeout): + calls.append(json) + return _Response(pages[len(calls) - 1]) + + _install_post(monkeypatch, post) + + resolved = stac_server.resolve_cid_from_stac_server( + COLLECTION, + DATASET, + server_url="https://stac.example", + ) + + assert _cid(resolved) == "bafy-default" + assert resolved.variant == "default" + assert len(calls) == 2 + + +def test_resolvers_return_cid_and_selected_variant(monkeypatch): + _install_post( + monkeypatch, + lambda *args, **kwargs: _Response( + { + "features": [ + _feature("latest", "bafy-latest"), + _feature("final", "bafy-final"), + ] + } + ), + ) + + server_resolved = stac_server.resolve_cid_from_stac_server( + COLLECTION, + DATASET, + server_url="https://stac.example", + ) + catalog = _catalog_with_items( + _item("latest", "bafy-latest"), + _item("final", "bafy-final"), + ) + catalog_resolved = stac_catalog.resolve_dataset_cid_from_stac( + catalog, + collection=COLLECTION, + dataset=DATASET, + organization="example", + ) + + assert all( + isinstance(resolved, stac_server.ResolvedDataset) + for resolved in (server_resolved, catalog_resolved) + ) + assert server_resolved.cid == catalog_resolved.cid == "bafy-final" + assert server_resolved.variant == catalog_resolved.variant == "final" + + +def test_no_variant_prefers_default_over_final(monkeypatch): + # Mutation-survivor pin: swapping "default"/"final" in the preference + # cascade must fail this test. + features = [ + _feature("final", "bafy-final"), + _feature("default", "bafy-default"), + ] + _install_post( + monkeypatch, + lambda *args, **kwargs: _Response({"features": features}), + ) + + resolved = stac_server.resolve_cid_from_stac_server( + COLLECTION, + DATASET, + server_url="https://stac.example", + ) + + assert resolved.cid == "bafy-default" + assert resolved.variant == "default" + + +def test_no_variant_first_match_fallback_reports_actual_variant(monkeypatch): + # No preferred variant exists: the first match wins and its OWN + # effective variant is reported, never a fabricated "default". + features = [ + _feature("raw", "bafy-raw"), + _feature("experimental", "bafy-experimental"), + ] + _install_post( + monkeypatch, + lambda *args, **kwargs: _Response({"features": features}), + ) + + resolved = stac_server.resolve_cid_from_stac_server( + COLLECTION, + DATASET, + server_url="https://stac.example", + ) + + assert resolved.cid == "bafy-raw" + assert resolved.variant == "raw" + + +def test_empty_string_variant_is_treated_as_explicit(monkeypatch): + # Deliberate edge: variant="" is an explicit (unresolvable) variant, + # not "no variant"; it must raise rather than run the cascade. + _install_post( + monkeypatch, + lambda *args, **kwargs: _Response( + {"features": [_feature("latest", "bafy-latest")]} + ), + ) + + with pytest.raises(ValueError, match="Variant ''"): + stac_server.resolve_cid_from_stac_server( + COLLECTION, + DATASET, + variant="", + server_url="https://stac.example", + ) diff --git a/tests/test_review_perf_async.py b/tests/test_review_perf_async.py new file mode 100644 index 0000000..df38ee0 --- /dev/null +++ b/tests/test_review_perf_async.py @@ -0,0 +1,101 @@ +import asyncio +import time +from unittest.mock import AsyncMock + +import httpx +import xarray as xr + +from dclimate_client_py import dclimate_client, stac_catalog, stac_server + + +async def test_load_dataset_does_not_stall_event_loop(monkeypatch, install_httpx_mock): + fake_cid = "bafy-fake-dataset-cid" + + def slow_stac_search(request: httpx.Request) -> httpx.Response: + time.sleep(0.25) + return httpx.Response( + 200, + json={ + "features": [ + { + "id": "example_temperature_default", + "collection": "example", + "properties": { + "dclimate:dataset_id": "temperature", + "dclimate:variant": "default", + }, + "assets": {"data": {"href": f"ipfs://{fake_cid}"}}, + } + ] + }, + request=request, + ) + + async def load_from_ipfs(**kwargs): + assert kwargs["ipfs_cid"] == fake_cid + return xr.Dataset({"temperature": ("time", [21.0])}, coords={"time": [0]}) + + install_httpx_mock(stac_server, slow_stac_search) + monkeypatch.setattr(dclimate_client, "_load_dataset_from_ipfs_cid", load_from_ipfs) + + client = dclimate_client.dClimateClient(stac_server_url="https://stac.invalid") + client._kubo_cas = AsyncMock() + + loop = asyncio.get_running_loop() + heartbeat_times = [loop.time()] + stop_heartbeat = asyncio.Event() + + async def heartbeat(): + while not stop_heartbeat.is_set(): + await asyncio.sleep(0.01) + heartbeat_times.append(loop.time()) + + heartbeat_task = asyncio.create_task(heartbeat()) + await asyncio.sleep(0) + dataset, metadata = await client.load_dataset( + collection="example", + dataset="temperature", + variant="default", + return_xarray=True, + ) + stop_heartbeat.set() + await heartbeat_task + + assert dataset["temperature"].values.tolist() == [21.0] + assert metadata["cid"] == fake_cid + max_tick_gap = max( + later - earlier for earlier, later in zip(heartbeat_times, heartbeat_times[1:]) + ) + assert max_tick_gap < 0.15, f"event loop stalled for {max_tick_gap:.3f}s" + + +def test_ipfs_stac_io_reuses_client(monkeypatch): + get_calls: list[str] = [] + clients: list[httpx.Client] = [] + httpx_client = httpx.Client + + def handler(request: httpx.Request) -> httpx.Response: + get_calls.append(str(request.url)) + return httpx.Response(200, text="{}", request=request) + + def client_factory(*args, **kwargs): + client = httpx_client( + *args, + **kwargs, + transport=httpx.MockTransport(handler), + ) + clients.append(client) + return client + + monkeypatch.setattr(stac_catalog.httpx, "Client", client_factory) + + stac_io = stac_catalog.IPFSStacIO("https://gateway.invalid") + try: + for index in range(5): + assert stac_io.read_text(f"ipfs://fake-cid-{index}") == "{}" + finally: + stac_io.close() + + assert len(clients) == 1 + assert stac_io.client is clients[0] + assert len(get_calls) == 5 diff --git a/tests/test_review_perf_concat.py b/tests/test_review_perf_concat.py new file mode 100644 index 0000000..8f0bcc6 --- /dev/null +++ b/tests/test_review_perf_concat.py @@ -0,0 +1,37 @@ +import importlib + +import numpy as np +import xarray as xr + + +concatenate = importlib.import_module("dclimate_client_py.concatenate") + + +async def test_concatenate_datasets_uses_single_xarray_concat(monkeypatch): + datasets = [] + expected_times = [0, 1, 2] + expected_values = [0, 1, 2] + + for variant in range(8): + start = variant * 2 + times = np.arange(start, start + 3) + values = variant * 100 + np.arange(3) + datasets.append(xr.Dataset({"value": ("time", values)}, coords={"time": times})) + if variant: + expected_times.extend(times[1:].tolist()) + expected_values.extend(values[1:].tolist()) + + real_concat = xr.concat + concat_calls = [] + + def counting_concat(*args, **kwargs): + concat_calls.append((args, kwargs)) + return real_concat(*args, **kwargs) + + monkeypatch.setattr(concatenate.xr, "concat", counting_concat) + + result = await concatenate.concatenate_datasets(datasets, dimension="time") + + assert result["time"].values.tolist() == expected_times + assert result["value"].values.tolist() == expected_values + assert len(concat_calls) == 1, f"xr.concat was called {len(concat_calls)} times" diff --git a/tests/test_review_perf_misc.py b/tests/test_review_perf_misc.py new file mode 100644 index 0000000..608c618 --- /dev/null +++ b/tests/test_review_perf_misc.py @@ -0,0 +1,244 @@ +import asyncio +import subprocess +import textwrap +from pathlib import Path +from types import SimpleNamespace + +import geopandas as gpd +import numpy as np +import pytest +import xarray as xr +from shapely.geometry import Point +from zarr.core.buffer import default_buffer_prototype + +from dclimate_client_py import encryption_codec as encryption_codec_module +from dclimate_client_py.encryption_codec import EncryptionCodec +from dclimate_client_py.geotemporal_data import GeotemporalData + + +def test_points_uses_vectorized_coordinate_access(monkeypatch): + latitudes = np.arange(10, dtype=float) + longitudes = np.arange(20, 30, dtype=float) + expected_values = np.arange(100).reshape(10, 10) + dataset = xr.Dataset( + {"temperature": (("latitude", "longitude"), expected_values)}, + coords={"latitude": latitudes, "longitude": longitudes}, + ) + coordinates = [ + (longitude, latitude) for latitude in latitudes for longitude in longitudes + ] + points_mask = gpd.GeoSeries( + [Point(longitude, latitude) for longitude, latitude in coordinates], + crs=4326, + ).array + + original_y = Point.y + property_accesses = 0 + + def counting_y(point): + nonlocal property_accesses + property_accesses += 1 + return original_y.__get__(point, type(point)) + + monkeypatch.setattr(Point, "y", property(counting_y)) + + selected = GeotemporalData(dataset, "vectorization-test").points( + points_mask, + epsg_crs=4326, + ) + + assert property_accesses == 0 + np.testing.assert_array_equal(selected.data.latitude.values, latitudes.repeat(10)) + np.testing.assert_array_equal( + selected.data.longitude.values, np.tile(longitudes, 10) + ) + np.testing.assert_array_equal( + selected.data["temperature"].values, + expected_values.reshape(-1), + ) + + +def _codec_inputs(payload: bytes): + prototype = default_buffer_prototype() + chunk_spec = SimpleNamespace(prototype=prototype) + chunk_bytes = prototype.buffer.from_bytes(payload) + return chunk_bytes, chunk_spec + + +@pytest.fixture +def configured_encryption_key(): + previous_key = EncryptionCodec._encryption_key + EncryptionCodec.set_encryption_key(b"k" * 32) + yield + EncryptionCodec._encryption_key = previous_key + + +async def _roundtrip(codec: EncryptionCodec, payload: bytes) -> bytes: + chunk_bytes, chunk_spec = _codec_inputs(payload) + encoded = await codec._encode_single(chunk_bytes, chunk_spec) + decoded = await codec._decode_single(encoded, chunk_spec) + return decoded.to_bytes() + + +@pytest.mark.parametrize( + ("size", "fill"), + [(16 * 1024, b"s"), (1024 * 1024, b"L")], + ids=["small", "large"], +) +async def test_encryption_codec_roundtrip(size, fill, configured_encryption_key): + codec = EncryptionCodec(header="offline-roundtrip") + payload = fill * size + + assert await _roundtrip(codec, payload) == payload + + +async def test_encryption_codec_only_offloads_large_chunks( + monkeypatch, configured_encryption_key +): + codec = EncryptionCodec(header="dispatch-test") + real_to_thread = asyncio.to_thread + calls = [] + + async def recording_to_thread(function, /, *args, **kwargs): + calls.append(function) + return await real_to_thread(function, *args, **kwargs) + + monkeypatch.setattr( + encryption_codec_module.asyncio, + "to_thread", + recording_to_thread, + ) + + small_payload = b"s" * (16 * 1024) + large_payload = b"L" * (1024 * 1024) + assert await _roundtrip(codec, small_payload) == small_payload + small_chunk_thread_calls = len(calls) + + assert await _roundtrip(codec, large_payload) == large_payload + large_chunk_thread_calls = len(calls) - small_chunk_thread_calls + + assert small_chunk_thread_calls == 0 + # Encode AND decode must each offload for large chunks. + assert large_chunk_thread_calls == 2 + + +async def test_encryption_codec_threshold_boundary( + monkeypatch, configured_encryption_key +): + # Pin the 128 KiB dispatch boundary itself: a plaintext one byte under + # the threshold encodes inline; at the threshold it offloads. (Decode + # sizes shift by the 40-byte nonce+tag overhead, so decode of the + # just-under payload may legitimately offload — only encode is pinned.) + threshold = encryption_codec_module._THREAD_OFFLOAD_THRESHOLD + codec = EncryptionCodec(header="boundary-test") + real_to_thread = asyncio.to_thread + calls = [] + + async def recording_to_thread(function, /, *args, **kwargs): + calls.append(function.__name__) + return await real_to_thread(function, *args, **kwargs) + + monkeypatch.setattr( + encryption_codec_module.asyncio, + "to_thread", + recording_to_thread, + ) + + class _Spec: + class prototype: + class buffer: + @staticmethod + def from_bytes(data): + return _Bytes(data) + + class _Bytes: + def __init__(self, data): + self._data = data + + def to_bytes(self): + return self._data + + under = await codec._encode_single(_Bytes(b"u" * (threshold - 1)), _Spec) + assert calls == [] + assert len(under.to_bytes()) == threshold - 1 + 40 + + await codec._encode_single(_Bytes(b"a" * threshold), _Spec) + assert calls == ["encrypt"] + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def _run_in_fresh_interpreter(script: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["uv", "run", "python", "-c", script], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + +def test_package_import_does_not_eagerly_load_heavy_dependencies(): + completed = _run_in_fresh_interpreter( + "import dclimate_client_py, sys; " + "print(','.join(m for m in ('s3fs','geopandas','pystac') if m in sys.modules))" + ) + + assert completed.returncode == 0, completed.stdout + completed.stderr + assert completed.stdout.strip() == "" + + +def test_lazy_public_api_still_supports_geospatial_selection(): + script = textwrap.dedent( + """ + from dclimate_client_py import GeotemporalData, dClimateClient + + import geopandas as gpd + import numpy as np + import xarray as xr + from shapely.geometry import Point + + assert dClimateClient.__name__ == "dClimateClient" + dataset = xr.Dataset( + {"temperature": (("latitude", "longitude"), [[1, 2], [3, 4]])}, + coords={"latitude": [0.0, 1.0], "longitude": [10.0, 11.0]}, + ) + mask = gpd.GeoSeries([Point(11.0, 1.0)], crs=4326).array + selected = GeotemporalData(dataset, "lazy-api-test").points(mask, 4326) + assert selected.data["temperature"].item() == 4 + """ + ) + + completed = _run_in_fresh_interpreter(script) + + assert completed.returncode == 0, completed.stdout + completed.stderr + + +def test_points_rejects_missing_geometries(dataset): + import geopandas as gpd + from shapely.geometry import Point + + from dclimate_client_py import dclimate_zarr_errors as errors + from dclimate_client_py.geotemporal_data import GeotemporalData + + mask = gpd.GeoSeries([Point(180.0, 0.0), None]).array + data = GeotemporalData(dataset, "missing-geometry") + + with pytest.raises(errors.InvalidSelectionError, match="missing geometries"): + data.points(mask, epsg_crs=4326) + + +@pytest.mark.parametrize( + ("point_kind", "message"), + [("empty", "empty geometries"), ("nan", "non-finite coordinates")], +) +def test_points_rejects_empty_and_non_finite_geometries(dataset, point_kind, message): + from dclimate_client_py import dclimate_zarr_errors as errors + + point = Point() if point_kind == "empty" else Point(float("nan"), 0.0) + mask = gpd.GeoSeries([point], crs=4326).array + data = GeotemporalData(dataset, "invalid-geometry") + + with pytest.raises(errors.InvalidSelectionError, match=message): + data.points(mask, epsg_crs=4326) diff --git a/tests/test_review_perf_spatial.py b/tests/test_review_perf_spatial.py new file mode 100644 index 0000000..c853a9b --- /dev/null +++ b/tests/test_review_perf_spatial.py @@ -0,0 +1,253 @@ +import tracemalloc + +import numpy as np +import xarray as xr + +from dclimate_client_py.geotemporal_data import GeotemporalData + + +def _rectangle_dataset(): + values = np.arange(84, dtype=float).reshape(2, 6, 7) + return xr.Dataset( + {"value": (("time", "latitude", "longitude"), values)}, + coords={ + "time": np.array(["2024-01-01", "2024-01-02"], dtype="datetime64[ns]"), + "latitude": [0.0, 10.0, 20.0, 30.0, 40.0, 50.0], + "longitude": [180.0, 185.0, 190.0, 195.0, 200.0, 205.0, 210.0], + }, + ) + + +def test_rectangle_ascending_coordinates_selects_expected_cells(): + result = GeotemporalData(_rectangle_dataset(), "synthetic").rectangle( + 10.0, 185.0, 30.0, 195.0 + ) + + np.testing.assert_array_equal(result.data.latitude, [10.0, 20.0, 30.0]) + np.testing.assert_array_equal(result.data.longitude, [185.0, 190.0, 195.0]) + np.testing.assert_array_equal( + result.data["value"], + [ + [[8.0, 9.0, 10.0], [15.0, 16.0, 17.0], [22.0, 23.0, 24.0]], + [[50.0, 51.0, 52.0], [57.0, 58.0, 59.0], [64.0, 65.0, 66.0]], + ], + ) + + +def test_rectangle_descending_latitude_selects_same_cells(): + descending = _rectangle_dataset().isel(latitude=slice(None, None, -1)) + + result = GeotemporalData(descending, "synthetic").rectangle( + 10.0, 185.0, 30.0, 195.0 + ) + + np.testing.assert_array_equal(result.data.latitude, [30.0, 20.0, 10.0]) + np.testing.assert_array_equal(result.data.longitude, [185.0, 190.0, 195.0]) + np.testing.assert_array_equal( + result.data["value"], + [ + [[22.0, 23.0, 24.0], [15.0, 16.0, 17.0], [8.0, 9.0, 10.0]], + [[64.0, 65.0, 66.0], [57.0, 58.0, 59.0], [50.0, 51.0, 52.0]], + ], + ) + + +def test_rectangle_bounds_between_grid_points_use_cells_inside_bounds(): + result = GeotemporalData(_rectangle_dataset(), "synthetic").rectangle( + 5.0, 181.0, 34.0, 199.0 + ) + + np.testing.assert_array_equal(result.data.latitude, [10.0, 20.0, 30.0]) + np.testing.assert_array_equal(result.data.longitude, [185.0, 190.0, 195.0]) + np.testing.assert_array_equal( + result.data["value"], + [ + [[8.0, 9.0, 10.0], [15.0, 16.0, 17.0], [22.0, 23.0, 24.0]], + [[50.0, 51.0, 52.0], [57.0, 58.0, 59.0], [64.0, 65.0, 66.0]], + ], + ) + + +def test_rectangle_bounds_outside_grid_are_clipped_to_available_cells(): + result = GeotemporalData(_rectangle_dataset(), "synthetic").rectangle( + -50.0, 175.0, 15.0, 500.0 + ) + + np.testing.assert_array_equal(result.data.latitude, [0.0, 10.0]) + np.testing.assert_array_equal( + result.data.longitude, [180.0, 185.0, 190.0, 195.0, 200.0, 205.0, 210.0] + ) + np.testing.assert_array_equal( + result.data["value"], + [ + [ + [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0], + [7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0], + ], + [ + [42.0, 43.0, 44.0, 45.0, 46.0, 47.0, 48.0], + [49.0, 50.0, 51.0, 52.0, 53.0, 54.0, 55.0], + ], + ], + ) + + +def test_rectangle_empty_selection_drops_both_spatial_dimensions(): + result = GeotemporalData(_rectangle_dataset(), "synthetic").rectangle( + 61.0, 185.0, 69.0, 195.0 + ) + + assert result.data.sizes == {"time": 2, "latitude": 0, "longitude": 0} + assert result.data["value"].shape == (2, 0, 0) + + +def test_circle_matches_current_tight_crop_and_nan_placement(): + dataset = xr.Dataset( + { + "value": ( + ("time", "latitude", "longitude"), + np.arange(25, dtype=float).reshape(1, 5, 5), + ) + }, + coords={ + "time": [0], + "latitude": [-2.0, -1.0, 0.0, 1.0, 2.0], + "longitude": [-2.0, -1.0, 0.0, 1.0, 2.0], + }, + ) + + result = GeotemporalData(dataset, "synthetic").circle(0.0, 0.0, 125.0) + + np.testing.assert_array_equal(result.data.latitude, [-1.0, 0.0, 1.0]) + np.testing.assert_array_equal(result.data.longitude, [-1.0, 0.0, 1.0]) + np.testing.assert_array_equal( + result.data["value"], + [[[np.nan, 7.0, np.nan], [11.0, 12.0, 13.0], [np.nan, 17.0, np.nan]]], + ) + + +def _large_dataset(): + return xr.Dataset( + { + "value": ( + ("time", "latitude", "longitude"), + np.zeros((50, 320, 320), dtype=np.float64), + ) + }, + coords={ + "time": np.arange(50), + "latitude": np.linspace(-20.0, 20.0, 320), + "longitude": np.linspace(-20.0, 20.0, 320), + }, + ) + + +def test_rectangle_tiny_selection_allocates_less_than_10_mb(): + data = GeotemporalData(_large_dataset(), "synthetic") + + tracemalloc.start() + try: + result = data.rectangle(-0.2, -0.2, 0.2, 0.2) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert result.data.sizes["latitude"] > 0 + assert result.data.sizes["longitude"] > 0 + assert peak < 10 * 1024 * 1024, f"peak allocation was {peak / 1024**2:.1f} MB" + + +def test_circle_small_radius_allocates_less_than_10_mb(): + data = GeotemporalData(_large_dataset(), "synthetic") + + tracemalloc.start() + try: + result = data.circle(0.0, 0.0, 50.0) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert result.data.sizes["latitude"] > 0 + assert result.data.sizes["longitude"] > 0 + assert peak < 10 * 1024 * 1024, f"peak allocation was {peak / 1024**2:.1f} MB" + + +def test_rectangle_descending_longitude_selects_same_cells(): + descending = _rectangle_dataset().isel(longitude=slice(None, None, -1)) + + result = GeotemporalData(descending, "synthetic").rectangle( + 10.0, 185.0, 30.0, 195.0 + ) + + np.testing.assert_array_equal(result.data.latitude, [10.0, 20.0, 30.0]) + np.testing.assert_array_equal(result.data.longitude, [195.0, 190.0, 185.0]) + np.testing.assert_array_equal( + result.data["value"], + [ + [[10.0, 9.0, 8.0], [17.0, 16.0, 15.0], [24.0, 23.0, 22.0]], + [[52.0, 51.0, 50.0], [59.0, 58.0, 57.0], [66.0, 65.0, 64.0]], + ], + ) + + +def _circle_reference(data, lat, lon, radius): + # Implementation-independent reference: full-grid haversine mask. + from dclimate_client_py.geotemporal_data import _haversine + + distances = _haversine(lat, lon, data["latitude"], data["longitude"]) + return data.where(distances < radius, drop=True) + + +def test_circle_high_latitude_widens_longitude_bounding_box(): + # At 60N a 300 km radius spans ~5.4 degrees of longitude but only + # ~2.7 degrees of latitude; a bbox that forgets the cos(latitude) + # widening would silently drop the outer longitude columns. + latitudes = np.arange(58.0, 63.0) + longitudes = np.arange(-10.0, 11.0) + values = np.arange(len(latitudes) * len(longitudes), dtype=float).reshape( + len(latitudes), len(longitudes) + ) + dataset = xr.Dataset( + {"value": (("latitude", "longitude"), values)}, + coords={"latitude": latitudes, "longitude": longitudes}, + ) + + result = GeotemporalData(dataset, "synthetic").circle(60.0, 0.0, 300.0) + + expected = _circle_reference(dataset, 60.0, 0.0, 300.0) + xr.testing.assert_identical(result.data, expected) + assert result.data.sizes["longitude"] == 11 + + +def test_rectangle_non_monotonic_coordinates_fall_back_to_mask_path(): + latitudes = np.array([0.0, 20.0, 10.0, 30.0, 40.0, 50.0]) + longitudes = np.arange(180.0, 210.0, 5.0) + values = np.arange(len(latitudes) * len(longitudes), dtype=float).reshape( + len(latitudes), len(longitudes) + ) + dataset = xr.Dataset( + {"value": (("latitude", "longitude"), values)}, + coords={"latitude": latitudes, "longitude": longitudes}, + ) + + result = GeotemporalData(dataset, "synthetic").rectangle(10.0, 185.0, 30.0, 195.0) + + np.testing.assert_array_equal(result.data.latitude, [20.0, 10.0, 30.0]) + np.testing.assert_array_equal(result.data.longitude, [185.0, 190.0, 195.0]) + + +def test_rectangle_nan_coordinate_falls_back_to_mask_path(): + latitudes = np.array([0.0, 10.0, np.nan, 30.0, 40.0, 50.0]) + longitudes = np.arange(180.0, 210.0, 5.0) + values = np.arange(len(latitudes) * len(longitudes), dtype=float).reshape( + len(latitudes), len(longitudes) + ) + dataset = xr.Dataset( + {"value": (("latitude", "longitude"), values)}, + coords={"latitude": latitudes, "longitude": longitudes}, + ) + + result = GeotemporalData(dataset, "synthetic").rectangle(10.0, 185.0, 30.0, 195.0) + + # The NaN latitude row must be excluded, exactly as the old mask did. + np.testing.assert_array_equal(result.data.latitude, [10.0, 30.0]) diff --git a/tests/test_review_suspects.py b/tests/test_review_suspects.py new file mode 100644 index 0000000..1992f06 --- /dev/null +++ b/tests/test_review_suspects.py @@ -0,0 +1,726 @@ +from __future__ import annotations + +import gc +from datetime import datetime, timezone +import json +from typing import Any +from unittest.mock import Mock + +import httpx +import pystac +import pytest + +from dclimate_client_py import ipfs_retrieval, stac_catalog, stac_server + + +_install_mock_client = None + + +@pytest.fixture(autouse=True) +def _use_managed_httpx_clients(install_httpx_mock): + global _install_mock_client + _install_mock_client = install_httpx_mock + yield + _install_mock_client = None + + +class _Response: + def __init__(self, payload: dict[str, Any]) -> None: + self._payload = payload + + def json(self) -> dict[str, Any]: + return self._payload + + def raise_for_status(self) -> None: + return None + + +def _install_post(monkeypatch, post) -> None: + def handler(request: httpx.Request) -> httpx.Response: + kwargs = { + "json": json.loads(request.content), + "timeout": request.extensions["timeout"]["read"], + } + if "authorization" in request.headers: + kwargs["headers"] = { + "Authorization": request.headers["authorization"], + } + response = post(str(request.url), **kwargs) + return httpx.Response(200, json=response._payload, request=request) + + assert _install_mock_client is not None + _install_mock_client(stac_server, handler) + + +def _catalog_with_item(item: pystac.Item) -> pystac.Catalog: + root = pystac.Catalog(id="root", description="Root") + organization = pystac.Catalog(id="org", description="Organization") + collection = pystac.Collection( + id="chirps", + description="CHIRPS", + extent=pystac.Extent( + pystac.SpatialExtent([[-180.0, -90.0, 180.0, 90.0]]), + pystac.TemporalExtent([[item.datetime, None]]), + ), + ) + collection.add_item(item) + organization.add_child(collection) + root.add_child(organization) + + organization_link = root.get_child_links()[0] + organization_link.extra_fields.update( + { + "dclimate:id": "org", + "dclimate:type": "organization", + "dclimate:collections:historical": ["chirps"], + "dclimate:datasets": ["chirps/precip-daily"], + } + ) + collection_link = organization.get_child_links()[0] + collection_link.extra_fields["dclimate:id"] = "chirps" + return root + + +def test_stac_resolvers_honor_hyphenated_dataset_and_variant(monkeypatch): + cid = "bafy-hyphenated-identifiers" + properties = { + "dclimate:dataset_id": "precip-daily", + "dclimate:variant": "final-p05", + } + feature = { + "id": "chirps-precip-daily-final-p05", + "collection": "chirps", + "properties": properties, + "assets": {"data": {"href": f"ipfs://{cid}"}}, + } + _install_post( + monkeypatch, + lambda *args, **kwargs: _Response({"features": [feature]}), + ) + + item = pystac.Item( + id=feature["id"], + geometry=None, + bbox=None, + datetime=datetime(2024, 1, 1, tzinfo=timezone.utc), + properties=properties, + ) + item.add_asset("data", pystac.Asset(href=f"ipfs://{cid}")) + catalog = _catalog_with_item(item) + + assert ( + stac_server.resolve_cid_from_stac_server( + collection="chirps", + dataset="precip-daily", + variant="final-p05", + server_url="https://example.test", + ).cid + == cid + ) + assert ( + stac_catalog.resolve_dataset_cid_from_stac( + catalog, + collection="chirps", + dataset="precip-daily", + variant="final-p05", + organization="org", + ).cid + == cid + ) + + +def test_stac_server_follows_next_link_to_resolve_later_item(monkeypatch): + collection = "test_collection" + first_page_features = [ + { + "id": f"{collection}-other_{index}-default", + "collection": collection, + "properties": { + "dclimate:dataset_id": f"other_{index}", + "dclimate:variant": "default", + }, + "assets": {"data": {"href": f"ipfs://bafy-other-{index}"}}, + } + for index in range(100) + ] + target = { + "id": f"{collection}-target-finalized", + "collection": collection, + "properties": { + "dclimate:dataset_id": "target", + "dclimate:variant": "finalized", + }, + "assets": {"data": {"href": "ipfs://bafy-page-two-target"}}, + } + pages = [ + { + "features": first_page_features, + "links": [ + { + "rel": "next", + "href": "https://example.test/search?token=page-2", + "method": "POST", + "body": {"token": "page-2"}, + } + ], + }, + {"features": [target], "links": []}, + ] + calls: list[tuple[str, dict[str, Any] | None]] = [] + + def post(url, json=None, **kwargs): + calls.append((url, json)) + return _Response(pages[len(calls) - 1]) + + _install_post(monkeypatch, post) + + resolved = stac_server.resolve_cid_from_stac_server( + collection=collection, + dataset="target", + variant="finalized", + server_url="https://example.test", + ) + + assert resolved.cid == "bafy-page-two-target" + assert len(calls) == 2 + + +def test_non_network_timeout_text_is_not_a_connection_error(): + error = RuntimeError( + "invalid Zarr metadata: variable 'timeout' has an unsupported dtype" + ) + + assert not ipfs_retrieval._is_connection_error(error) + + +def test_stac_resolvers_agree_on_default_variant_for_bare_items(monkeypatch): + # A bare item (no variant segment, no properties) is reported by the + # listing APIs as variant "default"; BOTH resolvers must accept that + # name so server->catalog fallback returns the same result. + cid = "bafy-bare-default" + feature = { + "id": "chirps-temp", + "collection": "chirps", + "assets": {"data": {"href": f"ipfs://{cid}"}}, + } + _install_post( + monkeypatch, + lambda *args, **kwargs: _Response({"features": [feature]}), + ) + + item = pystac.Item( + id="chirps-temp", + geometry=None, + bbox=None, + datetime=datetime(2024, 1, 1, tzinfo=timezone.utc), + properties={}, + ) + item.add_asset("data", pystac.Asset(href=f"ipfs://{cid}")) + catalog = _catalog_with_item(item) + + assert ( + stac_server.resolve_cid_from_stac_server( + collection="chirps", + dataset="temp", + variant="default", + server_url="https://example.test", + ).cid + == cid + ) + assert ( + stac_catalog.resolve_dataset_cid_from_stac( + catalog, + collection="chirps", + dataset="temp", + variant="default", + organization="org", + ).cid + == cid + ) + + +def test_stac_server_resolves_hyphenated_variant_without_properties(monkeypatch): + # Same hyphenated grammar, but with no dclimate:* properties at all — + # resolution must work from the item id alone given the dataset hint. + cid = "bafy-id-only-hyphens" + feature = { + "id": "chirps-precip-daily-final-p05", + "collection": "chirps", + "assets": {"data": {"href": f"ipfs://{cid}"}}, + } + _install_post( + monkeypatch, + lambda *args, **kwargs: _Response({"features": [feature]}), + ) + + assert ( + stac_server.resolve_cid_from_stac_server( + collection="chirps", + dataset="precip-daily", + variant="final-p05", + server_url="https://example.test", + ).cid + == cid + ) + + +def test_stac_server_merge_next_link_keeps_collections_filter(monkeypatch): + # STAC API next-link contract: "merge": true extends the original body. + # Dropping the collections filter on page 2 could poison resolution with + # foreign-collection items. + cid = "bafy-merged-page-two" + bodies = [] + + def post(url, json=None, timeout=None): + bodies.append(json) + if len(bodies) == 1: + return _Response( + { + "features": [ + { + "id": "chirps-other", + "collection": "chirps", + "assets": {"data": {"href": "ipfs://bafy-other"}}, + } + ], + "links": [ + { + "rel": "next", + "href": "https://example.test/search", + "method": "POST", + "merge": True, + "body": {"token": "page-2"}, + } + ], + } + ) + return _Response( + { + "features": [ + { + "id": "chirps-temp-final", + "collection": "chirps", + "assets": {"data": {"href": f"ipfs://{cid}"}}, + } + ] + } + ) + + _install_post(monkeypatch, post) + + assert ( + stac_server.resolve_cid_from_stac_server( + collection="chirps", + dataset="temp", + variant="final", + server_url="https://example.test", + ).cid + == cid + ) + assert bodies[1]["token"] == "page-2" + assert bodies[1]["collections"] == ["chirps"] + + +def test_stac_server_merge_next_link_without_body_keeps_original_body_and_headers( + monkeypatch, +): + calls = [] + + def post(url, json=None, timeout=None, headers=None): + calls.append({"url": url, "json": json, "headers": headers}) + if len(calls) == 1: + return _Response( + { + "features": [{"id": "chirps-other"}], + "links": [ + { + "rel": "next", + "href": "/search", + "method": "POST", + "merge": True, + "headers": {"Authorization": "Bearer page-two"}, + } + ], + } + ) + return _Response({"features": []}) + + _install_post(monkeypatch, post) + + list( + stac_server._search_pages( + "https://example.test", {"limit": 100, "collections": ["chirps"]}, 10 + ) + ) + + assert calls[1]["json"] == {"limit": 100, "collections": ["chirps"]} + assert calls[1]["headers"] == {"Authorization": "Bearer page-two"} + + +def test_stac_server_raises_when_page_limit_would_truncate_results(monkeypatch): + monkeypatch.setattr(stac_server, "_MAX_SEARCH_PAGES", 2) + calls = 0 + + def post(*args, **kwargs): + nonlocal calls + calls += 1 + return _Response( + { + "features": [{"id": f"chirps-other-{calls}"}], + "links": [ + { + "rel": "next", + "href": f"/search?page={calls + 1}", + "method": "POST", + "body": {"page": calls + 1}, + } + ], + } + ) + + _install_post(monkeypatch, post) + + with pytest.raises(ValueError, match="page limit of 2"): + list(stac_server._search_pages("https://example.test", {"limit": 100}, 10)) + + assert calls == 2 + + +def test_stac_server_default_variant_can_be_on_a_later_page(monkeypatch): + pages = [ + { + "features": [ + { + "id": "chirps-temp-latest", + "collection": "chirps", + "assets": {"data": {"href": "ipfs://bafy-latest"}}, + } + ], + "links": [ + { + "rel": "next", + "href": "/search?page=2", + "method": "POST", + "body": {"page": 2}, + } + ], + }, + { + "features": [ + { + "id": "chirps-temp", + "collection": "chirps", + "assets": {"data": {"href": "ipfs://bafy-default"}}, + } + ] + }, + ] + calls = 0 + + def post(*args, **kwargs): + nonlocal calls + response = _Response(pages[calls]) + calls += 1 + return response + + _install_post(monkeypatch, post) + + resolved = stac_server.resolve_cid_from_stac_server( + "chirps", "temp", server_url="https://example.test" + ) + + assert resolved.cid == "bafy-default" + assert calls == 2 + + +def test_known_hyphenated_dataset_does_not_match_shorter_prefix(monkeypatch): + legacy_feature = { + "id": "chirps-precip-daily-final-p05", + "collection": "chirps", + "assets": {"data": {"href": "ipfs://bafy-legacy-hyphenated"}}, + } + explicit_sibling = { + "id": "chirps-precip-daily-prelim-p05", + "collection": "chirps", + "properties": { + "dclimate:dataset_id": "precip-daily", + "dclimate:variant": "prelim-p05", + }, + "assets": {"data": {"href": "ipfs://bafy-explicit-sibling"}}, + } + _install_post( + monkeypatch, + lambda *args, **kwargs: _Response( + {"features": [legacy_feature, explicit_sibling]} + ), + ) + + with pytest.raises(ValueError, match="No items found"): + stac_server.resolve_cid_from_stac_server( + "chirps", "precip", server_url="https://example.test" + ) + + item = pystac.Item( + id=legacy_feature["id"], + geometry=None, + bbox=None, + datetime=datetime(2024, 1, 1, tzinfo=timezone.utc), + properties={}, + ) + item.add_asset("data", pystac.Asset(href="ipfs://bafy-legacy-hyphenated")) + catalog = _catalog_with_item(item) + + with pytest.raises(ValueError, match="Dataset 'precip' not found"): + stac_catalog.resolve_dataset_cid_from_stac( + catalog, + collection="chirps", + dataset="precip", + organization="org", + ) + + +def test_requested_hyphenated_dataset_is_a_disambiguation_candidate(monkeypatch): + features = [ + { + "id": "chirps-precip-daily-final", + "collection": "chirps", + "properties": {}, + "assets": {"data": {"href": "ipfs://bafy-precip-daily-final"}}, + }, + { + "id": "chirps-precip-default", + "collection": "chirps", + "properties": { + "dclimate:dataset_id": "precip", + "dclimate:variant": "default", + }, + "assets": {"data": {"href": "ipfs://bafy-precip-default"}}, + }, + ] + _install_post( + monkeypatch, + lambda *args, **kwargs: _Response({"features": features}), + ) + + resolved = stac_server.resolve_cid_from_stac_server( + "chirps", + "precip-daily", + variant="final", + server_url="https://example.test", + ) + + assert resolved.cid == "bafy-precip-daily-final" + + +def test_catalog_requested_dataset_is_a_disambiguation_candidate(): + target = pystac.Item( + id="chirps-precip-daily-final", + geometry=None, + bbox=None, + datetime=datetime(2024, 1, 1, tzinfo=timezone.utc), + properties={}, + ) + target.add_asset( + "data", pystac.Asset(href="ipfs://bafy-catalog-precip-daily-final") + ) + catalog = _catalog_with_item(target) + + # Simulate incomplete metadata that declares only the shorter sibling. + catalog.get_child_links()[0].extra_fields["dclimate:datasets"] = ["chirps/precip"] + collection = catalog.get_child("org").get_child("chirps") + sibling = pystac.Item( + id="chirps-precip-default", + geometry=None, + bbox=None, + datetime=datetime(2024, 1, 1, tzinfo=timezone.utc), + properties={ + "dclimate:dataset_id": "precip", + "dclimate:variant": "default", + }, + ) + sibling.add_asset("data", pystac.Asset(href="ipfs://bafy-catalog-precip")) + collection.add_item(sibling) + + cid = stac_catalog.resolve_dataset_cid_from_stac( + catalog, + collection="chirps", + dataset="precip-daily", + variant="final", + organization="org", + ) + + assert cid.cid == "bafy-catalog-precip-daily-final" + + +def test_load_stac_catalog_binds_io_without_mutating_pystac_default(monkeypatch): + observed = {} + + def from_file(cls, href, stac_io=None): + observed["href"] = href + observed["stac_io"] = stac_io + return pystac.Catalog(id="root", description="Root") + + monkeypatch.setattr(pystac.Catalog, "from_file", classmethod(from_file)) + monkeypatch.setattr( + pystac.StacIO, + "set_default", + lambda *args, **kwargs: pytest.fail("global pystac default was mutated"), + ) + + stac_catalog.load_stac_catalog("https://gateway-a.test", root_cid="bafy-root") + + assert observed["href"] == "ipfs://bafy-root" + assert isinstance(observed["stac_io"], stac_catalog.IPFSStacIO) + assert observed["stac_io"].gateway_url == "https://gateway-a.test" + + +def test_load_stac_catalog_closes_client_when_parsing_fails(monkeypatch): + client = httpx.Client(transport=httpx.MockTransport(lambda request: None)) + monkeypatch.setattr(stac_catalog.httpx, "Client", lambda *args, **kwargs: client) + + def fail_from_file(cls, href, stac_io=None): + raise RuntimeError("invalid catalog") + + monkeypatch.setattr(pystac.Catalog, "from_file", classmethod(fail_from_file)) + + with pytest.raises(RuntimeError, match="invalid catalog"): + stac_catalog.load_stac_catalog("https://gateway.test", root_cid="bafy-invalid") + + assert client.is_closed + + +def test_load_stac_catalog_closes_client_when_catalog_is_released(monkeypatch): + client = httpx.Client(transport=httpx.MockTransport(lambda request: None)) + monkeypatch.setattr(stac_catalog.httpx, "Client", lambda *args, **kwargs: client) + monkeypatch.setattr( + pystac.Catalog, + "from_file", + classmethod( + lambda cls, href, stac_io=None: pystac.Catalog( + id="root", description="Root" + ) + ), + ) + + catalog = stac_catalog.load_stac_catalog( + "https://gateway.test", root_cid="bafy-root" + ) + assert not client.is_closed + + del catalog + gc.collect() + + assert client.is_closed + + +def test_load_stac_catalog_uses_configured_pointer_endpoint(monkeypatch): + response = Mock() + response.json.return_value = {"cid": "bafy-configured-root"} + pointer_client = Mock() + pointer_client.get.return_value = response + monkeypatch.setattr(stac_catalog, "_client", lambda: pointer_client) + observed = {} + + def from_file(cls, href, stac_io=None): + observed["href"] = href + return pystac.Catalog(id="root", description="Root") + + monkeypatch.setattr(pystac.Catalog, "from_file", classmethod(from_file)) + + stac_catalog.load_stac_catalog( + "https://gateway.test", catalog_url="https://control.test/catalog-root" + ) + + pointer_client.get.assert_called_once_with( + "https://control.test/catalog-root", + timeout=30, + headers=None, + auth=httpx.USE_CLIENT_DEFAULT, + ) + assert observed["href"] == "ipfs://bafy-configured-root" + + +def test_load_stac_catalog_threads_gateway_credentials(monkeypatch): + """Auth/headers must reach both the pointer fetch and gateway I/O. + + Regression guard: authenticated gateways previously 401'd on catalog + fallback because credentials were dropped between the KuboCAS data path + and the STAC catalog reads. + """ + headers = {"Authorization": "Bearer token"} + auth = ("user", "secret") + + response = Mock() + response.json.return_value = {"cid": "bafy-auth-root"} + pointer_client = Mock() + pointer_client.get.return_value = response + monkeypatch.setattr(stac_catalog, "_client", lambda: pointer_client) + + captured: dict[str, Any] = {} + + def recording_init(self, gateway_url, *, headers=None, auth=None): + captured["gateway_url"] = gateway_url + captured["headers"] = headers + captured["auth"] = auth + self.gateway_url = gateway_url.rstrip("/") + self.client = Mock() + + monkeypatch.setattr(stac_catalog.IPFSStacIO, "__init__", recording_init) + + def from_file(cls, href, stac_io=None): + return pystac.Catalog(id="root", description="Root") + + monkeypatch.setattr(pystac.Catalog, "from_file", classmethod(from_file)) + + stac_catalog.load_stac_catalog( + "https://gateway.test", + catalog_url="https://control.test/catalog-root", + headers=headers, + auth=auth, + ) + + # Pointer endpoint (same host as the gateway) carries the credentials. + pointer_client.get.assert_called_once_with( + "https://control.test/catalog-root", + timeout=30, + headers=headers, + auth=auth, + ) + # Gateway I/O handler is constructed with the credentials too. + assert captured["headers"] == headers + assert captured["auth"] == auth + + +def test_catalog_lister_uses_dataset_metadata_for_partial_item_properties(): + item = pystac.Item( + id="chirps-precip-daily-final-p05", + geometry=None, + bbox=None, + datetime=datetime(2024, 1, 1, tzinfo=timezone.utc), + properties={"dclimate:dataset_id": "precip-daily"}, + ) + item.add_asset("data", pystac.Asset(href="ipfs://bafy-partial-catalog")) + + listing = stac_catalog.list_available_datasets(_catalog_with_item(item)) + variant = listing["chirps"]["variants"][0] + + assert variant["dataset"] == "precip-daily" + assert variant["variant"] == "final-p05" + assert variant["cid"] == "bafy-partial-catalog" + + +def test_catalog_lister_keeps_bare_item_with_default_variant_property(): + item = pystac.Item( + id="chirps-precip-daily", + geometry=None, + bbox=None, + datetime=datetime(2024, 1, 1, tzinfo=timezone.utc), + properties={"dclimate:variant": "default"}, + ) + item.add_asset("data", pystac.Asset(href="ipfs://bafy-bare-default-catalog")) + + listing = stac_catalog.list_available_datasets(_catalog_with_item(item)) + variant = listing["chirps"]["variants"][0] + + assert variant["dataset"] == "precip-daily" + assert variant["variant"] == "default" diff --git a/tests/test_s3_retrieval.py b/tests/test_s3_retrieval.py index 6a15c50..58d71b7 100644 --- a/tests/test_s3_retrieval.py +++ b/tests/test_s3_retrieval.py @@ -25,11 +25,14 @@ def test__given_a_dataset_name_and_bucket_name__it_fetch_the_dataset( s3Map_mock = mocker.patch("dclimate_client_py.s3_retrieval.S3Map") mock_dataset = namedtuple("Dataset", ["update_in_progress"])(False) - mocker.patch("xarray.open_zarr", return_value=mock_dataset) + open_zarr = mocker.patch("xarray.open_zarr", return_value=mock_dataset) ds = s3_retrieval.get_dataset_from_s3(dataset_name, bucket_name) assert ds is mock_dataset + open_zarr.assert_called_once_with( + s3Map_mock.return_value, chunks=None, decode_timedelta=True + ) s3Map_mock.assert_called_with( f"s3://{bucket_name}/datasets/{dataset_name}.zarr", s3=fake_s3fs, diff --git a/tests/test_stac_catalog.py b/tests/test_stac_catalog.py index dfc44e2..7cc2a18 100644 --- a/tests/test_stac_catalog.py +++ b/tests/test_stac_catalog.py @@ -1,25 +1,24 @@ """ Comprehensive tests for STAC catalog integration module. -Tests all functions and classes in stac_catalog.py using real data from the dClimate IPFS gateway. -No mocking is used - all tests interact with actual STAC catalog data. +Tests all functions and classes in stac_catalog.py. Integration cases use the +real dClimate IPFS gateway; isolated protocol cases use httpx MockTransport. """ +import httpx import pytest import pystac from dclimate_client_py import stac_catalog +from tests.ipfs_config import IPFS_GATEWAY_URL, STAC_CATALOG_URL -# Mark all tests in this module to use the IPFS connection check -pytestmark = pytest.mark.usefixtures("check_ipfs_connection") - - +@pytest.mark.stac_pointer class TestGetRootCatalogCid: """Test the get_root_catalog_cid function.""" def test_get_root_catalog_cid_returns_string(self): """Test that get_root_catalog_cid returns a non-empty string CID.""" - cid = stac_catalog.get_root_catalog_cid() + cid = stac_catalog.get_root_catalog_cid(STAC_CATALOG_URL) assert isinstance(cid, str) assert len(cid) > 0 @@ -28,8 +27,8 @@ def test_get_root_catalog_cid_returns_string(self): def test_get_root_catalog_cid_consistent(self): """Test that multiple calls return consistent CID format.""" - cid1 = stac_catalog.get_root_catalog_cid() - cid2 = stac_catalog.get_root_catalog_cid() + cid1 = stac_catalog.get_root_catalog_cid(STAC_CATALOG_URL) + cid2 = stac_catalog.get_root_catalog_cid(STAC_CATALOG_URL) # Both should be valid CIDs assert isinstance(cid1, str) @@ -58,13 +57,15 @@ def test_initialization_strips_trailing_slash(self): assert stac_io.gateway_url == "https://ipfs-gateway.dclimate.net" assert not stac_io.gateway_url.endswith("/") + @pytest.mark.ipfs + @pytest.mark.stac_pointer def test_read_text_with_ipfs_uri(self): """Test reading content from ipfs:// URI via gateway.""" - gateway_url = "https://ipfs-gateway.dclimate.net" + gateway_url = IPFS_GATEWAY_URL stac_io = stac_catalog.IPFSStacIO(gateway_url) # Get a real CID from the catalog - root_cid = stac_catalog.get_root_catalog_cid() + root_cid = stac_catalog.get_root_catalog_cid(STAC_CATALOG_URL) ipfs_uri = f"ipfs://{root_cid}" # Read the content @@ -79,13 +80,15 @@ def test_read_text_with_ipfs_uri(self): assert "type" in catalog_data assert catalog_data["type"] in ["Catalog", "Collection"] + @pytest.mark.ipfs + @pytest.mark.stac_pointer def test_read_text_handles_ipfs_uri_correctly(self): """Test that IPFS URIs are properly transformed to gateway HTTP URLs.""" - gateway_url = "https://ipfs-gateway.dclimate.net" + gateway_url = IPFS_GATEWAY_URL stac_io = stac_catalog.IPFSStacIO(gateway_url) # Get a real CID to test with - root_cid = stac_catalog.get_root_catalog_cid() + root_cid = stac_catalog.get_root_catalog_cid(STAC_CATALOG_URL) # Test that ipfs:// URI is handled ipfs_uri = f"ipfs://{root_cid}" @@ -101,13 +104,15 @@ def test_read_text_handles_ipfs_uri_correctly(self): data = json.loads(content) assert "type" in data + @pytest.mark.ipfs + @pytest.mark.stac_pointer def test_read_text_multiple_cids(self): """Test reading from multiple different CIDs.""" - gateway_url = "https://ipfs-gateway.dclimate.net" + gateway_url = IPFS_GATEWAY_URL stac_io = stac_catalog.IPFSStacIO(gateway_url) # Get root CID and load catalog - root_cid = stac_catalog.get_root_catalog_cid() + root_cid = stac_catalog.get_root_catalog_cid(STAC_CATALOG_URL) ipfs_uri = f"ipfs://{root_cid}" # First read @@ -120,6 +125,33 @@ def test_read_text_multiple_cids(self): assert isinstance(content2, str) assert content1 == content2 + @pytest.mark.parametrize("scheme", ["http", "https"]) + def test_read_text_fetches_http_urls(self, scheme): + requested_url = f"{scheme}://catalog.example/root.json" + + def handler(request: httpx.Request) -> httpx.Response: + assert str(request.url) == requested_url + return httpx.Response(200, text='{"type": "Catalog"}', request=request) + + stac_io = stac_catalog.IPFSStacIO("https://gateway.example") + stac_io.client.close() + stac_io.client = httpx.Client(transport=httpx.MockTransport(handler)) + try: + assert stac_io.read_text(requested_url) == '{"type": "Catalog"}' + finally: + stac_io.close() + + @pytest.mark.parametrize( + "source", ["file:///tmp/catalog.json", "s3://bucket/catalog.json"] + ) + def test_read_text_rejects_unsupported_schemes(self, source): + stac_io = stac_catalog.IPFSStacIO("https://gateway.example") + try: + with pytest.raises(ValueError, match=source.split(":", 1)[0]): + stac_io.read_text(source) + finally: + stac_io.close() + def test_write_text_raises_not_implemented(self): """Test that write_text raises NotImplementedError.""" gateway_url = "https://ipfs-gateway.dclimate.net" @@ -131,14 +163,18 @@ def test_write_text_raises_not_implemented(self): assert "not supported" in str(exc_info.value).lower() +@pytest.mark.ipfs +@pytest.mark.stac_pointer class TestLoadStacCatalog: """Test the load_stac_catalog function.""" def test_load_catalog_with_auto_cid(self): """Test loading catalog with automatically fetched CID.""" - gateway_url = "https://ipfs-gateway.dclimate.net" + gateway_url = IPFS_GATEWAY_URL - catalog = stac_catalog.load_stac_catalog(gateway_url) + catalog = stac_catalog.load_stac_catalog( + gateway_url, catalog_url=STAC_CATALOG_URL + ) assert isinstance(catalog, pystac.Catalog) assert catalog.id is not None @@ -146,8 +182,8 @@ def test_load_catalog_with_auto_cid(self): def test_load_catalog_with_explicit_cid(self): """Test loading catalog with explicitly provided CID.""" - gateway_url = "https://ipfs-gateway.dclimate.net" - root_cid = stac_catalog.get_root_catalog_cid() + gateway_url = IPFS_GATEWAY_URL + root_cid = stac_catalog.get_root_catalog_cid(STAC_CATALOG_URL) catalog = stac_catalog.load_stac_catalog(gateway_url, root_cid=root_cid) @@ -156,8 +192,10 @@ def test_load_catalog_with_explicit_cid(self): def test_loaded_catalog_has_children(self): """Test that loaded catalog has child links (collections).""" - gateway_url = "https://ipfs-gateway.dclimate.net" - catalog = stac_catalog.load_stac_catalog(gateway_url) + gateway_url = IPFS_GATEWAY_URL + catalog = stac_catalog.load_stac_catalog( + gateway_url, catalog_url=STAC_CATALOG_URL + ) child_links = list(catalog.get_child_links()) assert len(child_links) > 0 @@ -170,8 +208,10 @@ def test_loaded_catalog_has_children(self): def test_catalog_collections_are_accessible(self): """Test that collections in the catalog can be resolved and accessed.""" - gateway_url = "https://ipfs-gateway.dclimate.net" - catalog = stac_catalog.load_stac_catalog(gateway_url) + gateway_url = IPFS_GATEWAY_URL + catalog = stac_catalog.load_stac_catalog( + gateway_url, catalog_url=STAC_CATALOG_URL + ) # Get first child link child_links = list(catalog.get_child_links()) @@ -186,14 +226,16 @@ def test_catalog_collections_are_accessible(self): assert isinstance(collection, (pystac.Collection, pystac.Catalog)) +@pytest.mark.ipfs +@pytest.mark.stac_pointer class TestResolveDatasetCidFromStac: """Test the resolve_dataset_cid_from_stac function.""" @pytest.fixture def loaded_catalog(self): """Fixture providing a loaded STAC catalog.""" - gateway_url = "https://ipfs-gateway.dclimate.net" - return stac_catalog.load_stac_catalog(gateway_url) + gateway_url = IPFS_GATEWAY_URL + return stac_catalog.load_stac_catalog(gateway_url, catalog_url=STAC_CATALOG_URL) def test_resolve_dataset_cid_basic(self, loaded_catalog): """Test resolving a dataset CID from the catalog.""" @@ -214,16 +256,16 @@ def test_resolve_dataset_cid_basic(self, loaded_catalog): pytest.skip("No datasets available in catalog") # Try to resolve the CID - cid = stac_catalog.resolve_dataset_cid_from_stac( + resolved = stac_catalog.resolve_dataset_cid_from_stac( loaded_catalog, collection=collection_id, dataset=dataset_type ) - assert isinstance(cid, str) - assert len(cid) > 0 + assert isinstance(resolved.cid, str) + assert len(resolved.cid) > 0 # Should not have ipfs:// prefix - assert not cid.startswith("ipfs://") + assert not resolved.cid.startswith("ipfs://") # Should be a valid CID format - assert cid.startswith(("Qm", "bafy", "bafk", "bafz")) + assert resolved.cid.startswith(("Qm", "bafy", "bafk", "bafz")) def test_resolve_dataset_cid_with_variant(self, loaded_catalog): """Test resolving a dataset CID with a specific variant.""" @@ -264,7 +306,7 @@ def test_resolve_dataset_cid_with_variant(self, loaded_catalog): item_dataset = parts[1] item_variant = parts[2] - cid = stac_catalog.resolve_dataset_cid_from_stac( + resolved = stac_catalog.resolve_dataset_cid_from_stac( loaded_catalog, collection=item_collection, dataset=item_dataset, @@ -272,9 +314,9 @@ def test_resolve_dataset_cid_with_variant(self, loaded_catalog): organization=organization_id, ) - assert isinstance(cid, str) - assert len(cid) > 0 - assert not cid.startswith("ipfs://") + assert isinstance(resolved.cid, str) + assert len(resolved.cid) > 0 + assert not resolved.cid.startswith("ipfs://") def test_resolve_dataset_cid_invalid_collection(self, loaded_catalog): """Test that invalid collection raises ValueError.""" @@ -335,14 +377,16 @@ def test_resolve_dataset_cid_invalid_variant(self, loaded_catalog): assert "variant" in str(exc_info.value).lower() +@pytest.mark.ipfs +@pytest.mark.stac_pointer class TestListAvailableDatasets: """Test the list_available_datasets function.""" @pytest.fixture def loaded_catalog(self): """Fixture providing a loaded STAC catalog.""" - gateway_url = "https://ipfs-gateway.dclimate.net" - return stac_catalog.load_stac_catalog(gateway_url) + gateway_url = IPFS_GATEWAY_URL + return stac_catalog.load_stac_catalog(gateway_url, catalog_url=STAC_CATALOG_URL) def test_list_datasets_returns_dict(self, loaded_catalog): """Test that list_available_datasets returns a dictionary.""" @@ -424,17 +468,19 @@ def test_list_datasets_excludes_links_without_dclimate_id(self, loaded_catalog): assert len(collection_id) > 0 +@pytest.mark.ipfs +@pytest.mark.stac_pointer class TestIntegrationEndToEnd: """Integration tests that exercise the full workflow.""" def test_full_workflow_load_and_resolve(self): """Test the complete workflow: get CID, load catalog, resolve dataset.""" # Step 1: Get root catalog CID - root_cid = stac_catalog.get_root_catalog_cid() + root_cid = stac_catalog.get_root_catalog_cid(STAC_CATALOG_URL) assert isinstance(root_cid, str) # Step 2: Load catalog - gateway_url = "https://ipfs-gateway.dclimate.net" + gateway_url = IPFS_GATEWAY_URL catalog = stac_catalog.load_stac_catalog(gateway_url, root_cid=root_cid) assert isinstance(catalog, pystac.Catalog) @@ -452,19 +498,23 @@ def test_full_workflow_load_and_resolve(self): break if collection_id and dataset_type: - cid = stac_catalog.resolve_dataset_cid_from_stac( + resolved = stac_catalog.resolve_dataset_cid_from_stac( catalog, collection=collection_id, dataset=dataset_type ) - assert isinstance(cid, str) - assert len(cid) > 0 + assert isinstance(resolved.cid, str) + assert len(resolved.cid) > 0 def test_multiple_catalog_loads_work(self): """Test that multiple catalog loads don't interfere with each other.""" - gateway_url = "https://ipfs-gateway.dclimate.net" + gateway_url = IPFS_GATEWAY_URL # Load catalog twice - catalog1 = stac_catalog.load_stac_catalog(gateway_url) - catalog2 = stac_catalog.load_stac_catalog(gateway_url) + catalog1 = stac_catalog.load_stac_catalog( + gateway_url, catalog_url=STAC_CATALOG_URL + ) + catalog2 = stac_catalog.load_stac_catalog( + gateway_url, catalog_url=STAC_CATALOG_URL + ) # Both should be valid assert isinstance(catalog1, pystac.Catalog) @@ -476,8 +526,10 @@ def test_multiple_catalog_loads_work(self): def test_catalog_navigation(self): """Test navigating through catalog hierarchy.""" - gateway_url = "https://ipfs-gateway.dclimate.net" - catalog = stac_catalog.load_stac_catalog(gateway_url) + gateway_url = IPFS_GATEWAY_URL + catalog = stac_catalog.load_stac_catalog( + gateway_url, catalog_url=STAC_CATALOG_URL + ) # Get child links child_links = list(catalog.get_child_links()) diff --git a/tests/test_stac_server.py b/tests/test_stac_server.py index 335a069..bc367d0 100644 --- a/tests/test_stac_server.py +++ b/tests/test_stac_server.py @@ -6,14 +6,18 @@ """ import os +import httpx import pytest -import requests from dclimate_client_py.stac_server import ( + ResolvedDataset, resolve_cid_from_stac_server, STAC_SERVER_URL, ) +pytestmark = pytest.mark.integration + + @pytest.fixture(scope="module") def stac_server_url(): """Get STAC server URL from environment or use default.""" @@ -24,23 +28,25 @@ def stac_server_url(): def check_stac_server(stac_server_url): """Check if STAC server is available, skip tests if not.""" try: - response = requests.post( + response = httpx.post( f"{stac_server_url}/search", json={"limit": 1}, timeout=5, + follow_redirects=True, ) response.raise_for_status() - except (requests.ConnectionError, requests.Timeout, requests.HTTPError): + except httpx.HTTPError: pytest.skip(f"STAC server not available at {stac_server_url}") @pytest.fixture(scope="module") def available_dataset(stac_server_url, check_stac_server): """Get an available dataset from the STAC server for testing.""" - response = requests.post( + response = httpx.post( f"{stac_server_url}/search", json={"limit": 10}, timeout=10, + follow_redirects=True, ) response.raise_for_status() features = response.json().get("features", []) @@ -83,10 +89,11 @@ def test_default_server_url_constant(self): def test_server_search_endpoint(self, stac_server_url, check_stac_server): """Test that search endpoint responds.""" - response = requests.post( + response = httpx.post( f"{stac_server_url}/search", json={"limit": 1}, timeout=10, + follow_redirects=True, ) response.raise_for_status() data = response.json() @@ -98,64 +105,66 @@ def test_server_search_endpoint(self, stac_server_url, check_stac_server): class TestResolveCidFromStacServer: """Test the resolve_cid_from_stac_server function.""" - def test_resolve_cid_returns_string(self, stac_server_url, available_dataset): - """Test that resolve returns a non-empty string CID.""" - cid = resolve_cid_from_stac_server( + def test_resolve_cid_returns_dataset(self, stac_server_url, available_dataset): + """Test that resolve returns a result with a non-empty string CID.""" + resolved = resolve_cid_from_stac_server( collection=available_dataset["collection"], dataset=available_dataset["dataset"], server_url=stac_server_url, ) - assert isinstance(cid, str) - assert len(cid) > 0 + assert isinstance(resolved, ResolvedDataset) + assert isinstance(resolved.cid, str) + assert len(resolved.cid) > 0 def test_resolve_cid_no_ipfs_prefix(self, stac_server_url, available_dataset): """Test that returned CID has no ipfs:// prefix.""" - cid = resolve_cid_from_stac_server( + resolved = resolve_cid_from_stac_server( collection=available_dataset["collection"], dataset=available_dataset["dataset"], server_url=stac_server_url, ) - assert not cid.startswith("ipfs://") + assert not resolved.cid.startswith("ipfs://") def test_resolve_cid_valid_format(self, stac_server_url, available_dataset): """Test that returned CID has valid IPFS CID format.""" - cid = resolve_cid_from_stac_server( + resolved = resolve_cid_from_stac_server( collection=available_dataset["collection"], dataset=available_dataset["dataset"], server_url=stac_server_url, ) # IPFS CIDs typically start with these prefixes - assert cid.startswith(("Qm", "bafy", "bafk", "bafz", "bafyr")) + assert resolved.cid.startswith(("Qm", "bafy", "bafk", "bafz", "bafyr")) def test_resolve_cid_with_variant(self, stac_server_url, available_dataset): """Test CID resolution with specific variant.""" if not available_dataset["variant"]: pytest.skip("Test dataset has no variant") - cid = resolve_cid_from_stac_server( + resolved = resolve_cid_from_stac_server( collection=available_dataset["collection"], dataset=available_dataset["dataset"], variant=available_dataset["variant"], server_url=stac_server_url, ) - assert isinstance(cid, str) - assert len(cid) > 0 - assert not cid.startswith("ipfs://") + assert isinstance(resolved.cid, str) + assert len(resolved.cid) > 0 + assert not resolved.cid.startswith("ipfs://") + assert resolved.variant == available_dataset["variant"] def test_resolve_cid_without_variant(self, stac_server_url, available_dataset): """Test CID resolution without specifying variant.""" - cid = resolve_cid_from_stac_server( + resolved = resolve_cid_from_stac_server( collection=available_dataset["collection"], dataset=available_dataset["dataset"], server_url=stac_server_url, ) - assert isinstance(cid, str) - assert len(cid) > 0 + assert isinstance(resolved.cid, str) + assert len(resolved.cid) > 0 def test_resolve_cid_invalid_collection_raises( self, stac_server_url, check_stac_server @@ -199,9 +208,7 @@ def test_resolve_cid_invalid_variant_raises( def test_resolve_cid_connection_error_on_bad_url(self): """Test that connection error is raised for unreachable server.""" - with pytest.raises( - (requests.ConnectionError, requests.exceptions.RequestException) - ): + with pytest.raises(httpx.HTTPError): resolve_cid_from_stac_server( collection="any", dataset="any", @@ -231,10 +238,11 @@ class TestMultipleDatasets: def test_resolve_multiple_datasets(self, stac_server_url, check_stac_server): """Test resolving CIDs for multiple datasets from the server.""" # Get multiple datasets - response = requests.post( + response = httpx.post( f"{stac_server_url}/search", json={"limit": 50}, timeout=10, + follow_redirects=True, ) response.raise_for_status() features = response.json().get("features", []) @@ -267,12 +275,12 @@ def test_resolve_multiple_datasets(self, stac_server_url, check_stac_server): datasets_seen.add(key) try: - cid = resolve_cid_from_stac_server( + resolved = resolve_cid_from_stac_server( collection=collection, dataset=dataset, server_url=stac_server_url, ) - resolved_cids.append((key, cid)) + resolved_cids.append((key, resolved)) except ValueError: continue @@ -281,7 +289,7 @@ def test_resolve_multiple_datasets(self, stac_server_url, check_stac_server): assert len(resolved_cids) >= 1, "Should resolve at least one dataset" - for key, cid in resolved_cids: - assert isinstance(cid, str) - assert len(cid) > 0 - assert not cid.startswith("ipfs://") + for key, resolved in resolved_cids: + assert isinstance(resolved.cid, str) + assert len(resolved.cid) > 0 + assert not resolved.cid.startswith("ipfs://") diff --git a/tests/test_stac_server_listing.py b/tests/test_stac_server_listing.py index 8b8038f..a134d3e 100644 --- a/tests/test_stac_server_listing.py +++ b/tests/test_stac_server_listing.py @@ -1,7 +1,7 @@ """ Unit tests for ``list_available_datasets_from_stac_server``. -These are pure unit tests — ``requests.get`` / ``requests.post`` are mocked, so +These are pure unit tests — ``httpx.MockTransport`` handles all requests, so the tests run offline and don't depend on the public STAC server or the IPFS gateway. Integration coverage (parity with the IPFS walker) lives in ``test_list_datasets_parity.py`` and is gated behind ``--run-integration``. @@ -11,46 +11,48 @@ from typing import Any, Dict +import httpx import pytest -import requests +from dclimate_client_py import stac_server from dclimate_client_py.stac_server import ( list_available_datasets_from_stac_server, resolve_cid_from_stac_server, ) -def _mock_response(payload: Dict[str, Any], status: int = 200): - """Build a stand-in for a ``requests.Response`` that has ``json()`` and - ``raise_for_status()``.""" +_install_mock_client = None - class _Resp: - def __init__(self) -> None: - self.status_code = status - def json(self): - return payload +@pytest.fixture(autouse=True) +def _use_managed_httpx_clients(install_httpx_mock): + global _install_mock_client + _install_mock_client = install_httpx_mock + yield + _install_mock_client = None - def raise_for_status(self): - if status >= 400: - raise requests.HTTPError(f"HTTP {status}") - return _Resp() +def _mock_response( + request: httpx.Request, payload: Dict[str, Any], status: int = 200 +) -> httpx.Response: + return httpx.Response(status, json=payload, request=request) def _install_mocks(monkeypatch, *, collections_body, search_body): - """Patch requests.get/post to return canned bodies based on URL.""" + """Inject canned collection/search responses through the client accessor.""" - def fake_get(url, *args, **kwargs): - assert url.endswith("/collections"), f"unexpected GET {url}" - return _mock_response(collections_body) + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "GET": + assert request.url.path.endswith("/collections"), ( + f"unexpected GET {request.url}" + ) + return _mock_response(request, collections_body) + assert request.method == "POST" + assert request.url.path.endswith("/search"), f"unexpected POST {request.url}" + return _mock_response(request, search_body) - def fake_post(url, *args, **kwargs): - assert url.endswith("/search"), f"unexpected POST {url}" - return _mock_response(search_body) - - monkeypatch.setattr(requests, "get", fake_get) - monkeypatch.setattr(requests, "post", fake_post) + assert _install_mock_client is not None + _install_mock_client(stac_server, handler) SAMPLE_COLLECTIONS = { @@ -192,6 +194,102 @@ def test_falls_back_to_id_parsing_when_properties_missing(monkeypatch): assert "cid" not in variants[0] +def test_partial_properties_use_dataset_hint_and_asset_cid(monkeypatch): + _install_mocks( + monkeypatch, + collections_body={ + "collections": [ + { + "id": "chirps", + "title": "CHIRPS", + "dclimate:types": ["precip-daily"], + } + ] + }, + search_body={ + "features": [ + { + "id": "chirps-precip-daily-final-p05", + "collection": "chirps", + "properties": {"dclimate:dataset_id": "precip-daily"}, + "assets": {"data": {"href": "ipfs://bafy-partial-properties"}}, + } + ] + }, + ) + + variant = list_available_datasets_from_stac_server("https://example.test")[ + "chirps" + ]["variants"][0] + + assert variant["dataset"] == "precip-daily" + assert variant["variant"] == "final-p05" + assert variant["cid"] == "bafy-partial-properties" + + +def test_variant_only_property_keeps_bare_hyphenated_dataset(monkeypatch): + _install_mocks( + monkeypatch, + collections_body={ + "collections": [ + { + "id": "chirps", + "title": "CHIRPS", + "dclimate:types": ["precip-daily"], + } + ] + }, + search_body={ + "features": [ + { + "id": "chirps-precip-daily", + "collection": "chirps", + "properties": {"dclimate:variant": "default"}, + } + ] + }, + ) + + variant = list_available_datasets_from_stac_server("https://example.test")[ + "chirps" + ]["variants"][0] + + assert variant["dataset"] == "precip-daily" + assert variant["variant"] == "default" + + +def test_unknown_collection_uses_explicit_sibling_dataset_hints(monkeypatch): + _install_mocks( + monkeypatch, + collections_body={"collections": []}, + search_body={ + "features": [ + { + "id": "new_coll-precip-daily-final-p05", + "collection": "new_coll", + "properties": {}, + }, + { + "id": "new_coll-precip-daily-prelim-p05", + "collection": "new_coll", + "properties": { + "dclimate:dataset_id": "precip-daily", + "dclimate:variant": "prelim-p05", + }, + }, + ] + }, + ) + + listing = list_available_datasets_from_stac_server("https://example.test") + + assert listing["new_coll"]["types"] == ["precip-daily"] + assert {variant["variant"] for variant in listing["new_coll"]["variants"]} == { + "final-p05", + "prelim-p05", + } + + def test_category_unanimous_only(monkeypatch): """When items in a collection disagree on observation, category is dropped.""" _install_mocks( @@ -272,14 +370,14 @@ def test_resolve_cid_uses_exact_dataset_id_for_prefix_collisions(monkeypatch): }, ) - cid = resolve_cid_from_stac_server( + resolved = resolve_cid_from_stac_server( "ecmwf_era5", "precipitation_total", "finalized", "https://example.test", ) - assert cid == "bafy-era5-precip-finalized" + assert resolved.cid == "bafy-era5-precip-finalized" def test_resolve_cid_rejects_only_prefix_dataset_match(monkeypatch): @@ -336,26 +434,26 @@ def test_resolve_cid_legacy_id_fallback_is_exact(monkeypatch): }, ) - cid = resolve_cid_from_stac_server( + resolved = resolve_cid_from_stac_server( "ecmwf_era5", "temperature_2m", "finalized", "https://example.test", ) - assert cid == "bafy-era5-t2m" + assert resolved.cid == "bafy-era5-t2m" def test_collections_endpoint_error_propagates(monkeypatch): - def failing_get(url, *args, **kwargs): - return _mock_response({}, status=500) + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "GET": + return _mock_response(request, {}, status=500) + return _mock_response(request, {"features": []}) - monkeypatch.setattr(requests, "get", failing_get) - monkeypatch.setattr( - requests, "post", lambda *a, **k: _mock_response({"features": []}) - ) + assert _install_mock_client is not None + _install_mock_client(stac_server, handler) - with pytest.raises(requests.HTTPError): + with pytest.raises(httpx.HTTPStatusError): list_available_datasets_from_stac_server("https://example.test") diff --git a/tests/test_zarr_encryption_ipfs.py b/tests/test_zarr_encryption_ipfs.py index ac67a91..7262510 100644 --- a/tests/test_zarr_encryption_ipfs.py +++ b/tests/test_zarr_encryption_ipfs.py @@ -129,6 +129,8 @@ def test_compute_encoded_size(): @pytest.mark.asyncio +@pytest.mark.ipfs +@pytest.mark.ipfs_rpc async def test_upload_then_read( random_zarr_dataset: tuple[str, xr.Dataset], original_encryption_key: bytes ): diff --git a/uv.lock b/uv.lock index aead06f..58a7bbf 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,10 @@ version = 1 revision = 2 requires-python = ">=3.12" +resolution-markers = [ + "python_full_version >= '3.15'", + "python_full_version < '3.15'", +] [[package]] name = "affine" @@ -131,6 +135,47 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl", hash = "sha256:0287e96f4d26d4149305414d4e3bc32f0dcd0862365a4bddea19d7a1ec38c4fc", size = 109097, upload-time = "2025-09-23T09:19:10.601Z" }, ] +[[package]] +name = "ast-serialize" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/ad/0d70a3a2d6e01968d985415259e8ec7ad3f777903f9b1c1f3c8c44642c60/ast_serialize-0.6.0.tar.gz", hash = "sha256:aadd3ffcf4858c9726bf3515f7b199c7eadbe504f96028e4a87172c0da65a8fe", size = 61489, upload-time = "2026-06-30T20:02:55.555Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/12/3e5f575f156555547c250a8b0d1347517a3a20fc7f4492e9703a69d4f45e/ast_serialize-0.6.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:a7520b672827885bafeae7501f684d14d47d17e5f45256f9df547686cca52264", size = 1177640, upload-time = "2026-06-30T20:02:06.708Z" }, + { url = "https://files.pythonhosted.org/packages/a2/a4/921a9e27951627983b0f368859ea00f8330a551dc0bf4c2fdcb11855a98b/ast_serialize-0.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a14191beec7e0c078d2fc1f6edc0aee88bcd4db9f18e1bc9f8052b559c22dddc", size = 1168111, upload-time = "2026-06-30T20:02:08.366Z" }, + { url = "https://files.pythonhosted.org/packages/00/69/950cf404de7b8782cf95e5c1237e25e2aa46177b287f39f9eeddf481fd6f/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32ef62ec34cf6be20ad77d4799556638fbdf187f3ae10698dfb20ef9f2c89516", size = 1227656, upload-time = "2026-06-30T20:02:09.843Z" }, + { url = "https://files.pythonhosted.org/packages/4c/a8/46f8f6a6479d9d2273980957bb091a506c55f5b95d3c029ee58518a78407/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:13b7769970a39983b0adf2f38917b1cd3b8946f76df045756c3d741bc689f089", size = 1227706, upload-time = "2026-06-30T20:02:11.367Z" }, + { url = "https://files.pythonhosted.org/packages/b7/b9/9ac415bda0a40e49eab8fea3b2741c19c98bb84d57d62c4cfc6230eb67be/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6f7a408601bb3edaefb3bc67a4c01f5235e3253653b6a5729a2ee2382b35341c", size = 1431705, upload-time = "2026-06-30T20:02:12.737Z" }, + { url = "https://files.pythonhosted.org/packages/e5/06/8807115d441444879f7561b5eede5ac18fc80392f11826d61ccf31f503b1/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8670bfa51208a2c0c8d138928e40e998fab158f9200d53bb80c088b5b8eda7b8", size = 1249533, upload-time = "2026-06-30T20:02:14.571Z" }, + { url = "https://files.pythonhosted.org/packages/3e/c0/c2ba82ef9618650357d9421a1fdb27ffec862a7f57e8e2de82a3ccd11e12/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4826809eb8597a8cd59fd924b6d7c285b8969a1e0007e2cb652cab62376270f", size = 1252619, upload-time = "2026-06-30T20:02:16.219Z" }, + { url = "https://files.pythonhosted.org/packages/0f/a7/fa31d52dd4102cede29fb9634e98d214129b2783b4f95528c6dc6a8f6587/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:577a6c189068686869f5f1ddc38363f3ae1808a4753b577266f9202071a7bb66", size = 1242983, upload-time = "2026-06-30T20:02:17.813Z" }, + { url = "https://files.pythonhosted.org/packages/b1/20/ddf742b5ad3c4bafd3466f2265037cfd99bc1b9a5ee46a5d58c90d523242/ast_serialize-0.6.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:085de7f62dc9cc247eb01e965a362707d1d90b1d89a82c5bf78301a60a3c417b", size = 1296148, upload-time = "2026-06-30T20:02:19.146Z" }, + { url = "https://files.pythonhosted.org/packages/24/cb/9f6f217cce8b3b632c5568b478d195a35e79dce4dbe309438cb89ba6ea4f/ast_serialize-0.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9f8a8b78b13173de6a9ec22111d9be674874cd5bdccda04f14ae5ebc2bef403a", size = 1403826, upload-time = "2026-06-30T20:02:20.696Z" }, + { url = "https://files.pythonhosted.org/packages/2d/f8/9d16d4f0107a183924425cc0e7618d8bf76f96b45afa9ff19f924ed1ad57/ast_serialize-0.6.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f2ff3baffc3a29c1f15bc9098aa0c09763410262d5e6cef42116f7356c184554", size = 1502943, upload-time = "2026-06-30T20:02:22.034Z" }, + { url = "https://files.pythonhosted.org/packages/80/dd/bbc1c38756350dddf7e24acae1c9482ef42051c267417e019aecc1ed4075/ast_serialize-0.6.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:0067b25fce104eaae5b88383de9ab803faeb671831e14ca698b771b356e2600f", size = 1497632, upload-time = "2026-06-30T20:02:23.517Z" }, + { url = "https://files.pythonhosted.org/packages/42/7e/9daffefcf5b97e6bb4c3e0b3c024c1aee9722f23d3cf7cd2ff80d6fb4a40/ast_serialize-0.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c617417f9cbb0cb144f6283c3cbe0d2e0f01beaf9f608f662b21191058a626ec", size = 1448858, upload-time = "2026-06-30T20:02:24.889Z" }, + { url = "https://files.pythonhosted.org/packages/e5/1f/f9baaab81a677ea0af7d2458cac2f94ebcc85958f8a3c15ba9d9e5dab653/ast_serialize-0.6.0-cp314-cp314t-win32.whl", hash = "sha256:5337cb256dcea3df9288205213d1601581536526b8f4da44b6974f1180f3252a", size = 1052600, upload-time = "2026-06-30T20:02:26.263Z" }, + { url = "https://files.pythonhosted.org/packages/9e/1f/41b535866519512d8cf6669cb2cff7823b7672bb6279c0333b4ff89d7d9f/ast_serialize-0.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d947e45cafc4b09bd7528917fa84c517654a43de173c79785574b7b3068ac24", size = 1095570, upload-time = "2026-06-30T20:02:27.639Z" }, + { url = "https://files.pythonhosted.org/packages/50/64/e472fe3e3a2d33d874b987e8518aedf24562919e3b6161a4fa1797e89c0f/ast_serialize-0.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:6e15ec740436e1a0d62de848641abe5f3a2f89a7f94907d534795ac91bbacf14", size = 1067267, upload-time = "2026-06-30T20:02:28.949Z" }, + { url = "https://files.pythonhosted.org/packages/52/19/ac8348ae8711c9b5ae834634f635780cab62a0f5e6f988882e048b89c2ae/ast_serialize-0.6.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:093cb8bb91b720d8523580498d031791bb1bbaa048599c3d21085d380e11a596", size = 1185367, upload-time = "2026-06-30T20:02:30.427Z" }, + { url = "https://files.pythonhosted.org/packages/c1/f6/ec7ec652c51db77c2f61d8573338e13e4704303265ccc658cb4031d9f354/ast_serialize-0.6.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:e61580a69faf47e3689795367ed211f2a10fd741478cc0f36a0f128793360aad", size = 1178657, upload-time = "2026-06-30T20:02:31.964Z" }, + { url = "https://files.pythonhosted.org/packages/6f/02/613a7534a41d0122f37d1e0c64aa8ac78bfb831f8c92f6db057a311abb3c/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:305802f2ce2a7c4e87835078ea85c58b586ddda8095b92fe2ead9364ae19c80a", size = 1238620, upload-time = "2026-06-30T20:02:33.664Z" }, + { url = "https://files.pythonhosted.org/packages/4d/21/087957bba486242afc52f49b2d9e21c9dad00289356cf9efe67084015a9d/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c7b8b8f0c42f752ea00b2b7d7c090b3f80d9c1c5c75cadf16423790a0cc74081", size = 1236075, upload-time = "2026-06-30T20:02:34.936Z" }, + { url = "https://files.pythonhosted.org/packages/82/04/78128bbb170071c2c72a210a181f1c00e11cc1cec60a8beef747b07f9201/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd5b91b9e6f2356ace3a556963b0cd783b395fbbb0bb17b4defc283415466e77", size = 1441348, upload-time = "2026-06-30T20:02:36.245Z" }, + { url = "https://files.pythonhosted.org/packages/64/64/62fb99d6faf199b4c3e5b08a07136e9a0d7664bb249c6de3670e5b63e9b6/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4d6ef91590258ada18909b9caea344dac4de2013906b035473cd674a43f4b790", size = 1258580, upload-time = "2026-06-30T20:02:37.53Z" }, + { url = "https://files.pythonhosted.org/packages/ca/87/b4d6c38e0ccd5e85dc54cecdf933a152c60b28fe5d993a6d8a72fa6d5896/ast_serialize-0.6.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcbed41e9386059fc0261d602445ede0976c2ecec2939688bcbcb9ed0b6f28b7", size = 1261693, upload-time = "2026-06-30T20:02:39.123Z" }, + { url = "https://files.pythonhosted.org/packages/0e/4b/3676ca2191f39bafb75f93f99b2f429ec464586158fece2165f3572805dc/ast_serialize-0.6.0-cp39-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:cdc4e6f930b9090c2f92c9036ad12ffb8e6e44d4a5ba06f1458a05d60f203f7b", size = 1252517, upload-time = "2026-06-30T20:02:40.511Z" }, + { url = "https://files.pythonhosted.org/packages/f3/58/494ef8c4b4acb2f4a265ac934caf45f792a08fe27d6b853de35ad991941a/ast_serialize-0.6.0-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:897ac47b5637be41c0c07061c8a912fafa967ef1dc73fa115e4bfa70882a093b", size = 1304843, upload-time = "2026-06-30T20:02:41.961Z" }, + { url = "https://files.pythonhosted.org/packages/b1/f2/13736d920ab3d49bbee80ef1a277dd7b7aaf3b3545efd9d2a8114fe05525/ast_serialize-0.6.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c4af9a1386166e40ed01464991806f89038a2d89782576c7774876fa77034e32", size = 1413698, upload-time = "2026-06-30T20:02:44.179Z" }, + { url = "https://files.pythonhosted.org/packages/a8/5a/e046f3899e2acba4677d7427b76431443a1aa1a0e583dfb05b55b69d55cf/ast_serialize-0.6.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:c901adbd750029b9ac4ad3d6aa56853e0ad4875119fbf52b7b8298afc223828b", size = 1512209, upload-time = "2026-06-30T20:02:45.584Z" }, + { url = "https://files.pythonhosted.org/packages/cc/c7/e42aaca7bb2d22a7c06d5a8c7930086c5a334e93d716e6fa5e6647a4515f/ast_serialize-0.6.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae22a366b752ab4496191525b78b097b5b72d531752e3c1dd7e383a8f2c8a1a", size = 1508464, upload-time = "2026-06-30T20:02:46.942Z" }, + { url = "https://files.pythonhosted.org/packages/95/93/5524a3dc6c3f593de3228ed9cbef73afa047625b7000ec21b7f58e6eb4d4/ast_serialize-0.6.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4ed29121da8b3fdc291002801a1de0f76248fa07dce89157a5f277842cf6126e", size = 1457164, upload-time = "2026-06-30T20:02:48.294Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c0/36a6ffb4d653cf621427b4c4928671f53ad800c453474de2b82564a44ad9/ast_serialize-0.6.0-cp39-abi3-pyemscripten_2026_0_wasm32.whl", hash = "sha256:b1dac4e09d341c1300ba69cdcbe62867b32a8c75d90db9bf4d083bec3b039f0b", size = 863014, upload-time = "2026-06-30T20:02:49.742Z" }, + { url = "https://files.pythonhosted.org/packages/09/c7/7d5ad8b49e1278e1c2a1e0274bd7850560b3f09313aa00c13bc8d5544792/ast_serialize-0.6.0-cp39-abi3-win32.whl", hash = "sha256:82c312a7844d2fdeb4d5c48bd3d215bf940dafd4704e1a9bcf252a99010a99b1", size = 1063165, upload-time = "2026-06-30T20:02:50.98Z" }, + { url = "https://files.pythonhosted.org/packages/47/ae/6710c14ecb276031cf10249f6adf5a59e2d3fdb3b5183bd59f70524067ee/ast_serialize-0.6.0-cp39-abi3-win_amd64.whl", hash = "sha256:113b58346f9ceb664352032770caca817d4a3c86f611c6088e6ef65ddaa70f0e", size = 1101444, upload-time = "2026-06-30T20:02:52.554Z" }, + { url = "https://files.pythonhosted.org/packages/66/40/c53deb2cd0c9b0fb636d24d9f40924cf2e65028e6b20b10cd5c1eeb2c730/ast_serialize-0.6.0-cp39-abi3-win_arm64.whl", hash = "sha256:ccd132fe8db56f61fe743b1f644d01b8d65b83248a8da506f3132bda86d6ed5e", size = 1072965, upload-time = "2026-06-30T20:02:54.097Z" }, +] + [[package]] name = "attrs" version = "25.3.0" @@ -632,21 +677,21 @@ wheels = [ [[package]] name = "dclimate-client-py" -version = "0.5.11" +version = "0.6.0" source = { editable = "." } dependencies = [ { name = "aiobotocore" }, + { name = "aiohttp" }, { name = "geopandas" }, { name = "httpx" }, + { name = "multiformats" }, + { name = "numcodecs", extra = ["crc32c"] }, { name = "numpy" }, { name = "opentelemetry-api" }, { name = "pandas" }, { name = "py-hamt" }, - { name = "pyarrow" }, { name = "pycryptodome" }, { name = "pystac" }, - { name = "python-dotenv" }, - { name = "requests" }, { name = "rioxarray" }, { name = "s3fs" }, { name = "scipy" }, @@ -661,7 +706,11 @@ dev = [ { name = "pre-commit" }, { name = "ruff" }, ] +examples = [ + { name = "python-dotenv" }, +] testing = [ + { name = "mypy" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, @@ -670,33 +719,35 @@ testing = [ [package.metadata] requires-dist = [ - { name = "aiobotocore" }, - { name = "geopandas" }, + { name = "aiobotocore", specifier = ">=2.13.0" }, + { name = "aiohttp", specifier = ">=3.9.0" }, + { name = "geopandas", specifier = ">=1.0.0" }, { name = "httpx", specifier = ">=0.27.0" }, + { name = "multiformats", specifier = ">=0.3.1" }, + { name = "mypy", marker = "extra == 'testing'", specifier = ">=1.14" }, + { name = "numcodecs", extras = ["crc32c"], specifier = ">=0.14,<0.16" }, { name = "numpy", specifier = ">=2.1.3" }, { name = "opentelemetry-api", specifier = ">=1.30.0" }, - { name = "pandas" }, + { name = "pandas", specifier = ">=2.2.0" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=4.1.0" }, - { name = "py-hamt", specifier = ">=3.4.1" }, - { name = "pyarrow" }, + { name = "py-hamt", specifier = ">=3.5.0" }, { name = "pycryptodome", specifier = ">=3.21.0" }, { name = "pystac", specifier = ">=1.10.0" }, { name = "pytest", marker = "extra == 'testing'" }, { name = "pytest-asyncio", marker = "extra == 'testing'", specifier = ">=1.3.0" }, { name = "pytest-cov", marker = "extra == 'testing'" }, { name = "pytest-mock", marker = "extra == 'testing'" }, - { name = "python-dotenv", specifier = ">=1.0.0" }, - { name = "requests" }, - { name = "rioxarray" }, + { name = "python-dotenv", marker = "extra == 'examples'", specifier = ">=1.0.0" }, + { name = "rioxarray", specifier = ">=0.15.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.9.5" }, - { name = "s3fs" }, - { name = "scipy" }, - { name = "shapely" }, + { name = "s3fs", specifier = ">=2024.6.0" }, + { name = "scipy", specifier = ">=1.12.0" }, + { name = "shapely", specifier = ">=2.0.0" }, { name = "x402", extras = ["evm", "httpx"], specifier = ">=2.1.0" }, { name = "xarray", specifier = ">=2025.3.0" }, { name = "zarr", specifier = ">=3.0.8" }, ] -provides-extras = ["testing", "dev"] +provides-extras = ["testing", "dev", "examples"] [[package]] name = "deprecated" @@ -1041,6 +1092,68 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" }, ] +[[package]] +name = "librt" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/2f/3908645ddddab7120b46295e541ead308109fa48dbec7d67d7a778870d60/librt-0.13.0.tar.gz", hash = "sha256:1d2a610c14ac0d0750ee0a3ab8548e83155258387891caaca04def4bf7289781", size = 211402, upload-time = "2026-07-08T12:26:29.834Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/f4/b2933ddae222dac338476abb872641169a5cfed2c2bb5444a5b07b32b0c3/librt-0.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:30536798f4504c0fad0885b1d371b0539abb081e4570c9d7c641cb51141b49f0", size = 150990, upload-time = "2026-07-08T12:25:02.42Z" }, + { url = "https://files.pythonhosted.org/packages/90/ef/db98f744ca50e6efc9c95c70ee49b77aefac31f6a3fc7c83754a42d6a74f/librt-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:93d24ebb82aa4420b1409c389e7857bc35bd0b668007ac8172427d5c73cc8cc5", size = 155238, upload-time = "2026-07-08T12:25:03.681Z" }, + { url = "https://files.pythonhosted.org/packages/03/e7/a197e7bc72baf2c61ce7fdc6906a5054dc05bd8da0819aa894e4857bf87e/librt-0.13.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb8a1adce42d8b75485a5d56a9623a50bcab995b6079f1dac59fc44034dd93d9", size = 503073, upload-time = "2026-07-08T12:25:05.049Z" }, + { url = "https://files.pythonhosted.org/packages/f8/e7/7887712e27da7c1ab80fcabb1de6eb24243964f6557cae530d4b70706dbd/librt-0.13.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:0763ca2ab66058174f9dee426dc64f5e0a89c24a7df8d3fe3f1836c04e25de4b", size = 496528, upload-time = "2026-07-08T12:25:06.26Z" }, + { url = "https://files.pythonhosted.org/packages/94/f0/f2283385bb6b950b26a1410f4ce51ec27231e0b3a4b925c46366d218b198/librt-0.13.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b222493da6e7b6199db9bd79502436cf5a27da3c1f7fa83c7e285444fc93fd03", size = 531786, upload-time = "2026-07-08T12:25:07.658Z" }, + { url = "https://files.pythonhosted.org/packages/36/11/69ac3b54766ffba5fd7e5acebfb048d66dbe1f9f2d14516c2b3edc59cf87/librt-0.13.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fadc63331f4388c3dc90090448f682a7e9feafc11481391c1e94f2f907a3976e", size = 524393, upload-time = "2026-07-08T12:25:09.121Z" }, + { url = "https://files.pythonhosted.org/packages/61/5f/d72f95fd444a926a3c14b4e24979474116988dd57a45be242077c45d3c22/librt-0.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:70d9c62a4cffd9f23396cd5ef93fc5d11b31596b9b7d6306074abe3d5fcf09bd", size = 543026, upload-time = "2026-07-08T12:25:10.459Z" }, + { url = "https://files.pythonhosted.org/packages/c4/08/dcd9993ad192737a004ba263d549f8ea605b326b952e7d6205c7d4170b76/librt-0.13.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:66c0e7e6b02a155576df2c77ec933a70b72da726e248c494abf690923e624348", size = 546829, upload-time = "2026-07-08T12:25:11.716Z" }, + { url = "https://files.pythonhosted.org/packages/96/d5/6d9bb2f54e4109a956b7128836529653eb9d740f784bc47ed10a02c1000e/librt-0.13.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:ac04bcd3328eb91d99dfedf6a60d9c1f15d3434e6f6daf922f0420f7d90b85c7", size = 535700, upload-time = "2026-07-08T12:25:13.144Z" }, + { url = "https://files.pythonhosted.org/packages/8c/f2/10946922503858a359492fa27f13e86228bde702116a740ac7b3cd185f24/librt-0.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:db327e7271e653c32040b85ae6188059c924b57d7e1e29f935523fa017cd4e82", size = 573566, upload-time = "2026-07-08T12:25:14.336Z" }, + { url = "https://files.pythonhosted.org/packages/48/a8/94f00e3c99479a18088af3685ea016c42f3c7d5d1964d8dbb40c08d7f1aa/librt-0.13.0-cp312-cp312-win32.whl", hash = "sha256:860bd1d8ba48456ce08feaf8d343a8aaeb2fa086f2bcaa2a923fa3f7a3ff9aa3", size = 106099, upload-time = "2026-07-08T12:25:16.159Z" }, + { url = "https://files.pythonhosted.org/packages/c9/7b/2da9c74c1ed25a89cc4e1c8e007ea2eb4a0f1fafa3e70d757fe3242c5c5c/librt-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:e54a315caf843c8d77e388cadc56ea9ded569935ee2d2347d7ea94992e5aa6fa", size = 126934, upload-time = "2026-07-08T12:25:17.275Z" }, + { url = "https://files.pythonhosted.org/packages/d0/65/aead61bbf3b5358593f9d4779d2a0e88eaf6ec191a6342dde36dd1df6371/librt-0.13.0-cp312-cp312-win_arm64.whl", hash = "sha256:c718e99a0992127af84385378460db624103b559ab260435abcfe77a4e4ed1c1", size = 112236, upload-time = "2026-07-08T12:25:18.425Z" }, + { url = "https://files.pythonhosted.org/packages/67/3b/18e7b63255297a2bdc9c25c8d6d4ca8eca9f63aceb1252c0f7427ac7099e/librt-0.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a468951af16155824e88bdd8326ebe5bdb371f3ec0ac04642994b98201d914f3", size = 151027, upload-time = "2026-07-08T12:25:19.638Z" }, + { url = "https://files.pythonhosted.org/packages/4d/68/e2248452c00d1a03b45fee1752cdc8f790a476efd2402b75181da88a9e61/librt-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ae01d8512cc17079e53425635327dbf3f7ff57a42c00dec348bf79791c56444c", size = 155152, upload-time = "2026-07-08T12:25:20.851Z" }, + { url = "https://files.pythonhosted.org/packages/0e/16/52b1c99bf19057a062aac39c900cbb81499f6f75d6c537c14463d247ba78/librt-0.13.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32c26893cd085c1efe83219e78d866da23fb20a066101b8f68210004361d224c", size = 502499, upload-time = "2026-07-08T12:25:22.055Z" }, + { url = "https://files.pythonhosted.org/packages/9f/54/b811151805c795f55e0dedee6ec687b75f9982a8105d240ea3910737a77b/librt-0.13.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5929da1981a46bcf4b28b1b9499905f0ff58e2419da402a048234e9783acbc4b", size = 496108, upload-time = "2026-07-08T12:25:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/8f/f8/094d6b2bd93f3fdaa54db54cc788c4a365333bddad65ab02e04da0b1d004/librt-0.13.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94b85d664d777bab6c0d709416cb42938251fda9e221b79e3a2215d85df5f4f9", size = 531576, upload-time = "2026-07-08T12:25:24.648Z" }, + { url = "https://files.pythonhosted.org/packages/2e/40/541733d5755824f968f7ec39d78ffbd75d145964157ae5e69a09ec6d7326/librt-0.13.0-cp313-cp313-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:531b2df3e9fe96b1fcf73a6d165921e4656be5f58d631d384ebce344298368db", size = 524390, upload-time = "2026-07-08T12:25:25.898Z" }, + { url = "https://files.pythonhosted.org/packages/c6/b5/255673cfdbf5ba663339d36cd863c897289ab4337577e19f9405ce059f36/librt-0.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:109b84a9edf69ad89dc1f66358659e14a031baca95e3e5b0060bd903ede8efd6", size = 543053, upload-time = "2026-07-08T12:25:27.436Z" }, + { url = "https://files.pythonhosted.org/packages/9e/11/ab5005e9c9850710f21e354201bf090646349d3fabf5f951eaf70235729e/librt-0.13.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1304368a3e7ffc3e9db986796cc5326fdb5943a3567ecc137cff318e4240c0e7", size = 546387, upload-time = "2026-07-08T12:25:28.65Z" }, + { url = "https://files.pythonhosted.org/packages/a2/04/a5d7ce1d1df1afd15ca283dcdf7530ac073e12d69ae8c40879dda96f7868/librt-0.13.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e4f9b472e7d308d94b62c801982065661158c6ed02790d6c7ddb4337cea0f9c1", size = 535970, upload-time = "2026-07-08T12:25:30.171Z" }, + { url = "https://files.pythonhosted.org/packages/5a/76/927e267a6daa290174ac281b23c9804c8829b042ade9c6f24a065f540958/librt-0.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9f836c37478f167a81200d8c8b2c920a22224564bed2c23d7aeec760965c367a", size = 573582, upload-time = "2026-07-08T12:25:31.507Z" }, + { url = "https://files.pythonhosted.org/packages/10/24/b6c5213efe39c19f9e13605644d0cf063b4ddaa33ac2e45b088e23a70e2e/librt-0.13.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:4000d961ff9598ac6ea603c6c836a5ed49bc205ade5fc378b998dfe1e2c36628", size = 82189, upload-time = "2026-07-08T12:25:32.675Z" }, + { url = "https://files.pythonhosted.org/packages/4c/00/d29736be177a906ac0b84a5b04b4fbfa22c776dc2f366de4172b0f968c08/librt-0.13.0-cp313-cp313-win32.whl", hash = "sha256:79e44cff71750d299d61a678e49995b0d5935a9cda238c2574daeca3ba536927", size = 106193, upload-time = "2026-07-08T12:25:33.692Z" }, + { url = "https://files.pythonhosted.org/packages/c8/ac/aff6fb45393cb8912f39dfb156ef6b2d1cadb207ff465fc8f66141054be8/librt-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:54dab44a847d5ad1acd05c8a83fe518ae685516ecf4d3f7cc6e3df2a66767650", size = 126962, upload-time = "2026-07-08T12:25:34.769Z" }, + { url = "https://files.pythonhosted.org/packages/d9/3a/d68cb2b334d53fd30fac81d3a489ce4ba0d9506f4df43fcf676b68352b19/librt-0.13.0-cp313-cp313-win_arm64.whl", hash = "sha256:d4cb6fbfdf874340ab5e51450753c0f817b6958a3621125ee695bbc3de866566", size = 112127, upload-time = "2026-07-08T12:25:35.981Z" }, + { url = "https://files.pythonhosted.org/packages/7b/66/f49ae0d592bd45b6941e9a8bafcb6a87cddcd501ee7874707e767f01b585/librt-0.13.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:25218d94b1d2cbc0ba1d8a3f9dc9af578d9646e5ed16443a70cde1dfdcce6d71", size = 149818, upload-time = "2026-07-08T12:25:37.203Z" }, + { url = "https://files.pythonhosted.org/packages/3d/50/51c76d74014d04fb95b6506d286808984b78a2f7a41039094e6b2194ac48/librt-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f26629539d4893c2957a16c41bb058e1e135c1f150f6a2e25ed047f64cf3f5c6", size = 154071, upload-time = "2026-07-08T12:25:39.399Z" }, + { url = "https://files.pythonhosted.org/packages/b8/fe/f19b0f5f82d5a1f2da736586bc840abd00ce07d6388136ae80b7333883fc/librt-0.13.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a4517d47b2b8af26975a406fba7d314de9696d864252e0257c6ea90238cfe27f", size = 494168, upload-time = "2026-07-08T12:25:40.641Z" }, + { url = "https://files.pythonhosted.org/packages/94/bc/b8550c75775127fd31a5f20e8775997f7b527ad661fc8ddccd7497c064f7/librt-0.13.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:f19e181de5b3a1148bb3420b8c4b0b0ea0fce6950099724ad151d6cea5acc180", size = 491054, upload-time = "2026-07-08T12:25:41.905Z" }, + { url = "https://files.pythonhosted.org/packages/30/14/4d0204867623df3f33f86efd3d3692ba5e01321443f4d6eab35a22697618/librt-0.13.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22034924f5b42d5a56371cf271771bfeaabf235a7a8b6264bef2d20013f786c6", size = 523006, upload-time = "2026-07-08T12:25:43.327Z" }, + { url = "https://files.pythonhosted.org/packages/19/0a/c45fc9a260934696bace1ac5df1e148ac92bd71767aee3bf7cd7a4534f4c/librt-0.13.0-cp314-cp314-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c7897db4e95e22468bdda33d8e012ceacd0182abf001e6389d763f0def6286b9", size = 515058, upload-time = "2026-07-08T12:25:44.541Z" }, + { url = "https://files.pythonhosted.org/packages/13/0a/50c5ce45b326854ef8fa6ae4c36cf5142e5c55315eaf9e51d0ae73ac4da3/librt-0.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1ce61b3746545029d4f5c17d6bd74b676254ad98433086c846ffb5e8fa73f007", size = 534025, upload-time = "2026-07-08T12:25:45.825Z" }, + { url = "https://files.pythonhosted.org/packages/89/2d/08c413c8f93fc13b8103624fce38e5caa86cd08cbbc8465870ab287af54b/librt-0.13.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:46c330e82565962c761dbce7941be2cff7db674ee807455a8d0cadc5f9b759b0", size = 540557, upload-time = "2026-07-08T12:25:47.059Z" }, + { url = "https://files.pythonhosted.org/packages/b3/c1/93af71fb4a364952210051811dd4e40174e79656b050c89cacac18af3330/librt-0.13.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:375f5af8f99cbaa99dd293af986e3d57caabc9ba81a5d3f021603764854197a1", size = 523201, upload-time = "2026-07-08T12:25:48.392Z" }, + { url = "https://files.pythonhosted.org/packages/c1/6e/9766f07b676a4889d9f8bc2864e9ba5fff165653143ef4dda7df6aa34d16/librt-0.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9320d34c3376ae204b2cd176e8d4883a013934e0aef822f1aed9c536490c275d", size = 565740, upload-time = "2026-07-08T12:25:49.678Z" }, + { url = "https://files.pythonhosted.org/packages/a2/1e/664e3472ce2b6e10e9b83f29d4a36eb982ff6b5a169ae7567bba3a4c4ff5/librt-0.13.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:9af313c66157a69dc69ea0059a66961692250e0dc95af9c385a48ffb770a0d16", size = 81611, upload-time = "2026-07-08T12:25:50.857Z" }, + { url = "https://files.pythonhosted.org/packages/2f/d4/8582a4d65e2234673685e07309d02c230b28a85724eb0acbf13f019b7f6e/librt-0.13.0-cp314-cp314-win32.whl", hash = "sha256:f2a7253458e34f33543551394ae4fe104b497ec2a65ac266074de64c1df82e37", size = 100106, upload-time = "2026-07-08T12:25:52.03Z" }, + { url = "https://files.pythonhosted.org/packages/63/ce/0cb99efe6086b46cd985dc26672166fae312a239690e75871f7fafbd3fc5/librt-0.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:a3dfe4edf10e8ed7e55b026a8bfc2c2a8704218b659cd4bffdf604fab966dc39", size = 121209, upload-time = "2026-07-08T12:25:53.166Z" }, + { url = "https://files.pythonhosted.org/packages/26/85/4f3ccb083a3c9b0d42e223acdb3c3f507953324a59cdcab4826e8e2e3b89/librt-0.13.0-cp314-cp314-win_arm64.whl", hash = "sha256:68a5faee4bba381cb93b5961f684a514cf0053cb92308ff9c792c2fea0b174c6", size = 106404, upload-time = "2026-07-08T12:25:54.253Z" }, + { url = "https://files.pythonhosted.org/packages/b2/77/333191499538c8e8189de7a4cba8e6f49ee949fd6d6e6324b21fd1522466/librt-0.13.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a38fb81d8376dfa2f8963b265fec07637802b0d01e2a127c19c66cb070fb24f5", size = 159231, upload-time = "2026-07-08T12:25:55.432Z" }, + { url = "https://files.pythonhosted.org/packages/7a/9e/2aa83758f22c278b837a1d8025898434ce2b8bff36678d5330ecaef56dff/librt-0.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d4c8d9bd5abce34b2e75edb3bf37ab0f34e49b1f915a40ae8468eb7c85bc5b46", size = 161300, upload-time = "2026-07-08T12:25:56.585Z" }, + { url = "https://files.pythonhosted.org/packages/bb/c0/86791e936553ca763d6b3c2fb4d31d596cd00e14fa631c283a40ba01559a/librt-0.13.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:387e2f1d27e89bffe0d3f520f0da0662c973fd607ca16c1808f8a5085419485e", size = 582056, upload-time = "2026-07-08T12:25:58.144Z" }, + { url = "https://files.pythonhosted.org/packages/a8/d3/a9ec15984a185e000c4d2a16ba28bd623124ad4c38a10974c7ff78e3a893/librt-0.13.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:4f6db193d2e5e0ed60359b9a5a682cd67205d0d3b1e459a867dd4b5c4e7eaa7a", size = 562758, upload-time = "2026-07-08T12:25:59.544Z" }, + { url = "https://files.pythonhosted.org/packages/3c/af/dbe36b78b19c06a55097f99305e4ea9458e2273e6ae16a3cbecaad7ee978/librt-0.13.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0d38604854e8d22faadf683ec6c02bb0f886e2ba56ef981a1c36ee275f21ea22", size = 602095, upload-time = "2026-07-08T12:26:00.991Z" }, + { url = "https://files.pythonhosted.org/packages/2a/a8/2966891b4dd2830f5203fbee92ac2c4947653a2390ba73dfa44244fad025/librt-0.13.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:371f7ce73026815dafd51c50ce38416e91428b28c4b2ec97cd39271164b0045c", size = 593452, upload-time = "2026-07-08T12:26:02.352Z" }, + { url = "https://files.pythonhosted.org/packages/61/f5/4df8bfc8405ecf8c0d525b4d69636f694bdd8620b313ec8b76e54a5926cc/librt-0.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3aaedf52171bee90860704c560bc798fe83b76247df47568e0197e9b13c735a0", size = 623729, upload-time = "2026-07-08T12:26:04.294Z" }, + { url = "https://files.pythonhosted.org/packages/d6/13/9ac202dffc8db06f75d06c08c2f9f6ff054be67d21272dcc078fa1cc0c57/librt-0.13.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:96bad8725a4f196a798366c25ce075d1f7543a4ec045ffc13e6a7ec095cdab04", size = 617077, upload-time = "2026-07-08T12:26:05.845Z" }, + { url = "https://files.pythonhosted.org/packages/6e/f0/ebe38610716aee5cb28efd95089bb90192096179802779381e1c5dcf239c/librt-0.13.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:6bf6a559ffe4a93bbea6cf31ddf01a7fd9ba342ef51f27beb178e318b74acd61", size = 599561, upload-time = "2026-07-08T12:26:07.21Z" }, + { url = "https://files.pythonhosted.org/packages/4f/5c/c2e72e236fff7abc716d5b1753b8b8cd3ea85ac46fe17d2e7c51d4e1c723/librt-0.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:301067672387902c55f94b51d5022304b36c966ea9fe1f21caab99a9bef487c9", size = 645511, upload-time = "2026-07-08T12:26:08.562Z" }, + { url = "https://files.pythonhosted.org/packages/0c/99/6203ce619dee940d6bfbe099ec3fe4be00a68e9d60f70abf906cf124fe66/librt-0.13.0-cp314-cp314t-win32.whl", hash = "sha256:5fdcf34f86de8fb66d7dc7589f96ba91c4aa46671200d400e6fd6f109a483f18", size = 104357, upload-time = "2026-07-08T12:26:09.828Z" }, + { url = "https://files.pythonhosted.org/packages/52/dd/843b6314087c41657c7036d7914d8f294bdf9b580aa8513ea0588c8e9a3d/librt-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:260c33e92263fa629b4f6d3c51967a1c2158fe6c33237aaa3ebeac586b085259", size = 126998, upload-time = "2026-07-08T12:26:10.975Z" }, + { url = "https://files.pythonhosted.org/packages/5f/5d/3dcec2884ba1b0806d1408612555c38dd5d68e90156b59f75f6e36435c3a/librt-0.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2f281549a4c52ac7bb97997f14353f8bd0e53a34ca0dad1c905cfd0b4a58ae99", size = 110771, upload-time = "2026-07-08T12:26:12.303Z" }, +] + [[package]] name = "markdown-it-py" version = "3.0.0" @@ -1214,6 +1327,60 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f3/9b/c21a9c1d5ea4847989f1eb00e3147e38e79aaea7c4b4d1cbd4f1afae9740/multiformats_config-0.3.1-py3-none-any.whl", hash = "sha256:dec4c9d42ed0d9305889b67440f72e8e8d74b82b80abd7219667764b5b0a8e1d", size = 17153, upload-time = "2023-12-18T21:35:21.171Z" }, ] +[[package]] +name = "mypy" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ast-serialize" }, + { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/12/af/4e516a05d3ca2eb9283e9ec45b2c02225c1514dd6da49fd3c9eaa6639370/mypy-2.3.0.tar.gz", hash = "sha256:465965d41cd9a2726694e983e8ce7113259327bec798115d1e1dfa2a52fb666e", size = 3988104, upload-time = "2026-07-13T11:34:53.387Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/94/0e7e592619e2133596a47cdd642534b0456545c218430bd3b9d8fefdd1b1/mypy-2.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d53fc67b9d28a43c6199077f49fea0f05839e36cf6158500331c9549225e5a5", size = 15026523, upload-time = "2026-07-13T11:34:49.206Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/1e1731df090a857df2807177a4626863e5ac0f0256513c35780efe53986f/mypy-2.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fbc00cee7bdbb9291979ddc9d08034a29dfcda4932628c9bbc28c1edd589df0c", size = 14032189, upload-time = "2026-07-13T11:33:57.168Z" }, + { url = "https://files.pythonhosted.org/packages/44/95/cab921f4a806e171f34113e6181dd23c55358ccf6a80741269ef594a410e/mypy-2.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:04e617030eca5221909c8b7d8d7fd1c637948199aa2100b2ad9813feb07e1491", size = 14198696, upload-time = "2026-07-13T11:32:12.767Z" }, + { url = "https://files.pythonhosted.org/packages/66/80/e6d008bb19fe446e3662d85e0e2717bf9f2d611a2164fb29d6e067dbf46c/mypy-2.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:56c184d2c20ca6b6378d58d1960270a767f41f5e44acbbd27f05effef4f4e1d7", size = 15286904, upload-time = "2026-07-13T11:34:27.594Z" }, + { url = "https://files.pythonhosted.org/packages/db/83/94397c9293608a364aa03e8084fb34ede4ae976a260384b9b52929308135/mypy-2.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3961a4a34b05f7c74b0f05aa51fbfe99a2d1e126038df40318d15c8f558b7ef3", size = 15528342, upload-time = "2026-07-13T11:34:07.819Z" }, + { url = "https://files.pythonhosted.org/packages/cf/96/d8b37d819adec6cfccfb1fd3afc1735d94717ddeafb45536db9c6943e09b/mypy-2.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:b1942b9314d4c784b8ea1dbab4972603290e5dd5630f06675f13aec97526bc4c", size = 11218346, upload-time = "2026-07-13T11:28:27.745Z" }, + { url = "https://files.pythonhosted.org/packages/2b/cd/cd9f725b19b19e5b530a154cf9bcf9e94279c5d55b3c34fb42b3aa48ea1b/mypy-2.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:be51653d7669d7d7955d613b8d0bb57d5b652eaf71a873ddf65ac87254dd2595", size = 10204525, upload-time = "2026-07-13T11:31:02.552Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ae/f7d056eb0294586a572d0d0d89580ec633c064db520f11d37d5a2fb833bd/mypy-2.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:91ad22a52ae2c7e621c2f67c94d5a17f66b3209a4cff5cf8a573579835c69e97", size = 14947298, upload-time = "2026-07-13T11:27:47.734Z" }, + { url = "https://files.pythonhosted.org/packages/32/d5/db3e7af01e7844d21662c6ddc1f7825ec7cb4053f0391ac02faf3638396f/mypy-2.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:99ac767cc5d3b64c8d0ae226ead10c96694f94e4e7da1668642225dcd4e75aac", size = 13950768, upload-time = "2026-07-13T11:27:57.726Z" }, + { url = "https://files.pythonhosted.org/packages/d9/fb/43c031f0190513d1ec248ed037eceb742ddd2a4d74bbf406658a28173837/mypy-2.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de6d2c484742a4d7b0ed6d07b143375624d3b899c5749c7b3c947f56261f48a6", size = 14151586, upload-time = "2026-07-13T11:29:18.615Z" }, + { url = "https://files.pythonhosted.org/packages/ec/c3/f8b2ffc60883084da91be51af58e88a7ffd4ff9795acb7d902ff88d31eb1/mypy-2.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7da939dd335cfd2ad788bdfd081c9f4e47634ab995e5a45eb15fd1e5bc052f8b", size = 15227411, upload-time = "2026-07-13T11:30:29.904Z" }, + { url = "https://files.pythonhosted.org/packages/83/2e/16b917fc7adcf03f1aadddfc93aab804ffb234b1ab09c0ffd6d92a5d34a2/mypy-2.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7247eb2824f996722a949530183394921ca71deb9680052a338cf53cff7925c2", size = 15478790, upload-time = "2026-07-13T11:33:14.686Z" }, + { url = "https://files.pythonhosted.org/packages/c0/88/aaa65a93c73d0cdae7e42f8adb302bf6885bb281302084f99d0290a35347/mypy-2.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:75b0984bb3cbd76bb5c9291a8671f7ae66ca3b51c7584c358fc2e923259f0757", size = 11234919, upload-time = "2026-07-13T11:33:39.28Z" }, + { url = "https://files.pythonhosted.org/packages/35/19/b40de63f1a80e63bc2d40f0679a6a8dbd34e95176c8122119bdf406aa552/mypy-2.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:d78fcf900b59cb7e82cb7e3a235e31b462d9333d92285bd1e4952d355b8ffba1", size = 10201510, upload-time = "2026-07-13T11:31:52.619Z" }, + { url = "https://files.pythonhosted.org/packages/a4/58/fa0ae047da911f540284009b4f44b96fe09d83c076d7c103e9d645f46303/mypy-2.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ea317b060ce83e26050f8f9e4d7d6bf44ed7597c8ff9990bccffbb9d1d8522db", size = 14941909, upload-time = "2026-07-13T11:32:34.332Z" }, + { url = "https://files.pythonhosted.org/packages/15/14/2ba1d61452d7c2a7fe12741e8d374e52b183476b07aa7f9e2a0d02b0720a/mypy-2.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:094af99f92638aa92852326188b85a89e50f4a472f44827c03362228482f0762", size = 13967581, upload-time = "2026-07-13T11:30:00.587Z" }, + { url = "https://files.pythonhosted.org/packages/ed/5a/483fb9e5ffbbb1a28dccc7b0a13d141b17ac769b6c9f488c0a0c63698962/mypy-2.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de121747278144fc9ae7caa2e978cf5df12aebc82933182f5b3b86081a30baef", size = 14168807, upload-time = "2026-07-13T11:28:48.6Z" }, + { url = "https://files.pythonhosted.org/packages/ae/77/70d7a10732063beb74ad713682cf871e88f5c5fa39bfc8beff8a524bf9cb/mypy-2.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:37fa4de896a84e2dc9200d91e614c22563b43d1a266789d4bbac7b22ebe6192b", size = 15200144, upload-time = "2026-07-13T11:31:25.283Z" }, + { url = "https://files.pythonhosted.org/packages/56/72/766218ac783be4fdfcd699b90037b63017348a3e86fb2c1fbfb18302637d/mypy-2.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f1b3a98dfd21058bc759bb3337d5d1f61d0fdf9f3cf9c00f4291790fb5427bff", size = 15460389, upload-time = "2026-07-13T11:29:29.077Z" }, + { url = "https://files.pythonhosted.org/packages/38/4e/8a9db7411ecb8ec0cb1fd05dba432f28bafffcd38b4e887714a4a0506689/mypy-2.3.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:944c665d984157cb96a679dfb7a4a81dd1d36b24b9c284b699514e6e626b82d4", size = 7753664, upload-time = "2026-07-13T11:29:08.147Z" }, + { url = "https://files.pythonhosted.org/packages/65/4c/c3f8bfd6ed0e5e38b5a244403b27f821d433443df5a15a278417c10a3a3c/mypy-2.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:4359424140d985192c778c1ce2c114a10c1ca58a381ed79cfa70d37df94b299f", size = 11417237, upload-time = "2026-07-13T11:33:47.467Z" }, + { url = "https://files.pythonhosted.org/packages/3c/00/89a32eaf5ccf174bc4f90db0eaea5d70636c01b8d49f384bdab2e8834390/mypy-2.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:3dd0bed92c4bdec57c42505b96416fb9e6a5aa7be84d2809bcd5f2ecec2860d7", size = 10389252, upload-time = "2026-07-13T11:31:43.81Z" }, + { url = "https://files.pythonhosted.org/packages/31/56/104f93d69aa9f339b6b9d3b0a7faa699b8b466c942cf3ae86cc2a2ec0915/mypy-2.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:691fdc37132b1ae628d834f672e74de83462d9fb4aff621835767fb43a8dd373", size = 16385495, upload-time = "2026-07-13T11:29:49.818Z" }, + { url = "https://files.pythonhosted.org/packages/d2/03/f1d2123313f55efafdd27706960f43a771c62f1b68426c76043f3ab9ebf3/mypy-2.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:aec15d465d477558fd842757b487849007311cf3897849cdda0e3162ac0ac556", size = 15098155, upload-time = "2026-07-13T11:30:40.301Z" }, + { url = "https://files.pythonhosted.org/packages/e5/5d/d5f9200399b445e81726c4f23becee33f233aee81c72680b1ef3a258b641/mypy-2.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b352b7e49f5e6576009e8df730e1ff4f915cb565b851b396d2ffe2f5a6f5da88", size = 15514155, upload-time = "2026-07-13T11:34:38.569Z" }, + { url = "https://files.pythonhosted.org/packages/cd/ce/69977c555f08faa3190cfde44189b89dbd56861b1ab97aa18fc5f3a2e4a3/mypy-2.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c6c6bf687b17f90dbfcad95b960d32eaa0154c00da45f03ab50bf8952e047fe", size = 16766351, upload-time = "2026-07-13T11:33:29.195Z" }, + { url = "https://files.pythonhosted.org/packages/bc/92/6648b6caa3ab9e00f9ac0c2a78307805f873dd48139b24a6f6f7c3667bbf/mypy-2.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f4ed18f111bfe2d599bca7468e7f9251042c1c2118f762c8de2766a56d773c60", size = 17043490, upload-time = "2026-07-13T11:30:53.927Z" }, + { url = "https://files.pythonhosted.org/packages/7c/ab/0dc91d80f3f016634c68d451f294a97320fe903a9b6f90b9e57b3f7f1717/mypy-2.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0b025a93cffb9781d231f232be07a17912f35f10a313c24f301c81e842870654", size = 12146869, upload-time = "2026-07-13T11:29:38.874Z" }, + { url = "https://files.pythonhosted.org/packages/85/b5/4c964d02634ba81f4d1c84838e5c5b18ab06d13ed568960f5d6318495ccc/mypy-2.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:adebc76aab4f3495a88b41d48aa4aff0c03f2822501da76625afcca5975f19e5", size = 10965113, upload-time = "2026-07-13T11:28:07.056Z" }, + { url = "https://files.pythonhosted.org/packages/2c/fa/fdc54fe583ba3cafbcedfb70eeeaf03849f75b1827a07096c7bd996f582d/mypy-2.3.0-py3-none-any.whl", hash = "sha256:6b1cdb579446b60432432b2b2403a6201b4b475a004d7f488511c9ba177c9e88", size = 2753292, upload-time = "2026-07-13T11:33:18.48Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + [[package]] name = "nest-asyncio" version = "1.6.0" @@ -1362,6 +1529,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/0f/c8b64d9b54ea631fcad4e9e3c8dbe8c11bb32a623be94f22974c88e71eaf/parsimonious-0.10.0-py3-none-any.whl", hash = "sha256:982ab435fabe86519b57f6b35610aa4e4e977e9f02a14353edf4bbc75369fc0f", size = 48427, upload-time = "2022-09-03T17:01:13.814Z" }, ] +[[package]] +name = "pathspec" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, +] + [[package]] name = "platformdirs" version = "4.3.7" @@ -1455,7 +1631,7 @@ wheels = [ [[package]] name = "py-hamt" -version = "3.4.1" +version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dag-cbor" }, @@ -1466,37 +1642,9 @@ dependencies = [ { name = "pycryptodome" }, { name = "zarr" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fb/dd/26a3653588dfe9992d6c25d23aa0e2d38e2db1e026dfd4d7b781412bd846/py_hamt-3.4.1.tar.gz", hash = "sha256:4563bd875dfae95cda4a87a052a7765e485fce5347db0f9d178002a1c1c55442", size = 228893, upload-time = "2026-07-10T17:46:53.195Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/12/eb/3808e239135727accb07eeca9e4c049967e1ca4606838db94afaf38432e0/py_hamt-3.4.1-py3-none-any.whl", hash = "sha256:f5ef312844609ce14c56b32c0fe2fc388a57f1e639406dfb7a2c1aa7acf7c036", size = 59805, upload-time = "2026-07-10T17:46:52.09Z" }, -] - -[[package]] -name = "pyarrow" -version = "19.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7f/09/a9046344212690f0632b9c709f9bf18506522feb333c894d0de81d62341a/pyarrow-19.0.1.tar.gz", hash = "sha256:3bf266b485df66a400f282ac0b6d1b500b9d2ae73314a153dbe97d6d5cc8a99e", size = 1129437, upload-time = "2025-02-18T18:55:57.027Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/ad/42f19faceb109c85f457249b77aa6da95f63b30ff08aa350be32197b251e/py_hamt-3.5.0.tar.gz", hash = "sha256:5f0cc81fd9a360c481bc835163d2acfaa0862cf6544f5f9ea103cc3b1c17bd70", size = 296183, upload-time = "2026-07-21T15:11:40.521Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/78/b4/94e828704b050e723f67d67c3535cf7076c7432cd4cf046e4bb3b96a9c9d/pyarrow-19.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:80b2ad2b193e7d19e81008a96e313fbd53157945c7be9ac65f44f8937a55427b", size = 30670749, upload-time = "2025-02-18T18:53:00.062Z" }, - { url = "https://files.pythonhosted.org/packages/7e/3b/4692965e04bb1df55e2c314c4296f1eb12b4f3052d4cf43d29e076aedf66/pyarrow-19.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:ee8dec072569f43835932a3b10c55973593abc00936c202707a4ad06af7cb294", size = 32128007, upload-time = "2025-02-18T18:53:06.581Z" }, - { url = "https://files.pythonhosted.org/packages/22/f7/2239af706252c6582a5635c35caa17cb4d401cd74a87821ef702e3888957/pyarrow-19.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d5d1ec7ec5324b98887bdc006f4d2ce534e10e60f7ad995e7875ffa0ff9cb14", size = 41144566, upload-time = "2025-02-18T18:53:11.958Z" }, - { url = "https://files.pythonhosted.org/packages/fb/e3/c9661b2b2849cfefddd9fd65b64e093594b231b472de08ff658f76c732b2/pyarrow-19.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3ad4c0eb4e2a9aeb990af6c09e6fa0b195c8c0e7b272ecc8d4d2b6574809d34", size = 42202991, upload-time = "2025-02-18T18:53:17.678Z" }, - { url = "https://files.pythonhosted.org/packages/fe/4f/a2c0ed309167ef436674782dfee4a124570ba64299c551e38d3fdaf0a17b/pyarrow-19.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d383591f3dcbe545f6cc62daaef9c7cdfe0dff0fb9e1c8121101cabe9098cfa6", size = 40507986, upload-time = "2025-02-18T18:53:26.263Z" }, - { url = "https://files.pythonhosted.org/packages/27/2e/29bb28a7102a6f71026a9d70d1d61df926887e36ec797f2e6acfd2dd3867/pyarrow-19.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b4c4156a625f1e35d6c0b2132635a237708944eb41df5fbe7d50f20d20c17832", size = 42087026, upload-time = "2025-02-18T18:53:33.063Z" }, - { url = "https://files.pythonhosted.org/packages/16/33/2a67c0f783251106aeeee516f4806161e7b481f7d744d0d643d2f30230a5/pyarrow-19.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:5bd1618ae5e5476b7654c7b55a6364ae87686d4724538c24185bbb2952679960", size = 25250108, upload-time = "2025-02-18T18:53:38.462Z" }, - { url = "https://files.pythonhosted.org/packages/2b/8d/275c58d4b00781bd36579501a259eacc5c6dfb369be4ddeb672ceb551d2d/pyarrow-19.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e45274b20e524ae5c39d7fc1ca2aa923aab494776d2d4b316b49ec7572ca324c", size = 30653552, upload-time = "2025-02-18T18:53:44.357Z" }, - { url = "https://files.pythonhosted.org/packages/a0/9e/e6aca5cc4ef0c7aec5f8db93feb0bde08dbad8c56b9014216205d271101b/pyarrow-19.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:d9dedeaf19097a143ed6da37f04f4051aba353c95ef507764d344229b2b740ae", size = 32103413, upload-time = "2025-02-18T18:53:52.971Z" }, - { url = "https://files.pythonhosted.org/packages/6a/fa/a7033f66e5d4f1308c7eb0dfcd2ccd70f881724eb6fd1776657fdf65458f/pyarrow-19.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ebfb5171bb5f4a52319344ebbbecc731af3f021e49318c74f33d520d31ae0c4", size = 41134869, upload-time = "2025-02-18T18:53:59.471Z" }, - { url = "https://files.pythonhosted.org/packages/2d/92/34d2569be8e7abdc9d145c98dc410db0071ac579b92ebc30da35f500d630/pyarrow-19.0.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a21d39fbdb948857f67eacb5bbaaf36802de044ec36fbef7a1c8f0dd3a4ab2", size = 42192626, upload-time = "2025-02-18T18:54:06.062Z" }, - { url = "https://files.pythonhosted.org/packages/0a/1f/80c617b1084fc833804dc3309aa9d8daacd46f9ec8d736df733f15aebe2c/pyarrow-19.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:99bc1bec6d234359743b01e70d4310d0ab240c3d6b0da7e2a93663b0158616f6", size = 40496708, upload-time = "2025-02-18T18:54:12.347Z" }, - { url = "https://files.pythonhosted.org/packages/e6/90/83698fcecf939a611c8d9a78e38e7fed7792dcc4317e29e72cf8135526fb/pyarrow-19.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1b93ef2c93e77c442c979b0d596af45e4665d8b96da598db145b0fec014b9136", size = 42075728, upload-time = "2025-02-18T18:54:19.364Z" }, - { url = "https://files.pythonhosted.org/packages/40/49/2325f5c9e7a1c125c01ba0c509d400b152c972a47958768e4e35e04d13d8/pyarrow-19.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:d9d46e06846a41ba906ab25302cf0fd522f81aa2a85a71021826f34639ad31ef", size = 25242568, upload-time = "2025-02-18T18:54:25.846Z" }, - { url = "https://files.pythonhosted.org/packages/3f/72/135088d995a759d4d916ec4824cb19e066585b4909ebad4ab196177aa825/pyarrow-19.0.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:c0fe3dbbf054a00d1f162fda94ce236a899ca01123a798c561ba307ca38af5f0", size = 30702371, upload-time = "2025-02-18T18:54:30.665Z" }, - { url = "https://files.pythonhosted.org/packages/2e/01/00beeebd33d6bac701f20816a29d2018eba463616bbc07397fdf99ac4ce3/pyarrow-19.0.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:96606c3ba57944d128e8a8399da4812f56c7f61de8c647e3470b417f795d0ef9", size = 32116046, upload-time = "2025-02-18T18:54:35.995Z" }, - { url = "https://files.pythonhosted.org/packages/1f/c9/23b1ea718dfe967cbd986d16cf2a31fe59d015874258baae16d7ea0ccabc/pyarrow-19.0.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f04d49a6b64cf24719c080b3c2029a3a5b16417fd5fd7c4041f94233af732f3", size = 41091183, upload-time = "2025-02-18T18:54:42.662Z" }, - { url = "https://files.pythonhosted.org/packages/3a/d4/b4a3aa781a2c715520aa8ab4fe2e7fa49d33a1d4e71c8fc6ab7b5de7a3f8/pyarrow-19.0.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a9137cf7e1640dce4c190551ee69d478f7121b5c6f323553b319cac936395f6", size = 42171896, upload-time = "2025-02-18T18:54:49.808Z" }, - { url = "https://files.pythonhosted.org/packages/23/1b/716d4cd5a3cbc387c6e6745d2704c4b46654ba2668260d25c402626c5ddb/pyarrow-19.0.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:7c1bca1897c28013db5e4c83944a2ab53231f541b9e0c3f4791206d0c0de389a", size = 40464851, upload-time = "2025-02-18T18:54:57.073Z" }, - { url = "https://files.pythonhosted.org/packages/ed/bd/54907846383dcc7ee28772d7e646f6c34276a17da740002a5cefe90f04f7/pyarrow-19.0.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:58d9397b2e273ef76264b45531e9d552d8ec8a6688b7390b5be44c02a37aade8", size = 42085744, upload-time = "2025-02-18T18:55:08.562Z" }, + { url = "https://files.pythonhosted.org/packages/62/e4/c003760c4af897b47e1e18bef8bfa828f279cb9b1fd7ee86ffdf290b0b37/py_hamt-3.5.0-py3-none-any.whl", hash = "sha256:5e06627d3105ff92b3ece3f0dae5ebd9003117ea976a8740919dfec47eeaffb2", size = 70398, upload-time = "2026-07-21T15:11:39.513Z" }, ] [[package]] @@ -2054,40 +2202,53 @@ wheels = [ [[package]] name = "scipy" -version = "1.15.2" +version = "1.18.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b7/b9/31ba9cd990e626574baf93fbc1ac61cf9ed54faafd04c479117517661637/scipy-1.15.2.tar.gz", hash = "sha256:cd58a314d92838f7e6f755c8a2167ead4f27e1fd5c1251fd54289569ef3495ec", size = 59417316, upload-time = "2025-02-17T00:42:24.791Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/5d/3c78815cbab499610f26b5bae6aed33e227225a9fa5290008a733a64f6fc/scipy-1.15.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c4697a10da8f8765bb7c83e24a470da5797e37041edfd77fd95ba3811a47c4fd", size = 38756184, upload-time = "2025-02-17T00:31:50.623Z" }, - { url = "https://files.pythonhosted.org/packages/37/20/3d04eb066b471b6e171827548b9ddb3c21c6bbea72a4d84fc5989933910b/scipy-1.15.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:869269b767d5ee7ea6991ed7e22b3ca1f22de73ab9a49c44bad338b725603301", size = 30163558, upload-time = "2025-02-17T00:31:56.721Z" }, - { url = "https://files.pythonhosted.org/packages/a4/98/e5c964526c929ef1f795d4c343b2ff98634ad2051bd2bbadfef9e772e413/scipy-1.15.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:bad78d580270a4d32470563ea86c6590b465cb98f83d760ff5b0990cb5518a93", size = 22437211, upload-time = "2025-02-17T00:32:03.042Z" }, - { url = "https://files.pythonhosted.org/packages/1d/cd/1dc7371e29195ecbf5222f9afeedb210e0a75057d8afbd942aa6cf8c8eca/scipy-1.15.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:b09ae80010f52efddb15551025f9016c910296cf70adbf03ce2a8704f3a5ad20", size = 25232260, upload-time = "2025-02-17T00:32:07.847Z" }, - { url = "https://files.pythonhosted.org/packages/f0/24/1a181a9e5050090e0b5138c5f496fee33293c342b788d02586bc410c6477/scipy-1.15.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a6fd6eac1ce74a9f77a7fc724080d507c5812d61e72bd5e4c489b042455865e", size = 35198095, upload-time = "2025-02-17T00:32:14.565Z" }, - { url = "https://files.pythonhosted.org/packages/c0/53/eaada1a414c026673eb983f8b4a55fe5eb172725d33d62c1b21f63ff6ca4/scipy-1.15.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2b871df1fe1a3ba85d90e22742b93584f8d2b8e6124f8372ab15c71b73e428b8", size = 37297371, upload-time = "2025-02-17T00:32:21.411Z" }, - { url = "https://files.pythonhosted.org/packages/e9/06/0449b744892ed22b7e7b9a1994a866e64895363572677a316a9042af1fe5/scipy-1.15.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:03205d57a28e18dfd39f0377d5002725bf1f19a46f444108c29bdb246b6c8a11", size = 36872390, upload-time = "2025-02-17T00:32:29.421Z" }, - { url = "https://files.pythonhosted.org/packages/6a/6f/a8ac3cfd9505ec695c1bc35edc034d13afbd2fc1882a7c6b473e280397bb/scipy-1.15.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:601881dfb761311045b03114c5fe718a12634e5608c3b403737ae463c9885d53", size = 39700276, upload-time = "2025-02-17T00:32:37.431Z" }, - { url = "https://files.pythonhosted.org/packages/f5/6f/e6e5aff77ea2a48dd96808bb51d7450875af154ee7cbe72188afb0b37929/scipy-1.15.2-cp312-cp312-win_amd64.whl", hash = "sha256:e7c68b6a43259ba0aab737237876e5c2c549a031ddb7abc28c7b47f22e202ded", size = 40942317, upload-time = "2025-02-17T00:32:45.47Z" }, - { url = "https://files.pythonhosted.org/packages/53/40/09319f6e0f276ea2754196185f95cd191cb852288440ce035d5c3a931ea2/scipy-1.15.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:01edfac9f0798ad6b46d9c4c9ca0e0ad23dbf0b1eb70e96adb9fa7f525eff0bf", size = 38717587, upload-time = "2025-02-17T00:32:53.196Z" }, - { url = "https://files.pythonhosted.org/packages/fe/c3/2854f40ecd19585d65afaef601e5e1f8dbf6758b2f95b5ea93d38655a2c6/scipy-1.15.2-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:08b57a9336b8e79b305a143c3655cc5bdbe6d5ece3378578888d2afbb51c4e37", size = 30100266, upload-time = "2025-02-17T00:32:59.318Z" }, - { url = "https://files.pythonhosted.org/packages/dd/b1/f9fe6e3c828cb5930b5fe74cb479de5f3d66d682fa8adb77249acaf545b8/scipy-1.15.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:54c462098484e7466362a9f1672d20888f724911a74c22ae35b61f9c5919183d", size = 22373768, upload-time = "2025-02-17T00:33:04.091Z" }, - { url = "https://files.pythonhosted.org/packages/15/9d/a60db8c795700414c3f681908a2b911e031e024d93214f2d23c6dae174ab/scipy-1.15.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:cf72ff559a53a6a6d77bd8eefd12a17995ffa44ad86c77a5df96f533d4e6c6bb", size = 25154719, upload-time = "2025-02-17T00:33:08.909Z" }, - { url = "https://files.pythonhosted.org/packages/37/3b/9bda92a85cd93f19f9ed90ade84aa1e51657e29988317fabdd44544f1dd4/scipy-1.15.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9de9d1416b3d9e7df9923ab23cd2fe714244af10b763975bea9e4f2e81cebd27", size = 35163195, upload-time = "2025-02-17T00:33:15.352Z" }, - { url = "https://files.pythonhosted.org/packages/03/5a/fc34bf1aa14dc7c0e701691fa8685f3faec80e57d816615e3625f28feb43/scipy-1.15.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb530e4794fc8ea76a4a21ccb67dea33e5e0e60f07fc38a49e821e1eae3b71a0", size = 37255404, upload-time = "2025-02-17T00:33:22.21Z" }, - { url = "https://files.pythonhosted.org/packages/4a/71/472eac45440cee134c8a180dbe4c01b3ec247e0338b7c759e6cd71f199a7/scipy-1.15.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5ea7ed46d437fc52350b028b1d44e002646e28f3e8ddc714011aaf87330f2f32", size = 36860011, upload-time = "2025-02-17T00:33:29.446Z" }, - { url = "https://files.pythonhosted.org/packages/01/b3/21f890f4f42daf20e4d3aaa18182dddb9192771cd47445aaae2e318f6738/scipy-1.15.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:11e7ad32cf184b74380f43d3c0a706f49358b904fa7d5345f16ddf993609184d", size = 39657406, upload-time = "2025-02-17T00:33:39.019Z" }, - { url = "https://files.pythonhosted.org/packages/0d/76/77cf2ac1f2a9cc00c073d49e1e16244e389dd88e2490c91d84e1e3e4d126/scipy-1.15.2-cp313-cp313-win_amd64.whl", hash = "sha256:a5080a79dfb9b78b768cebf3c9dcbc7b665c5875793569f48bf0e2b1d7f68f6f", size = 40961243, upload-time = "2025-02-17T00:34:51.024Z" }, - { url = "https://files.pythonhosted.org/packages/4c/4b/a57f8ddcf48e129e6054fa9899a2a86d1fc6b07a0e15c7eebff7ca94533f/scipy-1.15.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:447ce30cee6a9d5d1379087c9e474628dab3db4a67484be1b7dc3196bfb2fac9", size = 38870286, upload-time = "2025-02-17T00:33:47.62Z" }, - { url = "https://files.pythonhosted.org/packages/0c/43/c304d69a56c91ad5f188c0714f6a97b9c1fed93128c691148621274a3a68/scipy-1.15.2-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:c90ebe8aaa4397eaefa8455a8182b164a6cc1d59ad53f79943f266d99f68687f", size = 30141634, upload-time = "2025-02-17T00:33:54.131Z" }, - { url = "https://files.pythonhosted.org/packages/44/1a/6c21b45d2548eb73be9b9bff421aaaa7e85e22c1f9b3bc44b23485dfce0a/scipy-1.15.2-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:def751dd08243934c884a3221156d63e15234a3155cf25978b0a668409d45eb6", size = 22415179, upload-time = "2025-02-17T00:33:59.948Z" }, - { url = "https://files.pythonhosted.org/packages/74/4b/aefac4bba80ef815b64f55da06f62f92be5d03b467f2ce3668071799429a/scipy-1.15.2-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:302093e7dfb120e55515936cb55618ee0b895f8bcaf18ff81eca086c17bd80af", size = 25126412, upload-time = "2025-02-17T00:34:06.328Z" }, - { url = "https://files.pythonhosted.org/packages/b1/53/1cbb148e6e8f1660aacd9f0a9dfa2b05e9ff1cb54b4386fe868477972ac2/scipy-1.15.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cd5b77413e1855351cdde594eca99c1f4a588c2d63711388b6a1f1c01f62274", size = 34952867, upload-time = "2025-02-17T00:34:12.928Z" }, - { url = "https://files.pythonhosted.org/packages/2c/23/e0eb7f31a9c13cf2dca083828b97992dd22f8184c6ce4fec5deec0c81fcf/scipy-1.15.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d0194c37037707b2afa7a2f2a924cf7bac3dc292d51b6a925e5fcb89bc5c776", size = 36890009, upload-time = "2025-02-17T00:34:19.55Z" }, - { url = "https://files.pythonhosted.org/packages/03/f3/e699e19cabe96bbac5189c04aaa970718f0105cff03d458dc5e2b6bd1e8c/scipy-1.15.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:bae43364d600fdc3ac327db99659dcb79e6e7ecd279a75fe1266669d9a652828", size = 36545159, upload-time = "2025-02-17T00:34:26.724Z" }, - { url = "https://files.pythonhosted.org/packages/af/f5/ab3838e56fe5cc22383d6fcf2336e48c8fe33e944b9037fbf6cbdf5a11f8/scipy-1.15.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f031846580d9acccd0044efd1a90e6f4df3a6e12b4b6bd694a7bc03a89892b28", size = 39136566, upload-time = "2025-02-17T00:34:34.512Z" }, - { url = "https://files.pythonhosted.org/packages/0a/c8/b3f566db71461cabd4b2d5b39bcc24a7e1c119535c8361f81426be39bb47/scipy-1.15.2-cp313-cp313t-win_amd64.whl", hash = "sha256:fe8a9eb875d430d81755472c5ba75e84acc980e4a8f6204d402849234d3017db", size = 40477705, upload-time = "2025-02-17T00:34:43.619Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/19/ca10ead60b0acc80b2b833c2c4a4f2ff753d0f58b811f70d911c7e94a25c/scipy-1.18.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:7bd21faaf5a1a3b2eff922d02db5f191b99a6518db9078a8fb23169f6d22259a", size = 31056519, upload-time = "2026-06-19T14:59:45.203Z" }, + { url = "https://files.pythonhosted.org/packages/96/72/1e6442a00cd2924d361aa1b642ab6373ec35c6fabf311a760be9f76e0f13/scipy-1.18.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:265915e79107de9f946b855e50d7470d5893ec3f54b342e1aa6201cbdcd8bb6b", size = 28681889, upload-time = "2026-06-19T14:59:48.103Z" }, + { url = "https://files.pythonhosted.org/packages/9b/2d/11dd93d21e147a73ba22bd75c0b9208d3a2e0ec76d53170ce7d9029b1015/scipy-1.18.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9ab7b758be6940954a713ee466e2043e9f6e2ed965c1fce5c91039f4be3d90a9", size = 20423580, upload-time = "2026-06-19T14:59:50.665Z" }, + { url = "https://files.pythonhosted.org/packages/9c/01/93552f75e0d2a7dd115a45e59209c51e8d514daff02fc887d2623be06fe1/scipy-1.18.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:97b6cddaaee0a779ef6b5ca83c9604b27cc16b2b8fc22c142652df8793319fb8", size = 23054441, upload-time = "2026-06-19T14:59:53.564Z" }, + { url = "https://files.pythonhosted.org/packages/3c/23/21f5e703643d66f21faa6b4c73195bfcad70c55efcb4f1ab327cd7c4101a/scipy-1.18.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:52a96e21517c7292375c0e27dd796a811f03fcea5fd4d108fdfea8145dcf17ab", size = 33968720, upload-time = "2026-06-19T14:59:56.415Z" }, + { url = "https://files.pythonhosted.org/packages/dd/aa/1b939f6c67ed68635bb538e6752d3dacc02f66535182e939a89581a44e9c/scipy-1.18.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f55797419e16e7f30cf88ffb3113ce0467f00cfe3f70d5c281730b21769bfc2", size = 35287115, upload-time = "2026-06-19T14:59:59.411Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ff/eec46be7e9234208f801062b53e1983085eddebd693f6c9bfb03b459830d/scipy-1.18.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ad033410e2e0672ffdc1042110cef20e1c46f8fd0616cee1d44d8d58fad8fc11", size = 35577989, upload-time = "2026-06-19T15:00:02.235Z" }, + { url = "https://files.pythonhosted.org/packages/84/ca/210d4759c7210bb7d269437421959b39a33434e2776b60c5cb8a763bb30a/scipy-1.18.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4a55985d54c769c872e64b7f4c8a81cc30ef700cc04296abbbf3705439c126de", size = 37421717, upload-time = "2026-06-19T15:00:05.102Z" }, + { url = "https://files.pythonhosted.org/packages/2b/54/9a9edb45345bd6744da5ddfb6628e5d5185920494c6a67ec45b6381004cb/scipy-1.18.0-cp312-cp312-win_amd64.whl", hash = "sha256:71ccc8faa2dd16ac310233203474a8b5cb67f10dedd54a3116d34943f4b19132", size = 36597428, upload-time = "2026-06-19T15:00:08.112Z" }, + { url = "https://files.pythonhosted.org/packages/99/0e/33f32a2a58987e26aec0f7df252cbbad1e90ae77bdbc76f40dd4ed0cf0ea/scipy-1.18.0-cp312-cp312-win_arm64.whl", hash = "sha256:d88363fd9d8fbd3511bd273f1a49efb2a540773ddf92a91d57498ce7dd7f3e76", size = 24351481, upload-time = "2026-06-19T15:00:11.103Z" }, + { url = "https://files.pythonhosted.org/packages/05/52/9c0136c2de7ae0779b7b366447766cec6d9f0702c56bb8ffeb04c8fd3af4/scipy-1.18.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:09143f676d157d9f546d663504ef9c1becb819824f1afc018814176411942446", size = 31036107, upload-time = "2026-06-19T15:00:14.03Z" }, + { url = "https://files.pythonhosted.org/packages/02/73/0291a64843270f4efb86cdcf2ee0f2048631b65ec6b405398b2b4dbf11bf/scipy-1.18.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:5efe260f69417b97ddae455bfb5a95e8359f7f66ad7fa9522a60feb66f169520", size = 28663303, upload-time = "2026-06-19T15:00:16.819Z" }, + { url = "https://files.pythonhosted.org/packages/d3/0f/10ffa0b697a572f4e0d48b92a88895d366422f019f723e7e14a84c050dac/scipy-1.18.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:68363b7eaacd8b5dd426df56d782cc156468ac79a127a1b87ca597d6e2e82197", size = 20404960, upload-time = "2026-06-19T15:00:19.635Z" }, + { url = "https://files.pythonhosted.org/packages/7e/d2/e896cea21ba8edd6c81d4c55b1ffcc717e79698dcbebf9641b4cfb4c6622/scipy-1.18.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:c5557d8be5da8e41353fcd4d21491fdbab83b062fc579e94dc09a7c8ab4f669b", size = 23034074, upload-time = "2026-06-19T15:00:22.107Z" }, + { url = "https://files.pythonhosted.org/packages/ea/b2/e83ea34279a52c03374477c74006256ec78df65fc877baa4617d6de1d202/scipy-1.18.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0d13bca67c096d89fb95ced0d8921807300fce0275643aef9533cc63a0773468", size = 33942038, upload-time = "2026-06-19T15:00:24.964Z" }, + { url = "https://files.pythonhosted.org/packages/f6/af/e8fe5fb136f51e2b01678b92cb4106d10d8cd68ec147ead2e7cb0ac75398/scipy-1.18.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a46f9273dbd0eb1cefba61c9b8648b4dfe3cbc14a080176f9a73e44b8336dc7f", size = 35266390, upload-time = "2026-06-19T15:00:28.059Z" }, + { url = "https://files.pythonhosted.org/packages/3a/49/2c5cbb907b56695fc67517811d1db234dfd83381a84814ec220aded2794d/scipy-1.18.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5aba46108853ddfc77906b6557aac839d2b52e900c1d72a1180adaaab58d265f", size = 35551324, upload-time = "2026-06-19T15:00:31.014Z" }, + { url = "https://files.pythonhosted.org/packages/bb/73/eda39f7a2d306ff0ffc574afd13c0bbb6d10a603d9a413998ee269487a80/scipy-1.18.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b6f758e35f12757b5d95c00bc6de2438e229c2664b7a92e96f205959d9f2dfa4", size = 37404785, upload-time = "2026-06-19T15:00:34.072Z" }, + { url = "https://files.pythonhosted.org/packages/b7/d2/ae881ee28d014f38e0ccbfd974a06a919ba9af34f1f74bf42b5301891d63/scipy-1.18.0-cp313-cp313-win_amd64.whl", hash = "sha256:1afac4a847207c7ff8efd321734a50b06d0280b3b2a2c0fc2f413101747ad7c7", size = 36554943, upload-time = "2026-06-19T15:00:36.903Z" }, + { url = "https://files.pythonhosted.org/packages/70/3a/21154e2d54eb3639c6bf4dbae2e531c68356bfe95990daa30df33b30d556/scipy-1.18.0-cp313-cp313-win_arm64.whl", hash = "sha256:c5dbddf60e58c2312316d097271a8e73d40eaf2eabfa4d95ed7d3695bbf2ce7b", size = 24350911, upload-time = "2026-06-19T15:00:40.062Z" }, + { url = "https://files.pythonhosted.org/packages/78/b5/915a19b3de2f7430062b509653563db1633ddbb6f021b06731521115d4e2/scipy-1.18.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4c256ee70c0d1a8a2ace807e199ccd4e3f57037433842abb3fb36bc17eaa9578", size = 31036253, upload-time = "2026-06-19T15:00:43.216Z" }, + { url = "https://files.pythonhosted.org/packages/d7/88/b72def7262e150d16be13fca37a96481138d624e700340bc3362a7588929/scipy-1.18.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:2ef3abc54a4ffc53765374b0d5728532dfdd2585ed23f6b11c206a1f0b1b9af8", size = 28673758, upload-time = "2026-06-19T15:00:46.663Z" }, + { url = "https://files.pythonhosted.org/packages/91/02/2e636a61a525632c373cf6a9c24442a3ffb79e364d38e98b32042964ac32/scipy-1.18.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:f2a6af57bd9e4a75d70e4117e78a1bbee84f79ae3fbb6d0111005d6ebcc4cb8d", size = 20415514, upload-time = "2026-06-19T15:00:49.399Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b6/2135974442f6aba159d9d39d774a1c8cb19947016725d69fecc685df45bf/scipy-1.18.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:3f1ac564d3bf6c03d861d2cd87a1bea0da2887136f7fb1bf519c05a8971452d6", size = 23034398, upload-time = "2026-06-19T15:00:51.941Z" }, + { url = "https://files.pythonhosted.org/packages/f6/e6/ba89ec5abf6ee9257c0d1ec985573f3ae32742c24bc03e016388a40b1b15/scipy-1.18.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40395a5fcd1abee49a5c7aaa98c29db393eedc835138560a588c47ec16156690", size = 33998032, upload-time = "2026-06-19T15:00:54.838Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c4/bc41eb19b0fd0db868f4132920879019318d80cc522ad8f2bca4611af808/scipy-1.18.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ca01e8ae69f1b18e9a58d91afead31be3cef0dd905a10249dac559ee15460a0", size = 35283333, upload-time = "2026-06-19T15:00:58.152Z" }, + { url = "https://files.pythonhosted.org/packages/53/a4/cbdeef6eb3830a8462a9d4ada814de5fc984345cc9ecf17cbec51a036f1e/scipy-1.18.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7a7f3b01647384dbc3a711e8c6778e0aabbe93959249fef5c7393396bcac0867", size = 35610216, upload-time = "2026-06-19T15:01:01.155Z" }, + { url = "https://files.pythonhosted.org/packages/80/4d/b2b82502b65f661d1b789c1665dcdf315d5f12194e06fc0b37946294ebae/scipy-1.18.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6aa94e78ec192a30063a5e72e561c28af769dc311190b24fe91774eff1969709", size = 37418960, upload-time = "2026-06-19T15:01:04.155Z" }, + { url = "https://files.pythonhosted.org/packages/93/3e/902d836831474b0ab5a37d16404f7bc5fafd9efba632890e271ba952635f/scipy-1.18.0-cp314-cp314-win_amd64.whl", hash = "sha256:2d8bbdc6c817f5b4006a54d799d4f5bab6f910193cbb9a1ff310833d4d270f61", size = 37288845, upload-time = "2026-06-19T15:01:07.822Z" }, + { url = "https://files.pythonhosted.org/packages/b6/43/8d73b337a3bdb14daa0314f0434210747c02d79d729ce1777574a817dcf6/scipy-1.18.0-cp314-cp314-win_arm64.whl", hash = "sha256:18e9575f1569b2c54174e6159d32942e03731177f63dce7975f0a0c88d102f5b", size = 24988971, upload-time = "2026-06-19T15:01:11.076Z" }, + { url = "https://files.pythonhosted.org/packages/b4/b4/f11918b0508a2787031a0499a03fbe3546f3bb5ca05d01038c45b278c09a/scipy-1.18.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:f351e0dd702687d12a402b867a1b4146a256923e1c38317cbc472f6372b94707", size = 31399325, upload-time = "2026-06-19T15:01:13.723Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d1/1f287b57c0ff0ee5185dff3946d92c8017d39b0e431f0ae79a3ff1859512/scipy-1.18.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:7c7a51b33ce387193c97f228320cf8e87361daa1bba750638677729598b3e677", size = 29092110, upload-time = "2026-06-19T15:01:16.908Z" }, + { url = "https://files.pythonhosted.org/packages/ff/1a/7b74eb6c392fdcb27d414c0e7558a6d0231eb3b6d73571f479bb81ea8794/scipy-1.18.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:84031d7b052a54fae2f8632e0ec802073d385476eb9a63079bce6e23ef9283d4", size = 20833811, upload-time = "2026-06-19T15:01:20.488Z" }, + { url = "https://files.pythonhosted.org/packages/7c/ad/f3941716320a7b9cb4d68734a903b45fe16eff5fb7da7e16f2e619304979/scipy-1.18.0-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:56abf29a7c067dde59be8b9a22d606a4ea1b2f2a4b756d9d903c62818f5dacce", size = 23396644, upload-time = "2026-06-19T15:01:23.364Z" }, + { url = "https://files.pythonhosted.org/packages/22/22/1446b62ffe07f9719b7d9b1b6a4e05a772833ae8f441fe4c22c34c9b250f/scipy-1.18.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ad44305cfa24b1ba5803cbbebf033590ccbac1aa5d612d727b785325ab408b0", size = 34079318, upload-time = "2026-06-19T15:01:26.002Z" }, + { url = "https://files.pythonhosted.org/packages/56/3b/b87da667098bb470fa30c7011b0ba351ee976dd395c78798c66e941665a3/scipy-1.18.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:945c1761b93f38d7f99ae81ae80c63e621471608c7eeead563f6df025585cd58", size = 35324320, upload-time = "2026-06-19T15:01:28.881Z" }, + { url = "https://files.pythonhosted.org/packages/f8/a1/c7932f91909759b0267f75fdea34e91309f96b895757534b76a90b6b4344/scipy-1.18.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1a4441f15d620578772a49e5ab48c0ee1f7a0220e387110283062729136b2553", size = 35699541, upload-time = "2026-06-19T15:01:31.968Z" }, + { url = "https://files.pythonhosted.org/packages/f7/86/5185061a1fcc41d18c5dc2463969b3a3964b31d9ac67b2fb05d4c7ff7670/scipy-1.18.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9aac6192fac56bf2ca534389d24623f07b39ff83317d58287285e7fbd622ff76", size = 37472480, upload-time = "2026-06-19T15:01:35.136Z" }, + { url = "https://files.pythonhosted.org/packages/31/8e/f04c68e39919a010d34f2ee1367fd705b0a25a02f609d755f0bfbc0a15fc/scipy-1.18.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e40baea28ae7f5475c779741e2d90b1247c78531207b49c7030e698ff81cee3f", size = 37365390, upload-time = "2026-06-19T15:01:38.091Z" }, + { url = "https://files.pythonhosted.org/packages/d5/19/969dc072906c84dd0a3b05dcf57ea750936087d7873549e408b35cfc3f97/scipy-1.18.0-cp314-cp314t-win_arm64.whl", hash = "sha256:368e0a705903c466aa5f08eefb39e6b1b6b2d659e7352a31fd9e2438365be0f8", size = 25279661, upload-time = "2026-06-19T15:01:40.817Z" }, ] [[package]]