Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# AGENTS.md

## Cross-client parity

This Python client and [dClimate/dclimate-client-js](https://github.com/dClimate/dclimate-client-js) are sibling libraries. Keep their user-visible capabilities and behavior aligned unless a language or runtime difference makes a change inapplicable.

- For every public API or behavior change—especially STAC/IPFS resolution, dataset loading and selection, metadata, errors, and catalog listing—inspect the corresponding implementation, tests, documentation, and relevant open work in the JavaScript client before finishing.
- Unless the user explicitly limits the task to one repository, treat an applicable sibling-library update as part of the same task. Add equivalent tests and documentation in both projects, using idiomatic APIs for each language rather than mechanically copying implementation details.
- If a change is not applicable to the sibling, or the sibling cannot be updated in the current task, state the reason and leave a concrete follow-up in the handoff or pull-request description. Do not silently allow accidental divergence.
- When reviewing either client, treat undocumented behavioral differences as possible defects and verify whether parity should be restored.
19 changes: 19 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,25 @@ for collection_id, info in datasets.items():
)
print(f" Dataset types: {', '.join(info['types'])}")

# Resolve a CID directly without blocking the event loop. Calls made without
# an injected client reuse a pooled httpx.AsyncClient for the current loop.
from dclimate_client_py import (
aclose_stac_server_client,
aresolve_cid_from_stac_server,
)

async def resolve_cid():
try:
resolved = await aresolve_cid_from_stac_server(
collection="ecmwf_aifs",
dataset="temperature_forecast",
variant="single",
)
print(resolved.cid)
finally:
# Call once when an application event loop shuts down.
await aclose_stac_server_client()

```

## Siren API usage
Expand Down
4 changes: 4 additions & 0 deletions dclimate_client_py/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
)
from .stac_server import (
ResolvedDataset,
aclose_stac_server_client,
aresolve_cid_from_stac_server,
resolve_cid_from_stac_server,
list_available_datasets_from_stac_server,
STAC_SERVER_URL,
Expand Down Expand Up @@ -76,6 +78,8 @@ def __dir__() -> list[str]:
"load_stac_catalog",
"list_available_datasets",
"ResolvedDataset",
"aclose_stac_server_client",
"aresolve_cid_from_stac_server",
"resolve_cid_from_stac_server",
"list_available_datasets_from_stac_server",
"STAC_SERVER_URL",
Expand Down
79 changes: 48 additions & 31 deletions dclimate_client_py/dclimate_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
from .dclimate_zarr_errors import InvalidSelectionError
from .stac_server import (
ResolvedDataset,
resolve_cid_from_stac_server,
aresolve_cid_from_stac_server,
list_available_datasets_from_stac_server,
)
from .siren import SirenClient
Expand All @@ -39,6 +39,23 @@
DEFAULT_PUBLIC_GATEWAY = "https://ipfs-gateway.dclimate.net"


def _merge_cleanup_error(
current: BaseException | None,
new: BaseException,
) -> BaseException:
"""Chain cleanup failures while ensuring cancellation remains dominant."""
if current is None:
return new
if isinstance(current, asyncio.CancelledError) and not isinstance(
new, asyncio.CancelledError
):
new.__context__ = current.__context__
current.__context__ = new
return current
new.__context__ = current
return new


class dClimateClient:
"""
Async context manager for loading dClimate datasets from IPFS.
Expand Down Expand Up @@ -133,6 +150,7 @@ def __init__(
self._client_factory = client_factory
self._stac_catalog: typing.Optional["pystac.Catalog"] = None
self._stac_catalog_lock = asyncio.Lock()
self._stac_http_client: typing.Optional[httpx.AsyncClient] = None
self._kubo_cas: typing.Optional[KuboCAS] = None
# Note: STAC catalog is loaded lazily (only if STAC server fails)

Expand Down Expand Up @@ -170,53 +188,52 @@ async def __aenter__(self) -> "dClimateClient":
return self

async def __aexit__(self, exc_type, exc_val, exc_tb):
"""Clean up KuboCAS when exiting async context."""
"""Clean up owned HTTP and KuboCAS resources."""
incoming_cancellation = isinstance(exc_val, asyncio.CancelledError)
siren_error: BaseException | None = None
cleanup_error: BaseException | None = None

try:
if self._stac_http_client is not None:
await self._stac_http_client.aclose()
except BaseException as error:
cleanup_error = _merge_cleanup_error(cleanup_error, error)
finally:
self._stac_http_client = None

try:
if self._siren_client is not None:
await self._siren_client.aclose()
except BaseException as error:
siren_error = error
cleanup_error = _merge_cleanup_error(cleanup_error, error)

try:
if self._kubo_cas is not None:
await self._kubo_cas.__aexit__(exc_type, exc_val, exc_tb)
except BaseException as kubo_error:
if incoming_cancellation and not isinstance(
kubo_error, asyncio.CancelledError
):
# Preserve cancellation from the context body. An ordinary
# cleanup failure must not replace task cancellation, but
# remains inspectable through the exception context chain.
if siren_error is not None:
kubo_error.__context__ = siren_error
exc_val.__context__ = kubo_error
return False
if siren_error is None:
raise
# Both cleanups failed. Follow the AsyncExitStack convention (the
# later error propagates with the earlier as __context__), except
# that a cancellation always outranks an ordinary error.
if isinstance(siren_error, asyncio.CancelledError) and not isinstance(
kubo_error, asyncio.CancelledError
):
raise siren_error
kubo_error.__context__ = siren_error
raise
cleanup_error = _merge_cleanup_error(cleanup_error, kubo_error)
finally:
self._kubo_cas = None

if siren_error is not None:
if cleanup_error is not None:
if incoming_cancellation and not isinstance(
siren_error, asyncio.CancelledError
cleanup_error, asyncio.CancelledError
):
exc_val.__context__ = siren_error
# Preserve cancellation from the context body. Ordinary
# cleanup failures remain inspectable through its context.
exc_val.__context__ = cleanup_error
else:
raise siren_error
raise cleanup_error

return False

def _get_stac_http_client(self) -> httpx.AsyncClient:
"""Return the pooled STAC transport owned by this client context."""
client = self._stac_http_client
if client is None or client.is_closed:
client = httpx.AsyncClient(timeout=30, follow_redirects=False)
self._stac_http_client = client
return client

@staticmethod
def _apply_zarr_group_metadata(ds: xr.Dataset, metadata: DatasetMetadata) -> None:
loaded_zarr_group = ds.attrs.get("_ipfs_zarr_group")
Expand Down Expand Up @@ -379,12 +396,12 @@ async def load_dataset(
# Try STAC server first (faster, avoids loading IPFS catalog)
if self._stac_server_url:
try:
resolved = await asyncio.to_thread(
resolve_cid_from_stac_server,
resolved = await aresolve_cid_from_stac_server(
Comment thread
0xSwego marked this conversation as resolved.
collection=resolved_collection,
dataset=dataset,
variant=variant,
server_url=self._stac_server_url,
client=self._get_stac_http_client(),
)
except (httpx.HTTPError, ValueError):
# Fall back when server lookup fails or returns no usable match.
Expand Down
Loading
Loading