Comprehensive audit: 12 proven bugs fixed, 160x spatial selection, test suite restored from 14 to 166 local tests

DATASET_CATALOG_USAGE.md

@@ -85,15 +85,15 @@ datasets = list_available_datasets(catalog)
 # }
 
 # Resolve a specific dataset to its CID
-cid = resolve_dataset_cid_from_stac(
+resolved = resolve_dataset_cid_from_stac(
     catalog=catalog,
     collection="ecmwf_era5",
     dataset="temperature_2m",
     variant="finalized",  # Optional, only needed if dataset has multiple variants
     organization="ecmwf",
 )
 
-print(f"Dataset CID: {cid}")
+print(f"Dataset CID: {resolved.cid} (variant: {resolved.variant})")
 ```
 
 ## STAC Catalog Structure
@@ -317,7 +317,7 @@ def resolve_dataset_cid_from_stac(
     collection: str,
     dataset: str,
     variant: Optional[str] = None
-) -> str
+) -> ResolvedDataset
 ```
 
 **Parameters:**
@@ -327,7 +327,7 @@ def resolve_dataset_cid_from_stac(
 - `variant`: Optional variant name
 
 **Returns:**
-- `str`: IPFS CID (without "ipfs://" prefix)
+- `ResolvedDataset`: IPFS CID (without "ipfs://" prefix) and selected variant
 
 **Raises:**
 - `ValueError`: If collection, dataset, or variant is not found
@@ -340,14 +340,14 @@ from dclimate_client_py import load_stac_catalog, resolve_dataset_cid_from_stac
 catalog = load_stac_catalog("https://ipfs-gateway.dclimate.net")
 
 # Resolve dataset to CID
-cid = resolve_dataset_cid_from_stac(
+resolved = resolve_dataset_cid_from_stac(
     catalog=catalog,
     collection="ifs",
     dataset="temperature",
     variant="single"
 )
 
-print(f"Dataset CID: {cid}")
+print(f"Dataset CID: {resolved.cid} (variant: {resolved.variant})")
 ```
 
 ### `get_root_catalog_cid()`
@@ -362,7 +362,7 @@ def get_root_catalog_cid() -> str
 - `str`: The IPFS CID of the latest root STAC catalog
 
 **Raises:**
-- `requests.HTTPError`: If the API request fails
+- `httpx.HTTPError`: If the API request fails
 - `KeyError`: If response doesn't contain expected 'cid' field
 
 **Example:**

README.md

@@ -5,7 +5,7 @@
 </p>
 
 # dClimate-Client-Py
-[![codecov](https://codecov.io/gh/dClimate/dClimate-Zarr-Client/graph/badge.svg?token=AovaMO6DX5)](https://codecov.io/gh/dClimate/dClimate-Zarr-Client)
+[![codecov](https://codecov.io/gh/dClimate/dclimate-client-py/graph/badge.svg)](https://codecov.io/gh/dClimate/dclimate-client-py)
 
 Retrieve dClimate GIS zarr datasets stored on IPFS
 
@@ -226,11 +226,11 @@ Various exceptions to be raised for bad or invalid user input.
 
 ---
 
-### geo_utils.py
+### geotemporal_data.py
 
-Functions to manipulate `xarray` datasets. Contains polygon, rectangle, circle and point spatial
-subsetting options, as well as temporal subsetting. Also allows for both spatial and temporal
-aggregations.
+`GeotemporalData`, a wrapper around `xarray` datasets. Contains polygon, rectangle, circle and
+point spatial subsetting options, as well as temporal subsetting. Also allows for both spatial
+and temporal aggregations.
 
 ---
 

dclimate_client_py/__init__.py

@@ -1,10 +1,6 @@
 # public API
-from .client import (
-    load_s3,
-    geo_temporal_query,
-)
-from .dclimate_client import dClimateClient
-from .geotemporal_data import GeotemporalData
+from importlib import import_module
+
 from .encryption_codec import (
     EncryptionCodec,
 )
@@ -16,11 +12,8 @@
     SpatialExtent,
     TemporalExtent,
 )
-from .stac_catalog import (
-    load_stac_catalog,
-    list_available_datasets,
-)
 from .stac_server import (
+    ResolvedDataset,
     resolve_cid_from_stac_server,
     list_available_datasets_from_stac_server,
     STAC_SERVER_URL,
@@ -43,6 +36,31 @@
     X402NotInstalledError,
 )
 
+_LAZY_IMPORTS = {
+    "load_s3": (".client", "load_s3"),
+    "geo_temporal_query": (".client", "geo_temporal_query"),
+    "dClimateClient": (".dclimate_client", "dClimateClient"),
+    "GeotemporalData": (".geotemporal_data", "GeotemporalData"),
+    "load_stac_catalog": (".stac_catalog", "load_stac_catalog"),
+    "list_available_datasets": (".stac_catalog", "list_available_datasets"),
+}
+
+
+def __getattr__(name: str):
+    try:
+        module_name, attribute_name = _LAZY_IMPORTS[name]
+    except KeyError:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None
+
+    value = getattr(import_module(module_name, __name__), attribute_name)
+    globals()[name] = value
+    return value
+
+
+def __dir__() -> list[str]:
+    return sorted(set(globals()) | set(_LAZY_IMPORTS))
+
+
 __all__ = [
     "dClimateClient",
     "load_s3",
@@ -57,6 +75,7 @@
     "TemporalExtent",
     "load_stac_catalog",
     "list_available_datasets",
+    "ResolvedDataset",
     "resolve_cid_from_stac_server",
     "list_available_datasets_from_stac_server",
     "STAC_SERVER_URL",

dclimate_client_py/client.py

@@ -36,21 +36,21 @@ def load_s3(
 def geo_temporal_query(
     dataset_name: str,
     source: typing.Literal["s3"] = "s3",
-    bucket_name: str = None,
-    var_name: str = None,
+    bucket_name: typing.Optional[str] = None,
+    var_name: typing.Optional[str] = None,
     gateway_uri_stem: str | None = None,
     rpc_uri_stem: str | None = None,
-    forecast_reference_time: str = None,
-    point_kwargs: dict = None,
-    circle_kwargs: dict = None,
-    rectangle_kwargs: dict = None,
-    polygon_kwargs: dict = None,
-    multiple_points_kwargs: dict = None,
+    forecast_reference_time: typing.Optional[str] = None,
+    point_kwargs: typing.Optional[dict] = None,
+    circle_kwargs: typing.Optional[dict] = None,
+    rectangle_kwargs: typing.Optional[dict] = None,
+    polygon_kwargs: typing.Optional[dict] = None,
+    multiple_points_kwargs: typing.Optional[dict] = None,
     bounds=None,
-    bounds_options: dict = None,
-    spatial_agg_kwargs: dict = None,
-    temporal_agg_kwargs: dict = None,
-    rolling_agg_kwargs: dict = None,
+    bounds_options: typing.Optional[dict] = None,
+    spatial_agg_kwargs: typing.Optional[dict] = None,
+    temporal_agg_kwargs: typing.Optional[dict] = None,
+    rolling_agg_kwargs: typing.Optional[dict] = None,
     time_range: typing.Optional[typing.List[datetime.datetime]] = None,
     # as_of: typing.Optional[datetime.datetime] = None, # Removed as_of
     point_limit: int = DEFAULT_POINT_LIMIT,

dclimate_client_py/concatenate.py

@@ -17,7 +17,6 @@
 
 
 def find_split_index(
-    combined_coords: typing.Any,
     next_coords: typing.Any,
     last_coord_value: typing.Any,
 ) -> int:
@@ -27,7 +26,6 @@ def find_split_index(
     This prevents duplicate data when concatenating datasets.
 
     Args:
-        combined_coords: Coordinates from the combined dataset (for reference)
         next_coords: Coordinates from the next variant to concatenate
         last_coord_value: The last coordinate value from the combined dataset
 
@@ -109,10 +107,11 @@ async def concatenate_datasets(
     Implements smart concatenation logic:
     1. Start with first dataset (highest priority)
     2. For each subsequent dataset:
-       - Find the last coordinate value in combined dataset
-       - Find split index in next dataset where coords > last coord
+       - Find split index in next dataset where coords > the last coord
+         accepted so far
        - Slice next dataset to only include new data
-       - Concatenate sliced dataset
+    3. Concatenate all accepted slices with a single ``xr.concat`` call
+       (an iterative per-dataset concat is O(n^2) time and ~2x peak memory)
 
     Args:
         datasets: List of xarray datasets to concatenate (in priority order)
@@ -144,16 +143,16 @@ async def concatenate_datasets(
 
     # Start with the first dataset (highest priority)
     combined = datasets[0]
+    datasets_to_concat = [combined]
+    last_coord_value = combined[dimension].values[-1]
+    total_coord_count = len(combined[dimension])
     logger.debug(
         f"Starting with dataset 1/{len(datasets)}, "
         f"{dimension} range: {combined[dimension].values[0]} to {combined[dimension].values[-1]}"
     )
 
     # Concatenate each subsequent dataset
     for i, next_ds in enumerate(datasets[1:], start=2):
-        # Get the last coordinate value from combined dataset
-        last_coord_value = combined[dimension].values[-1]
-
         # Get coordinates from next dataset
         next_coords = next_ds[dimension].values
 
@@ -165,7 +164,6 @@ async def concatenate_datasets(
         # Find where to split the next dataset
         try:
             split_index = find_split_index(
-                combined[dimension].values,
                 next_coords,
                 last_coord_value,
             )
@@ -177,25 +175,26 @@ async def concatenate_datasets(
 
             # Slice the next dataset to only include new data
             sliced_next = next_ds.isel({dimension: slice(split_index, None)})
+            datasets_to_concat.append(sliced_next)
+            last_coord_value = sliced_next[dimension].values[-1]
+            total_coord_count += len(sliced_next[dimension])
 
             logger.debug(
                 f"Sliced dataset {i} to {len(sliced_next[dimension])} new coords"
             )
 
-            # Concatenate with combined dataset
-            combined = xr.concat(
-                [combined, sliced_next],
-                dim=dimension,
-            )
             logger.debug(
-                f"After concatenating dataset {i}, total {dimension} coords: {len(combined[dimension])}"
+                f"After concatenating dataset {i}, total {dimension} coords: {total_coord_count}"
             )
 
         except NoDataFoundError as e:
             logger.warning(f"Skipping dataset {i} as it contains no new data: {e}")
             # Continue to next dataset
             continue
 
+    if len(datasets_to_concat) > 1:
+        combined = xr.concat(datasets_to_concat, dim=dimension)
+
     logger.info(
         f"Concatenation complete. Final dataset has {len(combined[dimension])} "
         f"{dimension} coordinates ranging from {combined[dimension].values[0]} "