Preliminary http2 optimization for clients

caterva2/api_utils.py

@@ -14,6 +14,35 @@
 import blosc2
 import httpx
 
+# Shared HTTP client instances for connection pooling
+_http_client = None
+_async_http_client = None
+
+
+def get_client(return_async_client=False):
+    """Get a shared HTTP client instance with HTTP/2 support and connection pooling."""
+    global _http_client, _async_http_client
+
+    if return_async_client:
+        if _async_http_client is None:
+            _async_http_client = httpx.AsyncClient(http2=True)
+        return _async_http_client
+    else:
+        if _http_client is None:
+            _http_client = httpx.Client(http2=True)
+        return _http_client
+
+
+def close_clients():
+    """Close shared HTTP clients. Call this when done with the API."""
+    global _http_client, _async_http_client
+    if _http_client is not None:
+        _http_client.close()
+        _http_client = None
+    if _async_http_client is not None:
+        _async_http_client.close()
+        _async_http_client = None
+
 
 def slice_to_string(slice_):
     if slice_ is None or slice_ == () or slice_ == slice(None):
@@ -178,15 +207,6 @@ def unfold_file(remotepath, urlbase, auth_cookie=None):
 #
 
 
-def get_client(return_async_client=False):
-    if return_async_client:
-        client_class = httpx.AsyncClient
-    else:
-        client_class = httpx.Client
-
-    return client_class()
-
-
 def _xget(url, params=None, headers=None, timeout=5, auth_cookie=None):
     client = get_client()
     if auth_cookie:

caterva2/client.py

@@ -719,6 +719,17 @@ def __init__(self, urlbase, auth=None, timeout=5):
                     self.urlbase, {"username": username, "password": password}, timeout=self.timeout
                 )
 
+    def __enter__(self):
+        """Enter context manager - HTTP clients created lazily on first use."""
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Exit context manager - close HTTP clients."""
+        from .api_utils import close_clients
+
+        close_clients()
+        return False
+
     def get_roots(self):
         """
         Retrieves the list of available roots.

examples/large-dataset-indexing.py

@@ -0,0 +1,48 @@
+###############################################################################
+# Caterva2 - On demand access to remote Blosc2 data repositories
+#
+# Copyright (c) 2025 ironArray SLU <contact@ironarray.io>
+# https://www.blosc.org
+# License: GNU Affero General Public License v3.0
+# See LICENSE.txt for details about copyright and rights to use.
+###############################################################################
+
+import math
+import time
+
+import blosc2
+
+import caterva2 as cat2
+
+urlbase = "https://cat2.cloud/demo"
+root = "@public"
+dataset = "examples/cube-1k-1k-1k.b2nd"
+
+# Blosc2 approach: simple, but allows slicing of remote datasets
+start = time.time()
+urlpath = blosc2.URLPath(f"{root}/{dataset}", urlbase)
+blosc2_ds = blosc2.open(urlpath, mode="r")
+print(f"Dataset size: {math.prod(blosc2_ds.shape) * blosc2_ds.dtype.itemsize / 2**20:.3f} MB")
+print(f"Blosc2 - Open dataset: {(time.time() - start) * 1000:.0f} ms")
+
+print(type(blosc2_ds), blosc2_ds.shape, blosc2_ds.dtype)
+start = time.time()
+print(blosc2_ds[500:502, 302, 900:905])
+print(f"Blosc2 - Slice dataset: {(time.time() - start) * 1000:.0f} ms")
+
+# Caterva2 approach: more flexible, allowing remote management in the server
+# Use Client as context manager for proper HTTP/2 connection cleanup
+with cat2.Client(urlbase) as client:
+    # Warmup: establish HTTP/2 connection
+    _ = client.get(root)
+
+    # Now measure with warm connection
+    start = time.time()
+    myroot = client.get(root)
+    cat2_ds = myroot[dataset]
+    print(f"Caterva2 - Get dataset handle: {(time.time() - start) * 1000:.0f} ms")
+
+    print(type(cat2_ds), cat2_ds.shape, cat2_ds.dtype)
+    start = time.time()
+    print(cat2_ds[500:502, 302, 900:905])
+    print(f"Caterva2 - Slice dataset: {(time.time() - start) * 1000:.0f} ms")

pyproject.toml

@@ -40,7 +40,7 @@ classifiers = [
 ]
 dependencies = [
     "blosc2>=3.2.0",
-    "httpx",
+    "httpx[http2]",
 ]
 
 [tool.hatch.version]