From c55655827ddf4b982b459b95d0c549737926b51d Mon Sep 17 00:00:00 2001 From: lshaw8317 Date: Fri, 7 Nov 2025 11:02:51 +0100 Subject: [PATCH 1/5] Adding cbytes/cratio to HDF5Proxy --- caterva2/hdf5.py | 8 ++++++++ caterva2/tests/test_hdf5_proxy.py | 3 +++ 2 files changed, 11 insertions(+) diff --git a/caterva2/hdf5.py b/caterva2/hdf5.py index dfe79d45..7932fd83 100644 --- a/caterva2/hdf5.py +++ b/caterva2/hdf5.py @@ -414,6 +414,14 @@ def schunk(self): # This is basically needed to certificate that it is an NDArray in the LazyArray machinery return self.b2arr.schunk + @property + def cbytes(self): + return self.b2arr.cbytes + + @property + def cratio(self): + return self.b2arr.cratio + @property def fields(self) -> Mapping[str, numpy.dtype]: return self.b2arr.fields diff --git a/caterva2/tests/test_hdf5_proxy.py b/caterva2/tests/test_hdf5_proxy.py index 1aa61822..222d8a70 100644 --- a/caterva2/tests/test_hdf5_proxy.py +++ b/caterva2/tests/test_hdf5_proxy.py @@ -243,6 +243,9 @@ def test_unfold_download(examples_dir, tmp_path, auth_client): continue assert b2f.dtype == h5ds.dtype assert b2f.shape == (h5ds.shape or ()) + b2nd_pointer = auth_client.get(remote_path) + assert b2f.cbytes == b2nd_pointer.meta["cbytes"] + assert b2f.cratio == b2nd_pointer.meta["cratio"] if b2f.shape == (): continue if h5ds.chunks: From 36794b5287ecf5280abb89c0acfe889296cff64a Mon Sep 17 00:00:00 2001 From: Luke Shaw Date: Fri, 7 Nov 2025 16:16:50 +0100 Subject: [PATCH 2/5] Now show cbytes/cratio in web client for proxies --- caterva2/hdf5.py | 14 +++++++++----- caterva2/services/srv_utils.py | 7 ++++++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/caterva2/hdf5.py b/caterva2/hdf5.py index 7932fd83..3404f56d 100644 --- a/caterva2/hdf5.py +++ b/caterva2/hdf5.py @@ -386,7 +386,7 @@ def shape(self) -> tuple[int, ...]: return self.b2arr.shape @property - def ndim(self) -> tuple[int, ...]: + def ndim(self) -> int: return self.b2arr.ndim @property @@ -415,12 +415,16 @@ def schunk(self): return self.b2arr.schunk @property - def cbytes(self): - return self.b2arr.cbytes + def cbytes(self) -> int: + return self.dset.id.get_storage_size() @property - def cratio(self): - return self.b2arr.cratio + def cratio(self) -> float: + return self.nbytes / self.cbytes + + @property + def nbytes(self) -> int: + return self.b2arr.nbytes @property def fields(self) -> Mapping[str, numpy.dtype]: diff --git a/caterva2/services/srv_utils.py b/caterva2/services/srv_utils.py index 4b4b1c2e..99ca4217 100644 --- a/caterva2/services/srv_utils.py +++ b/caterva2/services/srv_utils.py @@ -26,7 +26,7 @@ from sqlalchemy.future import select # Project -from caterva2 import models +from caterva2 import hdf5, models from caterva2.services import db, schemas, settings, users @@ -119,9 +119,14 @@ def read_metadata(obj): # Read metadata if isinstance(obj, blosc2.ndarray.NDArray): array = obj + vlmeta = array.schunk.vlmeta cparams = get_model_from_obj(array.schunk.cparams, models.CParams) cparams = reformat_cparams(cparams) schunk = get_model_from_obj(array.schunk, models.SChunk, cparams=cparams) + if "_ftype" in vlmeta and vlmeta["_ftype"] == "hdf5": + array = hdf5.HDF5Proxy(array) + schunk.cratio = array.cratio # overwrite cratio (which will be 0) with HDF5Proxy value + schunk.cbytes = array.cbytes return get_model_from_obj(array, models.Metadata, schunk=schunk, mtime=mtime) elif isinstance(obj, blosc2.schunk.SChunk): schunk = obj From 4850736fa3a0fdcb98bf225c3c564dace62cdb3a Mon Sep 17 00:00:00 2001 From: Luke Shaw Date: Fri, 7 Nov 2025 16:19:51 +0100 Subject: [PATCH 3/5] Slight streamline --- caterva2/services/srv_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/caterva2/services/srv_utils.py b/caterva2/services/srv_utils.py index 99ca4217..75725ab7 100644 --- a/caterva2/services/srv_utils.py +++ b/caterva2/services/srv_utils.py @@ -119,11 +119,10 @@ def read_metadata(obj): # Read metadata if isinstance(obj, blosc2.ndarray.NDArray): array = obj - vlmeta = array.schunk.vlmeta cparams = get_model_from_obj(array.schunk.cparams, models.CParams) cparams = reformat_cparams(cparams) schunk = get_model_from_obj(array.schunk, models.SChunk, cparams=cparams) - if "_ftype" in vlmeta and vlmeta["_ftype"] == "hdf5": + if "_ftype" in schunk.vlmeta and schunk.vlmeta["_ftype"] == "hdf5": array = hdf5.HDF5Proxy(array) schunk.cratio = array.cratio # overwrite cratio (which will be 0) with HDF5Proxy value schunk.cbytes = array.cbytes From 9487b7e841dfb1e1a8d4370a997e8f9a43e39a5e Mon Sep 17 00:00:00 2001 From: lshaw8317 Date: Mon, 10 Nov 2025 10:11:09 +0100 Subject: [PATCH 4/5] Modify test --- caterva2/hdf5.py | 2 +- caterva2/tests/test_hdf5_proxy.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/caterva2/hdf5.py b/caterva2/hdf5.py index 3404f56d..b6f515a5 100644 --- a/caterva2/hdf5.py +++ b/caterva2/hdf5.py @@ -420,7 +420,7 @@ def cbytes(self) -> int: @property def cratio(self) -> float: - return self.nbytes / self.cbytes + return 0 if self.cbytes == 0 else self.nbytes / self.cbytes @property def nbytes(self) -> int: diff --git a/caterva2/tests/test_hdf5_proxy.py b/caterva2/tests/test_hdf5_proxy.py index 222d8a70..a20dd74c 100644 --- a/caterva2/tests/test_hdf5_proxy.py +++ b/caterva2/tests/test_hdf5_proxy.py @@ -244,8 +244,8 @@ def test_unfold_download(examples_dir, tmp_path, auth_client): assert b2f.dtype == h5ds.dtype assert b2f.shape == (h5ds.shape or ()) b2nd_pointer = auth_client.get(remote_path) - assert b2f.cbytes == b2nd_pointer.meta["cbytes"] - assert b2f.cratio == b2nd_pointer.meta["cratio"] + assert b2f.cbytes == b2nd_pointer.meta["schunk"]["cbytes"] + assert b2f.cratio == b2nd_pointer.meta["schunk"]["cratio"] if b2f.shape == (): continue if h5ds.chunks: From e8454d12c338a9637026b887b371b232af15b7a6 Mon Sep 17 00:00:00 2001 From: Luke Shaw Date: Mon, 10 Nov 2025 21:04:12 +0100 Subject: [PATCH 5/5] Fix test for empty arrays --- caterva2/tests/test_hdf5_proxy.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/caterva2/tests/test_hdf5_proxy.py b/caterva2/tests/test_hdf5_proxy.py index a20dd74c..5208c64e 100644 --- a/caterva2/tests/test_hdf5_proxy.py +++ b/caterva2/tests/test_hdf5_proxy.py @@ -244,8 +244,9 @@ def test_unfold_download(examples_dir, tmp_path, auth_client): assert b2f.dtype == h5ds.dtype assert b2f.shape == (h5ds.shape or ()) b2nd_pointer = auth_client.get(remote_path) - assert b2f.cbytes == b2nd_pointer.meta["schunk"]["cbytes"] - assert b2f.cratio == b2nd_pointer.meta["schunk"]["cratio"] + if b2f.shape != (): # skip empty datasets + assert b2f.cbytes == b2nd_pointer.meta["schunk"]["cbytes"] + assert b2f.cratio == b2nd_pointer.meta["schunk"]["cratio"] if b2f.shape == (): continue if h5ds.chunks: