From 7d6138f686c1bebcc72b90d663e7653a9cdb6bc0 Mon Sep 17 00:00:00 2001 From: lshaw8317 Date: Thu, 30 Oct 2025 17:16:28 +0100 Subject: [PATCH 1/5] Commit CRLF --- doc/tutorials/hdf5.md | 218 +++++++++++++++++++++--------------------- 1 file changed, 109 insertions(+), 109 deletions(-) diff --git a/doc/tutorials/hdf5.md b/doc/tutorials/hdf5.md index 73c69645..374fcfe9 100644 --- a/doc/tutorials/hdf5.md +++ b/doc/tutorials/hdf5.md @@ -1,109 +1,109 @@ -(hdf5)= -# Working with HDF5 files -Caterva2 offers native support for working with HDF5 files. See [here](https://ironarray.wistia.com/medias/y64r14mojw) for a video demonstration. The notebook used is available on [github](https://github.com/ironArray/Caterva2/blob/main/examples/Video6-Cat2Cloud_and_HDF5.ipynb). In this tutorial, we will cover the same material: This tutorial demonstrates how to use the Cat2Cloud system with large HDF5 files. You'll learn how to upload, access, manipulate, and visualize HDF5 data both through proxy objects and Blosc2 arrays using the Caterva2 Python client. - -## What is an HDF5 file? -HDF stands for Hierarchical Data Format. [HDF5](https://www.hdfgroup.org/solutions/hdf5/) is a file format and set of tools for managing complex data. It is designed to store and organize large amounts of heterogeneous data, enabling quick access and efficient storage, using compression for example. HDF5 files are made up of a root (`/`) which may then contain the file contents. These contents are organised using the two main structures of HDF5 files - groups (which are like subdirectories) and datasets. Both datasets and groups possess metadata, which may include special, bespoke information about the object in the form of attributes (`attrs`). - -```{figure} images/hdf5-file.webp ---- -class: with-border - ---- - -Schematic of hdf5 file structure -``` - - -## Loading a HDF5 file -HDF5 files are common in many scientific and industrial applications, and so examples abound online. We're going to use some example diffraction data from a synchrotron, provided by the [silx project](http://www.silx.org/). This may be downloaded locally (after importing necessary libraries - see the notebook mentioned at the beginning of this tutorial): - -``` -dir_path = "kevlar" -if not os.path.exists(f"{dir_path}.h5"): - response = requests.get("http://www.silx. - org/pub/pyFAI/pyFAI_UM_2020/data_ID13 - /kevlar.h5") - with open(f"{dir_path}.h5", "wb") as file: - file.write(response.content) -``` - -## Unfolding the file -In order to handle the `.h5` file using Cat2Cloud, we must expose the hierarchical structure of the HDF5 file on the server, which in our case will be the `demo` server at https://cat2.cloud/demo. This is done by uploading the file to the Caterva2 server and then unfolding it, using a memory-light structure of subdirectories and proxy datasets. - -``` -url = "https://cat2.cloud/demo" -client = cat2.Client(url, - ("user@example.com", "foobar11")) -myroot = client.get("@shared") -print(f"Before uploading and unfolding: - {myroot.file_list}") -local_address = f"{dir_path}.h5" -remote_address = myroot.name + "/" + local_address -apath = client.upload(local_address, remote_address) -bloscpath = client.unfold(apath) -``` - -By running the line ``print(f"After uploading and unfolding: {myroot.file_list}")``, one can check that the `.h5` indeed has been correctly exposed. Note that one may also use the `unfold` command in the prompt on the web client, applying it to the uploaded file: -```{image} images/hdf5-unfold.webp -:alt: Unfold command -:width: 49.5% -``` -```{image} images/hdf5-unfold2.webp -:alt: Unfold result -:width: 49.5% -``` -The unfolded file structure is clearly visible in the second image. -We may now perform operations on the `.b2nd` proxy. - -## Examining the unfolded data -We can define a local reference which points to the data on the server (specifically the proxy), and we can obtain the easily obtain some of the data and plot it like so: -``` -proxy = myroot["kevlar/entry/data/data.b2nd"] -cmap = plt.cm.viridis -example_image = proxy[5] -fig = plt.figure() -plt.imshow(example_image / 65535, - figure=fig, cmap=cmap, vmax=1, vmin=0) -``` - -```{figure} images/hdf5-output1.webp ---- -class: with-border - ---- - -First visualisation -``` - -As is clear, not much detail can be seen in the image; this is due to outlier pixels maxing out the image range. We can use a `lazyexpr` to apply a function to the data, which will be executed on the server side, and then we can visualise the result. The following code applies a conditional expression to the data, increasing the signal pixels. - -``` -image = client.lazyexpr("expr", - "where(a < 10, a * 32000, a)", - {"a": proxy.path}) -example_image = client.get(image)[5] / 65535 -fig = plt.figure() -plt.imshow(example_image, figure=fig, - cmap=cmap, vmax=1, vmin=0) -``` -The result is an image with the desired diffraction pattern visible, as shown below: - -```{figure} images/hdf5-output2.webp ---- -class: with-border - ---- - -Second visualisation -``` -We can also go to the web client and directly visualize the lazy expression we have just generated and saved via the Tomography tab for the saved expression: - -```{figure} images/hdf5-tomo.webp ---- -class: with-border - ---- - -Second visualisation -``` +(hdf5)= +# Working with HDF5 files +Caterva2 offers native support for working with HDF5 files. See [here](https://ironarray.wistia.com/medias/y64r14mojw) for a video demonstration. The notebook used is available on [github](https://github.com/ironArray/Caterva2/blob/main/examples/Video6-Cat2Cloud_and_HDF5.ipynb). In this tutorial, we will cover the same material: This tutorial demonstrates how to use the Cat2Cloud system with large HDF5 files. You'll learn how to upload, access, manipulate, and visualize HDF5 data both through proxy objects and Blosc2 arrays using the Caterva2 Python client. + +## What is an HDF5 file? +HDF stands for Hierarchical Data Format. [HDF5](https://www.hdfgroup.org/solutions/hdf5/) is a file format and set of tools for managing complex data. It is designed to store and organize large amounts of heterogeneous data, enabling quick access and efficient storage, using compression for example. HDF5 files are made up of a root (`/`) which may then contain the file contents. These contents are organised using the two main structures of HDF5 files - groups (which are like subdirectories) and datasets. Both datasets and groups possess metadata, which may include special, bespoke information about the object in the form of attributes (`attrs`). + +```{figure} images/hdf5-file.webp +--- +class: with-border + +--- + +Schematic of hdf5 file structure +``` + + +## Loading a HDF5 file +HDF5 files are common in many scientific and industrial applications, and so examples abound online. We're going to use some example diffraction data from a synchrotron, provided by the [silx project](http://www.silx.org/). This may be downloaded locally (after importing necessary libraries - see the notebook mentioned at the beginning of this tutorial): + +``` +dir_path = "kevlar" +if not os.path.exists(f"{dir_path}.h5"): + response = requests.get("http://www.silx. + org/pub/pyFAI/pyFAI_UM_2020/data_ID13 + /kevlar.h5") + with open(f"{dir_path}.h5", "wb") as file: + file.write(response.content) +``` + +## Unfolding the file +In order to handle the `.h5` file using Cat2Cloud, we must expose the hierarchical structure of the HDF5 file on the server, which in our case will be the `demo` server at https://cat2.cloud/demo. This is done by uploading the file to the Caterva2 server and then unfolding it, using a memory-light structure of subdirectories and proxy datasets. + +``` +url = "https://cat2.cloud/demo" +client = cat2.Client(url, + ("user@example.com", "foobar11")) +myroot = client.get("@shared") +print(f"Before uploading and unfolding: + {myroot.file_list}") +local_address = f"{dir_path}.h5" +remote_address = myroot.name + "/" + local_address +apath = client.upload(local_address, remote_address) +bloscpath = client.unfold(apath) +``` + +By running the line ``print(f"After uploading and unfolding: {myroot.file_list}")``, one can check that the `.h5` indeed has been correctly exposed. Note that one may also use the `unfold` command in the prompt on the web client, applying it to the uploaded file: +```{image} images/hdf5-unfold.webp +:alt: Unfold command +:width: 49.5% +``` +```{image} images/hdf5-unfold2.webp +:alt: Unfold result +:width: 49.5% +``` +The unfolded file structure is clearly visible in the second image. +We may now perform operations on the `.b2nd` proxy. + +## Examining the unfolded data +We can define a local reference which points to the data on the server (specifically the proxy), and we can obtain the easily obtain some of the data and plot it like so: +``` +proxy = myroot["kevlar/entry/data/data.b2nd"] +cmap = plt.cm.viridis +example_image = proxy[5] +fig = plt.figure() +plt.imshow(example_image / 65535, + figure=fig, cmap=cmap, vmax=1, vmin=0) +``` + +```{figure} images/hdf5-output1.webp +--- +class: with-border + +--- + +First visualisation +``` + +As is clear, not much detail can be seen in the image; this is due to outlier pixels maxing out the image range. We can use a `lazyexpr` to apply a function to the data, which will be executed on the server side, and then we can visualise the result. The following code applies a conditional expression to the data, increasing the signal pixels. + +``` +image = client.lazyexpr("expr", + "where(a < 10, a * 32000, a)", + {"a": proxy.path}) +example_image = client.get(image)[5] / 65535 +fig = plt.figure() +plt.imshow(example_image, figure=fig, + cmap=cmap, vmax=1, vmin=0) +``` +The result is an image with the desired diffraction pattern visible, as shown below: + +```{figure} images/hdf5-output2.webp +--- +class: with-border + +--- + +Second visualisation +``` +We can also go to the web client and directly visualize the lazy expression we have just generated and saved via the Tomography tab for the saved expression: + +```{figure} images/hdf5-tomo.webp +--- +class: with-border + +--- + +Second visualisation +``` From 41ec15c269d0d19844967e493175373d23991f0c Mon Sep 17 00:00:00 2001 From: lshaw8317 Date: Sat, 1 Nov 2025 13:11:06 +0100 Subject: [PATCH 2/5] Add ndim attr to HDF5 for linalg funcs --- caterva2/hdf5.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/caterva2/hdf5.py b/caterva2/hdf5.py index fe7c4809..dfe79d45 100644 --- a/caterva2/hdf5.py +++ b/caterva2/hdf5.py @@ -385,6 +385,10 @@ def __init__(self, b2arr, h5file=None, dsetname=None): def shape(self) -> tuple[int, ...]: return self.b2arr.shape + @property + def ndim(self) -> tuple[int, ...]: + return self.b2arr.ndim + @property def chunks(self): return self.b2arr.chunks From 1b768323111c7863494b67c991056e5c2b2f8c4b Mon Sep 17 00:00:00 2001 From: lshaw8317 Date: Sat, 1 Nov 2025 13:12:45 +0100 Subject: [PATCH 3/5] Add linalg test --- caterva2/tests/test_hdf5_proxy.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/caterva2/tests/test_hdf5_proxy.py b/caterva2/tests/test_hdf5_proxy.py index 51556744..8ae6a8ea 100644 --- a/caterva2/tests/test_hdf5_proxy.py +++ b/caterva2/tests/test_hdf5_proxy.py @@ -297,12 +297,7 @@ def test_unfold_fetch(fetch_or_slice, examples_dir, tmp_path, auth_client): # Exercises the expression evaluation with HDF5 proxies @pytest.mark.parametrize( "expression", - [ - "a + 50", - "a ** 2.3 + b / 2.3", - "sqrt(a) ** sin(b)", - "where(a < 50, a + 50, b)", - ], + ["a + 50", "a ** 2.3 + b / 2.3", "sqrt(a) ** sin(b)", "where(a < 50, a + 50, b)", "matmul(a, b)"], ) def test_expression(expression, examples_dir, tmp_path, auth_client): if not auth_client: From b0fbef936961042db480bbbacdb91f596136fcf6 Mon Sep 17 00:00:00 2001 From: lshaw8317 Date: Sat, 1 Nov 2025 13:24:49 +0100 Subject: [PATCH 4/5] Fix test --- caterva2/tests/test_hdf5_proxy.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/caterva2/tests/test_hdf5_proxy.py b/caterva2/tests/test_hdf5_proxy.py index 8ae6a8ea..385dae0e 100644 --- a/caterva2/tests/test_hdf5_proxy.py +++ b/caterva2/tests/test_hdf5_proxy.py @@ -326,5 +326,8 @@ def test_expression(expression, examples_dir, tmp_path, auth_client): # Check the data na = h5f[ds_a][:] nb = h5f[ds_b][:] - nresult = ne.evaluate(expression, {"a": na, "b": nb}) + if expression != "matmul(a, b)": + nresult = ne.evaluate(expression, {"a": na, "b": nb}) + else: + nresult = np.matmul(na, nb) np.testing.assert_allclose(result[:], nresult) From 97987d1d7da0f990e359544080428e696c31b75f Mon Sep 17 00:00:00 2001 From: lshaw8317 Date: Sat, 1 Nov 2025 15:33:23 +0100 Subject: [PATCH 5/5] Reformat --- caterva2/tests/test_hdf5_proxy.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/caterva2/tests/test_hdf5_proxy.py b/caterva2/tests/test_hdf5_proxy.py index 385dae0e..fc6296b0 100644 --- a/caterva2/tests/test_hdf5_proxy.py +++ b/caterva2/tests/test_hdf5_proxy.py @@ -297,7 +297,13 @@ def test_unfold_fetch(fetch_or_slice, examples_dir, tmp_path, auth_client): # Exercises the expression evaluation with HDF5 proxies @pytest.mark.parametrize( "expression", - ["a + 50", "a ** 2.3 + b / 2.3", "sqrt(a) ** sin(b)", "where(a < 50, a + 50, b)", "matmul(a, b)"], + [ + "a + 50", + "a ** 2.3 + b / 2.3", + "sqrt(a) ** sin(b)", + "where(a < 50, a + 50, b)", + "matmul(a, b)", + ], ) def test_expression(expression, examples_dir, tmp_path, auth_client): if not auth_client: