From d1f129429b85176a9e34d281fd70082eb4110db5 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Tue, 30 Jun 2026 16:39:56 +0100 Subject: [PATCH 01/21] Start by moving test for utils --- tests/tools/conftest.py | 12 ++++++++++++ tests/tools/test_VDS_tools.py | 16 ---------------- tests/tools/vds/__init__.py | 0 tests/tools/vds/test_utils.py | 8 ++++++++ 4 files changed, 20 insertions(+), 16 deletions(-) create mode 100644 tests/tools/conftest.py create mode 100644 tests/tools/vds/__init__.py create mode 100644 tests/tools/vds/test_utils.py diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py new file mode 100644 index 00000000..d70af372 --- /dev/null +++ b/tests/tools/conftest.py @@ -0,0 +1,12 @@ +import tempfile + +import h5py +import pytest + + +@pytest.fixture +def nexus_file_with_single_dataset(): + test_hdf_file = tempfile.TemporaryFile() + test_nexus_file = h5py.File(test_hdf_file, "w") + test_nexus_file["/entry/data/data_0001"] = h5py.ExternalLink("filename", "path") + yield test_nexus_file diff --git a/tests/tools/test_VDS_tools.py b/tests/tools/test_VDS_tools.py index 722eb6e4..334933e3 100644 --- a/tests/tools/test_VDS_tools.py +++ b/tests/tools/test_VDS_tools.py @@ -9,7 +9,6 @@ Dataset, create_virtual_layout, define_vds_dtype_from_bit_depth, - find_datasets_in_file, image_vds_writer, jungfrau_vds_writer, split_datasets, @@ -91,21 +90,6 @@ def test_when_start_idx_negative_then_exception_raised(): split_datasets(["test1"], (1100, 10, 10), -100) -@pytest.fixture -def nexus_file_with_single_dataset(): - test_hdf_file = tempfile.TemporaryFile() - test_nexus_file = h5py.File(test_hdf_file, "w") - test_nexus_file["/entry/data/data_0001"] = h5py.ExternalLink("filename", "path") - yield test_nexus_file - - -def test_find_datasets_int_file(nexus_file_with_single_dataset): - nxdata = nexus_file_with_single_dataset["/entry/data"] - dsets = find_datasets_in_file(nxdata) - assert len(dsets) == 1 - assert dsets[0] == "data_0001" - - def test_when_float_shape_passed_to_vds_writer_then_no_exception( nexus_file_with_single_dataset, ): diff --git a/tests/tools/vds/__init__.py b/tests/tools/vds/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/tools/vds/test_utils.py b/tests/tools/vds/test_utils.py new file mode 100644 index 00000000..9e000c82 --- /dev/null +++ b/tests/tools/vds/test_utils.py @@ -0,0 +1,8 @@ +from nexgen.tools.vds_tools.utils import find_datasets_in_file + + +def test_find_datasets_int_file(nexus_file_with_single_dataset): + nxdata = nexus_file_with_single_dataset["/entry/data"] + dsets = find_datasets_in_file(nxdata) + assert len(dsets) == 1 + assert dsets[0] == "data_0001" From e508c94230543f1e6c947a35ae54b9252815632b Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Tue, 30 Jun 2026 16:44:54 +0100 Subject: [PATCH 02/21] Add a test for data error --- tests/tools/vds/test_utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/tools/vds/test_utils.py b/tests/tools/vds/test_utils.py index 9e000c82..7bc07c6f 100644 --- a/tests/tools/vds/test_utils.py +++ b/tests/tools/vds/test_utils.py @@ -1,3 +1,8 @@ +import tempfile + +import h5py +import pytest + from nexgen.tools.vds_tools.utils import find_datasets_in_file @@ -6,3 +11,11 @@ def test_find_datasets_int_file(nexus_file_with_single_dataset): dsets = find_datasets_in_file(nxdata) assert len(dsets) == 1 assert dsets[0] == "data_0001" + + +def test_find_datasets_fails_if_no_links_in_file(): + test_hdf_file = tempfile.TemporaryFile() + test_nexus_file = h5py.File(test_hdf_file, "w") + test_nexus_file["/entry/data/data_0001"] = [0, 0, 0] + with pytest.raises(KeyError): + find_datasets_in_file(test_nexus_file) From 9d2463809883c9e8bb5b695f759c28d27537c17b Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Jul 2026 11:13:43 +0100 Subject: [PATCH 03/21] Fix typo and add tests for vds writer --- src/nexgen/tools/vds_tools/strided_mapping.py | 2 +- tests/tools/conftest.py | 24 ++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/nexgen/tools/vds_tools/strided_mapping.py b/src/nexgen/tools/vds_tools/strided_mapping.py index 2ef8f0d1..30f57c46 100644 --- a/src/nexgen/tools/vds_tools/strided_mapping.py +++ b/src/nexgen/tools/vds_tools/strided_mapping.py @@ -17,7 +17,7 @@ class SingleDataset: name: str # Full size of the source dataset - src_shape = Sequence[int] + src_shape: Sequence[int] # Index to start the mapping from, usually 0 or 1 for this start_index: int = 0 # Step for slicing the dataset. Defaults to every other image diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index d70af372..4915afdc 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -1,6 +1,7 @@ import tempfile import h5py +import numpy as np import pytest @@ -8,5 +9,26 @@ def nexus_file_with_single_dataset(): test_hdf_file = tempfile.TemporaryFile() test_nexus_file = h5py.File(test_hdf_file, "w") - test_nexus_file["/entry/data/data_0001"] = h5py.ExternalLink("filename", "path") + test_nexus_file["/entry/data/data_0001"] = h5py.ExternalLink("filename", "data") + yield test_nexus_file + + +@pytest.fixture +def dummy_data_file(): + test_data_file = tempfile.NamedTemporaryFile(suffix=".h5", delete=True) + with h5py.File(test_data_file, "w") as fh: + fh["data"] = np.zeros((10, 2, 3)) + yield test_data_file + + +@pytest.fixture +def nexus_file_with_multiple_datasets(dummy_data_file): + test_hdf_file = tempfile.TemporaryFile() + test_nexus_file = h5py.File(test_hdf_file, "w") + test_nexus_file["/entry/data/data_0001"] = h5py.ExternalLink( + dummy_data_file.name, "data" + ) + test_nexus_file["/entry/data/data_0002"] = h5py.ExternalLink( + dummy_data_file.name, "data" + ) yield test_nexus_file From 4e1a8585d88ca7e6017a5e2329ea1dea350a8edd Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Jul 2026 11:14:53 +0100 Subject: [PATCH 04/21] Maybe save all files --- tests/tools/vds/test_strided_vds.py | 47 +++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tests/tools/vds/test_strided_vds.py diff --git a/tests/tools/vds/test_strided_vds.py b/tests/tools/vds/test_strided_vds.py new file mode 100644 index 00000000..d45d8a05 --- /dev/null +++ b/tests/tools/vds/test_strided_vds.py @@ -0,0 +1,47 @@ +from unittest.mock import patch + +import numpy as np + +from nexgen.tools.vds_tools.strided_mapping import ( + SingleDataset, + create_dataset_list, + create_vds_layout, + write_strided_vds, +) + + +def test_create_dataset_list(nexus_file_with_multiple_datasets): + dsets = create_dataset_list( + nexus_file_with_multiple_datasets["/entry/data"], start_index=0 + ) + + assert len(dsets) == 2 + assert dsets[0].name == "data_0001" + assert dsets[1].name == "data_0002" + + for dset in dsets: + assert dset.start_index == 0 + assert dset.stride == 2 + + +def test_create_vds_layout(): + dset_list = [ + SingleDataset(name="data_01", src_shape=(6, 2, 2), start_index=1, stride=2) + ] + expected_shape = (3, 2, 2) + + layout = create_vds_layout(dset_list, expected_shape, np.uint16) + + assert layout.shape == expected_shape + assert list(layout._src_filenames)[0] == b"." + + +@patch("nexgen.tools.vds_tools.strided_mapping.create_vds_layout") +def test_write_strided_vds(mock_layout, nexus_file_with_multiple_datasets): + with patch( + "nexgen.tools.vds_tools.strided_mapping.h5py.Group.create_virtual_dataset" + ) as mock_create: + write_strided_vds(nexus_file_with_multiple_datasets, (10, 2, 3), 0) + + mock_layout.assert_called_once() + mock_create.assert_called_once() From 7624ac3ab003454c6ef79a2a5e282b9029c1ba77 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Jul 2026 11:28:09 +0100 Subject: [PATCH 05/21] Try to fix ci --- tests/tools/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index 4915afdc..4158f45c 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -13,7 +13,7 @@ def nexus_file_with_single_dataset(): yield test_nexus_file -@pytest.fixture +@pytest.fixture(scope="session") def dummy_data_file(): test_data_file = tempfile.NamedTemporaryFile(suffix=".h5", delete=True) with h5py.File(test_data_file, "w") as fh: From 42552227b37ac5d9240928d7637b7b4efed8df63 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Jul 2026 11:31:16 +0100 Subject: [PATCH 06/21] Try to fix ci again --- tests/tools/conftest.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index 4158f45c..4099b1db 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -22,13 +22,16 @@ def dummy_data_file(): @pytest.fixture -def nexus_file_with_multiple_datasets(dummy_data_file): +def nexus_file_with_multiple_datasets(): + test_data_file = tempfile.NamedTemporaryFile(suffix=".h5", delete=True) + with h5py.File(test_data_file, "w") as fh: + fh["data"] = np.zeros((10, 2, 3)) test_hdf_file = tempfile.TemporaryFile() test_nexus_file = h5py.File(test_hdf_file, "w") test_nexus_file["/entry/data/data_0001"] = h5py.ExternalLink( - dummy_data_file.name, "data" + test_data_file.name, "data" ) test_nexus_file["/entry/data/data_0002"] = h5py.ExternalLink( - dummy_data_file.name, "data" + test_data_file.name, "data" ) yield test_nexus_file From 9207ce6f54c5a39016e9abca4cd31262f04c101a Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Jul 2026 11:35:14 +0100 Subject: [PATCH 07/21] Maybe this --- tests/tools/conftest.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index 4099b1db..1f53c55f 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -1,3 +1,4 @@ +import os import tempfile import h5py @@ -15,23 +16,22 @@ def nexus_file_with_single_dataset(): @pytest.fixture(scope="session") def dummy_data_file(): - test_data_file = tempfile.NamedTemporaryFile(suffix=".h5", delete=True) - with h5py.File(test_data_file, "w") as fh: + test_data_file = tempfile.NamedTemporaryFile(suffix=".h5", delete=False) + with h5py.File(test_data_file.name, "w") as fh: fh["data"] = np.zeros((10, 2, 3)) - yield test_data_file + yield test_data_file.name + + os.remove(test_data_file.name) @pytest.fixture -def nexus_file_with_multiple_datasets(): - test_data_file = tempfile.NamedTemporaryFile(suffix=".h5", delete=True) - with h5py.File(test_data_file, "w") as fh: - fh["data"] = np.zeros((10, 2, 3)) +def nexus_file_with_multiple_datasets(dummy_data_file): test_hdf_file = tempfile.TemporaryFile() test_nexus_file = h5py.File(test_hdf_file, "w") test_nexus_file["/entry/data/data_0001"] = h5py.ExternalLink( - test_data_file.name, "data" + dummy_data_file, "data" ) test_nexus_file["/entry/data/data_0002"] = h5py.ExternalLink( - test_data_file.name, "data" + dummy_data_file, "data" ) yield test_nexus_file From 0a5ab612fb3dc9011839418fe5ff62de8db4b16b Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Jul 2026 11:38:25 +0100 Subject: [PATCH 08/21] Maybe this once more --- tests/tools/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index 1f53c55f..64388eaa 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -17,8 +17,10 @@ def nexus_file_with_single_dataset(): @pytest.fixture(scope="session") def dummy_data_file(): test_data_file = tempfile.NamedTemporaryFile(suffix=".h5", delete=False) + test_data_file.close() with h5py.File(test_data_file.name, "w") as fh: fh["data"] = np.zeros((10, 2, 3)) + fh.flush() yield test_data_file.name os.remove(test_data_file.name) From e9f3ac104cd2d1b2a1b89eb70c6f4c48cb604414 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Jul 2026 12:57:20 +0100 Subject: [PATCH 09/21] Mock parts of the test --- tests/tools/conftest.py | 5 ++--- tests/tools/vds/test_strided_vds.py | 29 ++++++++++++++++++----------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index 64388eaa..460a2841 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -1,4 +1,3 @@ -import os import tempfile import h5py @@ -16,14 +15,14 @@ def nexus_file_with_single_dataset(): @pytest.fixture(scope="session") def dummy_data_file(): - test_data_file = tempfile.NamedTemporaryFile(suffix=".h5", delete=False) + test_data_file = tempfile.NamedTemporaryFile(suffix=".h5", delete=True) test_data_file.close() with h5py.File(test_data_file.name, "w") as fh: fh["data"] = np.zeros((10, 2, 3)) fh.flush() yield test_data_file.name - os.remove(test_data_file.name) + # os.remove(test_data_file.name) @pytest.fixture diff --git a/tests/tools/vds/test_strided_vds.py b/tests/tools/vds/test_strided_vds.py index d45d8a05..a8e1ae00 100644 --- a/tests/tools/vds/test_strided_vds.py +++ b/tests/tools/vds/test_strided_vds.py @@ -1,4 +1,4 @@ -from unittest.mock import patch +from unittest.mock import MagicMock, patch import numpy as np @@ -11,17 +11,20 @@ def test_create_dataset_list(nexus_file_with_multiple_datasets): - dsets = create_dataset_list( - nexus_file_with_multiple_datasets["/entry/data"], start_index=0 - ) + with patch("nexgen.tools.vds_tools.strided_mapping.h5py.Dataset") as patch_dset: + patch_dset.return_value.__enter__.return_value = MagicMock() + patch_dset.return_value.__enter__.return_value.shape = (10, 2, 3) + dsets = create_dataset_list( + nexus_file_with_multiple_datasets["/entry/data"], start_index=0 + ) - assert len(dsets) == 2 - assert dsets[0].name == "data_0001" - assert dsets[1].name == "data_0002" + assert len(dsets) == 2 + assert dsets[0].name == "data_0001" + assert dsets[1].name == "data_0002" - for dset in dsets: - assert dset.start_index == 0 - assert dset.stride == 2 + for dset in dsets: + assert dset.start_index == 0 + assert dset.stride == 2 def test_create_vds_layout(): @@ -36,12 +39,16 @@ def test_create_vds_layout(): assert list(layout._src_filenames)[0] == b"." +@patch("nexgen.tools.vds_tools.strided_mapping.create_dataset_list") @patch("nexgen.tools.vds_tools.strided_mapping.create_vds_layout") -def test_write_strided_vds(mock_layout, nexus_file_with_multiple_datasets): +def test_write_strided_vds( + mock_dset_list, mock_layout, nexus_file_with_multiple_datasets +): with patch( "nexgen.tools.vds_tools.strided_mapping.h5py.Group.create_virtual_dataset" ) as mock_create: write_strided_vds(nexus_file_with_multiple_datasets, (10, 2, 3), 0) + mock_dset_list.assert_called_once() mock_layout.assert_called_once() mock_create.assert_called_once() From 04301ff69dbf86b328fb8d44f49b525965a2bb47 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Jul 2026 13:23:57 +0100 Subject: [PATCH 10/21] Try not in write mode --- tests/tools/conftest.py | 5 +---- tests/tools/vds/test_strided_vds.py | 7 +++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index 460a2841..87dbe0b2 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -19,16 +19,13 @@ def dummy_data_file(): test_data_file.close() with h5py.File(test_data_file.name, "w") as fh: fh["data"] = np.zeros((10, 2, 3)) - fh.flush() yield test_data_file.name - # os.remove(test_data_file.name) - @pytest.fixture def nexus_file_with_multiple_datasets(dummy_data_file): test_hdf_file = tempfile.TemporaryFile() - test_nexus_file = h5py.File(test_hdf_file, "w") + test_nexus_file = h5py.File(test_hdf_file, "r+") test_nexus_file["/entry/data/data_0001"] = h5py.ExternalLink( dummy_data_file, "data" ) diff --git a/tests/tools/vds/test_strided_vds.py b/tests/tools/vds/test_strided_vds.py index a8e1ae00..098c43c5 100644 --- a/tests/tools/vds/test_strided_vds.py +++ b/tests/tools/vds/test_strided_vds.py @@ -11,6 +11,7 @@ def test_create_dataset_list(nexus_file_with_multiple_datasets): + # with tempfile.NamedTemporaryFile(suffix=".nxs", delete=True) with patch("nexgen.tools.vds_tools.strided_mapping.h5py.Dataset") as patch_dset: patch_dset.return_value.__enter__.return_value = MagicMock() patch_dset.return_value.__enter__.return_value.shape = (10, 2, 3) @@ -41,13 +42,11 @@ def test_create_vds_layout(): @patch("nexgen.tools.vds_tools.strided_mapping.create_dataset_list") @patch("nexgen.tools.vds_tools.strided_mapping.create_vds_layout") -def test_write_strided_vds( - mock_dset_list, mock_layout, nexus_file_with_multiple_datasets -): +def test_write_strided_vds(mock_dset_list, mock_layout, nexus_file_with_single_dataset): with patch( "nexgen.tools.vds_tools.strided_mapping.h5py.Group.create_virtual_dataset" ) as mock_create: - write_strided_vds(nexus_file_with_multiple_datasets, (10, 2, 3), 0) + write_strided_vds(nexus_file_with_single_dataset, (10, 2, 3), 0) mock_dset_list.assert_called_once() mock_layout.assert_called_once() From d0a65196cc430f7d1add2a8548cb8d1f80363578 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Jul 2026 13:40:37 +0100 Subject: [PATCH 11/21] Yet another fixture --- tests/tools/conftest.py | 24 +++++++++++++---------- tests/tools/vds/test_strided_vds.py | 30 ++++++++++++++--------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index 87dbe0b2..6e2e15ed 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -23,13 +23,17 @@ def dummy_data_file(): @pytest.fixture -def nexus_file_with_multiple_datasets(dummy_data_file): - test_hdf_file = tempfile.TemporaryFile() - test_nexus_file = h5py.File(test_hdf_file, "r+") - test_nexus_file["/entry/data/data_0001"] = h5py.ExternalLink( - dummy_data_file, "data" - ) - test_nexus_file["/entry/data/data_0002"] = h5py.ExternalLink( - dummy_data_file, "data" - ) - yield test_nexus_file +def nexus_file_with_multiple_datasets(): # dummy_data_file): + with tempfile.NamedTemporaryFile(suffix=".h5", delete=True) as test_data_file: + with h5py.File(test_data_file.name, "w") as fh: + fh["data"] = np.zeros((5, 2, 3)) + fh.flush() + test_hdf_file = tempfile.TemporaryFile() + test_nexus_file = h5py.File(test_hdf_file, "r+") + test_nexus_file["/entry/data/data_0001"] = h5py.ExternalLink( + test_data_file.name, "data" + ) + test_nexus_file["/entry/data/data_0002"] = h5py.ExternalLink( + test_data_file.name, "data" + ) + yield test_nexus_file diff --git a/tests/tools/vds/test_strided_vds.py b/tests/tools/vds/test_strided_vds.py index 098c43c5..6d80c1dc 100644 --- a/tests/tools/vds/test_strided_vds.py +++ b/tests/tools/vds/test_strided_vds.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import patch import numpy as np @@ -12,20 +12,20 @@ def test_create_dataset_list(nexus_file_with_multiple_datasets): # with tempfile.NamedTemporaryFile(suffix=".nxs", delete=True) - with patch("nexgen.tools.vds_tools.strided_mapping.h5py.Dataset") as patch_dset: - patch_dset.return_value.__enter__.return_value = MagicMock() - patch_dset.return_value.__enter__.return_value.shape = (10, 2, 3) - dsets = create_dataset_list( - nexus_file_with_multiple_datasets["/entry/data"], start_index=0 - ) - - assert len(dsets) == 2 - assert dsets[0].name == "data_0001" - assert dsets[1].name == "data_0002" - - for dset in dsets: - assert dset.start_index == 0 - assert dset.stride == 2 + # with patch("nexgen.tools.vds_tools.strided_mapping.h5py.Dataset") as patch_dset: + # patch_dset.return_value.__enter__.return_value = MagicMock() + # patch_dset.return_value.__enter__.return_value.shape = (5, 2, 3) + dsets = create_dataset_list( + nexus_file_with_multiple_datasets["/entry/data"], start_index=0 + ) + + assert len(dsets) == 2 + assert dsets[0].name == "data_0001" + assert dsets[1].name == "data_0002" + + for dset in dsets: + assert dset.start_index == 0 + assert dset.stride == 2 def test_create_vds_layout(): From 04b382016eef55be0155ef450b51e7a8aa898015 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Jul 2026 13:45:29 +0100 Subject: [PATCH 12/21] Just another test to check the file makes sense --- tests/tools/conftest.py | 4 +++- tests/tools/vds/test_utils.py | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index 6e2e15ed..8f39c436 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -26,10 +26,12 @@ def dummy_data_file(): def nexus_file_with_multiple_datasets(): # dummy_data_file): with tempfile.NamedTemporaryFile(suffix=".h5", delete=True) as test_data_file: with h5py.File(test_data_file.name, "w") as fh: - fh["data"] = np.zeros((5, 2, 3)) + fh.create_dataset("data", data=np.zeros((5, 2, 3))) + # fh["data"] = np.zeros((5, 2, 3)) fh.flush() test_hdf_file = tempfile.TemporaryFile() test_nexus_file = h5py.File(test_hdf_file, "r+") + test_nexus_file.require_group("/entry/data") test_nexus_file["/entry/data/data_0001"] = h5py.ExternalLink( test_data_file.name, "data" ) diff --git a/tests/tools/vds/test_utils.py b/tests/tools/vds/test_utils.py index 7bc07c6f..77c41a60 100644 --- a/tests/tools/vds/test_utils.py +++ b/tests/tools/vds/test_utils.py @@ -6,13 +6,20 @@ from nexgen.tools.vds_tools.utils import find_datasets_in_file -def test_find_datasets_int_file(nexus_file_with_single_dataset): +def test_find_datasets_in_file(nexus_file_with_single_dataset): nxdata = nexus_file_with_single_dataset["/entry/data"] dsets = find_datasets_in_file(nxdata) assert len(dsets) == 1 assert dsets[0] == "data_0001" +def test_find_multiple_datasets_in_file(nexus_file_with_multiple_datasets): + nxdata = nexus_file_with_multiple_datasets["/entry/data"] + dsets = find_datasets_in_file(nxdata) + assert len(dsets) == 2 + assert dsets[0] == "data_0001" and dsets[1] == "data_0002" + + def test_find_datasets_fails_if_no_links_in_file(): test_hdf_file = tempfile.TemporaryFile() test_nexus_file = h5py.File(test_hdf_file, "w") From 98e95e0831847d45e40a8dd6540dfaf6c732faad Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Jul 2026 13:50:50 +0100 Subject: [PATCH 13/21] I don't even know --- tests/tools/vds/test_strided_vds.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/tools/vds/test_strided_vds.py b/tests/tools/vds/test_strided_vds.py index 6d80c1dc..a04a7eea 100644 --- a/tests/tools/vds/test_strided_vds.py +++ b/tests/tools/vds/test_strided_vds.py @@ -15,9 +15,9 @@ def test_create_dataset_list(nexus_file_with_multiple_datasets): # with patch("nexgen.tools.vds_tools.strided_mapping.h5py.Dataset") as patch_dset: # patch_dset.return_value.__enter__.return_value = MagicMock() # patch_dset.return_value.__enter__.return_value.shape = (5, 2, 3) - dsets = create_dataset_list( - nexus_file_with_multiple_datasets["/entry/data"], start_index=0 - ) + nxdata = nexus_file_with_multiple_datasets["/entry/data"] + assert nxdata["data_0001"] + dsets = create_dataset_list(nxdata, start_index=0) assert len(dsets) == 2 assert dsets[0].name == "data_0001" From 054af028fc48fa90796f1d7c20f344fe15ba7040 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Jul 2026 13:59:51 +0100 Subject: [PATCH 14/21] try using tempdir --- tests/tools/conftest.py | 37 +++++++++++++++++++++++------ tests/tools/vds/test_strided_vds.py | 10 +++++--- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index 8f39c436..fe60c07d 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -1,4 +1,5 @@ import tempfile +from pathlib import Path import h5py import numpy as np @@ -22,20 +23,42 @@ def dummy_data_file(): yield test_data_file.name +# @pytest.fixture +# def nexus_file_with_multiple_datasets(): # dummy_data_file): +# # with tempfile.NamedTemporaryFile(suffix=".h5", delete=True) as test_data_file: +# # with h5py.File(test_data_file.name, "w") as fh: +# # fh.create_dataset("data", data=np.zeros((5, 2, 3))) +# # # fh["data"] = np.zeros((5, 2, 3)) +# # fh.flush() +# test_hdf_file = tempfile.TemporaryFile() +# test_nexus_file = h5py.File(test_hdf_file, "r+") +# test_nexus_file.require_group("/entry/data") +# test_nexus_file["/entry/data/data_0001"] = h5py.ExternalLink( +# "filename1", "data" +# # test_data_file.name, "data" +# ) +# test_nexus_file["/entry/data/data_0002"] = h5py.ExternalLink( +# "filename2", "data" +# # test_data_file.name, "data" +# ) +# yield test_nexus_file + + @pytest.fixture -def nexus_file_with_multiple_datasets(): # dummy_data_file): - with tempfile.NamedTemporaryFile(suffix=".h5", delete=True) as test_data_file: - with h5py.File(test_data_file.name, "w") as fh: +def nexus_file_with_multiple_datasets(): + with tempfile.TemporaryDirectory() as tmpdir: + data_path = Path(tmpdir) / "data.h5" + with h5py.File(data_path, "w") as fh: fh.create_dataset("data", data=np.zeros((5, 2, 3))) - # fh["data"] = np.zeros((5, 2, 3)) - fh.flush() + test_hdf_file = tempfile.TemporaryFile() test_nexus_file = h5py.File(test_hdf_file, "r+") test_nexus_file.require_group("/entry/data") test_nexus_file["/entry/data/data_0001"] = h5py.ExternalLink( - test_data_file.name, "data" + str(data_path), "data" ) test_nexus_file["/entry/data/data_0002"] = h5py.ExternalLink( - test_data_file.name, "data" + str(data_path), "data" ) + yield test_nexus_file diff --git a/tests/tools/vds/test_strided_vds.py b/tests/tools/vds/test_strided_vds.py index a04a7eea..711b3048 100644 --- a/tests/tools/vds/test_strided_vds.py +++ b/tests/tools/vds/test_strided_vds.py @@ -10,14 +10,18 @@ ) +# @patch("nexgen.tools.vds_tools.strided_mapping.find_datasets_in_file") def test_create_dataset_list(nexus_file_with_multiple_datasets): + # mock_find.return_value = ["data_0001", "data_0002"] # with tempfile.NamedTemporaryFile(suffix=".nxs", delete=True) # with patch("nexgen.tools.vds_tools.strided_mapping.h5py.Dataset") as patch_dset: # patch_dset.return_value.__enter__.return_value = MagicMock() # patch_dset.return_value.__enter__.return_value.shape = (5, 2, 3) - nxdata = nexus_file_with_multiple_datasets["/entry/data"] - assert nxdata["data_0001"] - dsets = create_dataset_list(nxdata, start_index=0) + # nxdata = nexus_file_with_multiple_datasets["/entry/data"] + # assert nxdata["data_0001"] + dsets = create_dataset_list( + nexus_file_with_multiple_datasets["/entry/data"], start_index=0 + ) assert len(dsets) == 2 assert dsets[0].name == "data_0001" From ace898102f73c9b768043f14b503fff6398a3bef Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Jul 2026 14:15:35 +0100 Subject: [PATCH 15/21] Debug assert --- tests/tools/conftest.py | 1 + tests/tools/vds/test_strided_vds.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index fe60c07d..89bc3b6c 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -50,6 +50,7 @@ def nexus_file_with_multiple_datasets(): data_path = Path(tmpdir) / "data.h5" with h5py.File(data_path, "w") as fh: fh.create_dataset("data", data=np.zeros((5, 2, 3))) + fh.flush() test_hdf_file = tempfile.TemporaryFile() test_nexus_file = h5py.File(test_hdf_file, "r+") diff --git a/tests/tools/vds/test_strided_vds.py b/tests/tools/vds/test_strided_vds.py index 711b3048..1c362889 100644 --- a/tests/tools/vds/test_strided_vds.py +++ b/tests/tools/vds/test_strided_vds.py @@ -17,8 +17,8 @@ def test_create_dataset_list(nexus_file_with_multiple_datasets): # with patch("nexgen.tools.vds_tools.strided_mapping.h5py.Dataset") as patch_dset: # patch_dset.return_value.__enter__.return_value = MagicMock() # patch_dset.return_value.__enter__.return_value.shape = (5, 2, 3) - # nxdata = nexus_file_with_multiple_datasets["/entry/data"] - # assert nxdata["data_0001"] + nxdata = nexus_file_with_multiple_datasets["/entry/data"] + assert nxdata["data_0001"] dsets = create_dataset_list( nexus_file_with_multiple_datasets["/entry/data"], start_index=0 ) From 024fa632aea8b431de9d0f5b9b10162cca7bfd97 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Jul 2026 14:21:47 +0100 Subject: [PATCH 16/21] Other attempt at debug --- tests/tools/conftest.py | 30 ----------------------------- tests/tools/vds/test_strided_vds.py | 17 +++++++++------- 2 files changed, 10 insertions(+), 37 deletions(-) diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index 89bc3b6c..65c94ae2 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -14,36 +14,6 @@ def nexus_file_with_single_dataset(): yield test_nexus_file -@pytest.fixture(scope="session") -def dummy_data_file(): - test_data_file = tempfile.NamedTemporaryFile(suffix=".h5", delete=True) - test_data_file.close() - with h5py.File(test_data_file.name, "w") as fh: - fh["data"] = np.zeros((10, 2, 3)) - yield test_data_file.name - - -# @pytest.fixture -# def nexus_file_with_multiple_datasets(): # dummy_data_file): -# # with tempfile.NamedTemporaryFile(suffix=".h5", delete=True) as test_data_file: -# # with h5py.File(test_data_file.name, "w") as fh: -# # fh.create_dataset("data", data=np.zeros((5, 2, 3))) -# # # fh["data"] = np.zeros((5, 2, 3)) -# # fh.flush() -# test_hdf_file = tempfile.TemporaryFile() -# test_nexus_file = h5py.File(test_hdf_file, "r+") -# test_nexus_file.require_group("/entry/data") -# test_nexus_file["/entry/data/data_0001"] = h5py.ExternalLink( -# "filename1", "data" -# # test_data_file.name, "data" -# ) -# test_nexus_file["/entry/data/data_0002"] = h5py.ExternalLink( -# "filename2", "data" -# # test_data_file.name, "data" -# ) -# yield test_nexus_file - - @pytest.fixture def nexus_file_with_multiple_datasets(): with tempfile.TemporaryDirectory() as tmpdir: diff --git a/tests/tools/vds/test_strided_vds.py b/tests/tools/vds/test_strided_vds.py index 1c362889..0a82df7e 100644 --- a/tests/tools/vds/test_strided_vds.py +++ b/tests/tools/vds/test_strided_vds.py @@ -1,6 +1,7 @@ from unittest.mock import patch import numpy as np +import pytest from nexgen.tools.vds_tools.strided_mapping import ( SingleDataset, @@ -10,15 +11,17 @@ ) -# @patch("nexgen.tools.vds_tools.strided_mapping.find_datasets_in_file") +@patch("nexgen.tools.vds_tools.strided_mapping.find_datasets_in_file") +def test_create_dataset_list_fails_if_no_external_links_found(mock_find): + mock_find.return_value = [] + with pytest.raises(ValueError): + create_dataset_list("", 0) + + def test_create_dataset_list(nexus_file_with_multiple_datasets): - # mock_find.return_value = ["data_0001", "data_0002"] - # with tempfile.NamedTemporaryFile(suffix=".nxs", delete=True) - # with patch("nexgen.tools.vds_tools.strided_mapping.h5py.Dataset") as patch_dset: - # patch_dset.return_value.__enter__.return_value = MagicMock() - # patch_dset.return_value.__enter__.return_value.shape = (5, 2, 3) nxdata = nexus_file_with_multiple_datasets["/entry/data"] - assert nxdata["data_0001"] + nxdata.create_dataset("boh", data=(0, 0, 0)) + assert nxdata["boh"] dsets = create_dataset_list( nexus_file_with_multiple_datasets["/entry/data"], start_index=0 ) From 95aaa4695ef41c631d74ca533256266086cea5e0 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Jul 2026 14:30:43 +0100 Subject: [PATCH 17/21] Play with delete --- tests/tools/conftest.py | 36 +++++++++++++++++++++++------ tests/tools/vds/test_strided_vds.py | 6 +---- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index 65c94ae2..d31b6239 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -1,5 +1,4 @@ import tempfile -from pathlib import Path import h5py import numpy as np @@ -14,11 +13,32 @@ def nexus_file_with_single_dataset(): yield test_nexus_file -@pytest.fixture +# @pytest.fixture +# def nexus_file_with_multiple_datasets(): +# with tempfile.TemporaryDirectory() as tmpdir: +# data_path = Path(tmpdir) / "data.h5" +# with h5py.File(data_path, "w") as fh: +# fh.create_dataset("data", data=np.zeros((5, 2, 3))) +# fh.flush() + +# test_hdf_file = tempfile.TemporaryFile() +# test_nexus_file = h5py.File(test_hdf_file, "r+") +# test_nexus_file.require_group("/entry/data") +# test_nexus_file["/entry/data/data_0001"] = h5py.ExternalLink( +# str(data_path), "data" +# ) +# test_nexus_file["/entry/data/data_0002"] = h5py.ExternalLink( +# str(data_path), "data" +# ) + +# yield test_nexus_file + + +@pytest.fixture(scope="session") def nexus_file_with_multiple_datasets(): - with tempfile.TemporaryDirectory() as tmpdir: - data_path = Path(tmpdir) / "data.h5" - with h5py.File(data_path, "w") as fh: + with tempfile.NamedTemporaryFile(suffix=".h5", delete=False) as test_data_file: + # data_path = Path(tmpdir) / "data.h5" + with h5py.File(test_data_file.name, "w") as fh: fh.create_dataset("data", data=np.zeros((5, 2, 3))) fh.flush() @@ -26,10 +46,12 @@ def nexus_file_with_multiple_datasets(): test_nexus_file = h5py.File(test_hdf_file, "r+") test_nexus_file.require_group("/entry/data") test_nexus_file["/entry/data/data_0001"] = h5py.ExternalLink( - str(data_path), "data" + test_data_file.name, "data" ) test_nexus_file["/entry/data/data_0002"] = h5py.ExternalLink( - str(data_path), "data" + test_data_file.name, "data" ) yield test_nexus_file + + test_data_file.close() diff --git a/tests/tools/vds/test_strided_vds.py b/tests/tools/vds/test_strided_vds.py index 0a82df7e..ac759dda 100644 --- a/tests/tools/vds/test_strided_vds.py +++ b/tests/tools/vds/test_strided_vds.py @@ -20,11 +20,7 @@ def test_create_dataset_list_fails_if_no_external_links_found(mock_find): def test_create_dataset_list(nexus_file_with_multiple_datasets): nxdata = nexus_file_with_multiple_datasets["/entry/data"] - nxdata.create_dataset("boh", data=(0, 0, 0)) - assert nxdata["boh"] - dsets = create_dataset_list( - nexus_file_with_multiple_datasets["/entry/data"], start_index=0 - ) + dsets = create_dataset_list(nxdata, start_index=0) assert len(dsets) == 2 assert dsets[0].name == "data_0001" From c635a9500e6d3fc264395867b4f7a8c7bd83252e Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Jul 2026 14:32:14 +0100 Subject: [PATCH 18/21] What if I don't close it at all --- tests/tools/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index d31b6239..3c2864fb 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -54,4 +54,4 @@ def nexus_file_with_multiple_datasets(): yield test_nexus_file - test_data_file.close() + # test_data_file.close() From 432be4f500a5aa25021a24841ab6e2e3daeb4639 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Jul 2026 14:33:42 +0100 Subject: [PATCH 19/21] Fix typo --- tests/tools/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index 3c2864fb..5fb6ca35 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -38,7 +38,7 @@ def nexus_file_with_single_dataset(): def nexus_file_with_multiple_datasets(): with tempfile.NamedTemporaryFile(suffix=".h5", delete=False) as test_data_file: # data_path = Path(tmpdir) / "data.h5" - with h5py.File(test_data_file.name, "w") as fh: + with h5py.File(test_data_file, "w") as fh: fh.create_dataset("data", data=np.zeros((5, 2, 3))) fh.flush() From 6f0a61ab06c953eeac94f19ba361a841b7944840 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Jul 2026 14:37:39 +0100 Subject: [PATCH 20/21] Just tidy up --- tests/tools/conftest.py | 46 ++++------------------------- tests/tools/vds/test_strided_vds.py | 13 -------- 2 files changed, 6 insertions(+), 53 deletions(-) diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index 5fb6ca35..8daa1458 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -1,7 +1,6 @@ import tempfile import h5py -import numpy as np import pytest @@ -13,45 +12,12 @@ def nexus_file_with_single_dataset(): yield test_nexus_file -# @pytest.fixture -# def nexus_file_with_multiple_datasets(): -# with tempfile.TemporaryDirectory() as tmpdir: -# data_path = Path(tmpdir) / "data.h5" -# with h5py.File(data_path, "w") as fh: -# fh.create_dataset("data", data=np.zeros((5, 2, 3))) -# fh.flush() - -# test_hdf_file = tempfile.TemporaryFile() -# test_nexus_file = h5py.File(test_hdf_file, "r+") -# test_nexus_file.require_group("/entry/data") -# test_nexus_file["/entry/data/data_0001"] = h5py.ExternalLink( -# str(data_path), "data" -# ) -# test_nexus_file["/entry/data/data_0002"] = h5py.ExternalLink( -# str(data_path), "data" -# ) - -# yield test_nexus_file - - @pytest.fixture(scope="session") def nexus_file_with_multiple_datasets(): - with tempfile.NamedTemporaryFile(suffix=".h5", delete=False) as test_data_file: - # data_path = Path(tmpdir) / "data.h5" - with h5py.File(test_data_file, "w") as fh: - fh.create_dataset("data", data=np.zeros((5, 2, 3))) - fh.flush() - - test_hdf_file = tempfile.TemporaryFile() - test_nexus_file = h5py.File(test_hdf_file, "r+") - test_nexus_file.require_group("/entry/data") - test_nexus_file["/entry/data/data_0001"] = h5py.ExternalLink( - test_data_file.name, "data" - ) - test_nexus_file["/entry/data/data_0002"] = h5py.ExternalLink( - test_data_file.name, "data" - ) - - yield test_nexus_file + test_hdf_file = tempfile.TemporaryFile() + test_nexus_file = h5py.File(test_hdf_file, "r+") + test_nexus_file.require_group("/entry/data") + test_nexus_file["/entry/data/data_0001"] = h5py.ExternalLink("file_1", "data") + test_nexus_file["/entry/data/data_0002"] = h5py.ExternalLink("file_2", "data") - # test_data_file.close() + yield test_nexus_file diff --git a/tests/tools/vds/test_strided_vds.py b/tests/tools/vds/test_strided_vds.py index ac759dda..dfaa0d13 100644 --- a/tests/tools/vds/test_strided_vds.py +++ b/tests/tools/vds/test_strided_vds.py @@ -18,19 +18,6 @@ def test_create_dataset_list_fails_if_no_external_links_found(mock_find): create_dataset_list("", 0) -def test_create_dataset_list(nexus_file_with_multiple_datasets): - nxdata = nexus_file_with_multiple_datasets["/entry/data"] - dsets = create_dataset_list(nxdata, start_index=0) - - assert len(dsets) == 2 - assert dsets[0].name == "data_0001" - assert dsets[1].name == "data_0002" - - for dset in dsets: - assert dset.start_index == 0 - assert dset.stride == 2 - - def test_create_vds_layout(): dset_list = [ SingleDataset(name="data_01", src_shape=(6, 2, 2), start_index=1, stride=2) From e695670a0ad0e7aca84f51af55e65e1c83c66e12 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Jul 2026 14:45:03 +0100 Subject: [PATCH 21/21] Add a couple of docstrings --- src/nexgen/beamlines/i19_2/eiger.py | 3 +-- src/nexgen/beamlines/i19_2/serial.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/nexgen/beamlines/i19_2/eiger.py b/src/nexgen/beamlines/i19_2/eiger.py index 52f8490c..69e43948 100644 --- a/src/nexgen/beamlines/i19_2/eiger.py +++ b/src/nexgen/beamlines/i19_2/eiger.py @@ -177,8 +177,7 @@ def eiger_writer( "Not using meta file to update metadata, only the external links will be set up." ) vds_dtype = define_vds_dtype_from_bit_depth(eiger_settings.bit_depth) - # wavelength = parameters.wavelength - # beam_center = parameters.beam_center + # Update axes # Goniometer for gax in parameters.axes_pos: diff --git a/src/nexgen/beamlines/i19_2/serial.py b/src/nexgen/beamlines/i19_2/serial.py index 14d0a29c..39500cb1 100644 --- a/src/nexgen/beamlines/i19_2/serial.py +++ b/src/nexgen/beamlines/i19_2/serial.py @@ -36,6 +36,28 @@ def serial_nexus_writer( vds_mapping: VdsMapping = VdsMapping.BLOCKED, notes: dict[str, Any] | None = None, ): + """Wrapper function to gather all parameters from the beamline and kick off the nexus writer for a + serial experiment on I19-2. + + Args: + params (dict[str, Any]): Dictionary representation of CollectionParams. + master_file (Path): Full path to the nexus file to be written. + use_meta (bool, optional): Eiger option only, if True use metadata from meta.h5 file. Otherwise + all parameters will need to be passed manually. Defaults to False. + vds_offset (int, optional): Start index for the vds writer. Defaults to 0. + n_frames (int | None, optional): Number of images for the nexus file. Only needed if different + from the tot_num_images in the collection params. If passed, the VDS will only contain the + number of frames specified here. Defaults to None. + bit_depth(int, optional): Default bit depth for eiger collections, used to define dtype of vds data. \ + Defaults to 32. + data_entry_key (str, optional): Where to find the dataset. Defaults to "data". + eiger_stream_format (EigerStreamFormat, optional): Stream format setting on the new fastcs eiger. + The metafile in the new cbor format is slightly different. Defaults to "legacy". + vds_mapping (VdsMapping, optional): How to map the frames when building the VDS. + notes (dict[str, Any] | None, optional): Any additional information to be written as NXnote, + passed as a dictionary of (key, value) pairs where key represents the dataset name and + value its data. Defaults to None. + """ _setup_logging(master_file.parent) collection_params = CollectionParams(**params)