From 1ac0cb8f6a9367c3b68dc48dac78536536e9af56 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Mon, 6 Jul 2026 17:21:50 +0100 Subject: [PATCH 1/7] Start moving vds utilities --- src/nexgen/beamlines/I19_2_nxs.py | 2 +- src/nexgen/beamlines/SSX_Eiger_nxs.py | 2 +- src/nexgen/beamlines/i19_2/eiger.py | 2 +- src/nexgen/command_line/nexus_generator.py | 2 +- src/nexgen/tools/vds_tools/__init__.py | 14 ++++++++++++-- src/nexgen/tools/vds_tools/utils.py | 15 +++++++++++++-- src/nexgen/tools/vds_w_tools.py | 10 ---------- tests/tools/test_VDS_tools.py | 10 ---------- tests/tools/vds/test_utils.py | 15 ++++++++++++++- 9 files changed, 43 insertions(+), 29 deletions(-) diff --git a/src/nexgen/beamlines/I19_2_nxs.py b/src/nexgen/beamlines/I19_2_nxs.py index 21868501..1c3be01b 100644 --- a/src/nexgen/beamlines/I19_2_nxs.py +++ b/src/nexgen/beamlines/I19_2_nxs.py @@ -15,7 +15,7 @@ from pydantic import field_validator from nexgen.nxs_utils.detector import EigerStreamFormat -from nexgen.tools.vds_w_tools import define_vds_dtype_from_bit_depth +from nexgen.tools.vds_tools import define_vds_dtype_from_bit_depth from nexgen.utils import get_iso_timestamp from .. import log diff --git a/src/nexgen/beamlines/SSX_Eiger_nxs.py b/src/nexgen/beamlines/SSX_Eiger_nxs.py index 745a8d92..0fd3d477 100644 --- a/src/nexgen/beamlines/SSX_Eiger_nxs.py +++ b/src/nexgen/beamlines/SSX_Eiger_nxs.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import Literal, get_args -from nexgen.tools.vds_w_tools import define_vds_dtype_from_bit_depth +from nexgen.tools.vds_tools import define_vds_dtype_from_bit_depth from .. import log from ..nxs_utils import ( diff --git a/src/nexgen/beamlines/i19_2/eiger.py b/src/nexgen/beamlines/i19_2/eiger.py index f13a9d20..fe9f523b 100644 --- a/src/nexgen/beamlines/i19_2/eiger.py +++ b/src/nexgen/beamlines/i19_2/eiger.py @@ -22,8 +22,8 @@ from nexgen.tools.vds_tools import ( VdsMapping, VdsSettings, + define_vds_dtype_from_bit_depth, ) -from nexgen.tools.vds_w_tools import define_vds_dtype_from_bit_depth logger = logging.getLogger("nexgen.beamlines.I19_2.eiger") diff --git a/src/nexgen/command_line/nexus_generator.py b/src/nexgen/command_line/nexus_generator.py index 76b167f3..41bc725f 100644 --- a/src/nexgen/command_line/nexus_generator.py +++ b/src/nexgen/command_line/nexus_generator.py @@ -28,7 +28,7 @@ from nexgen.nxs_write.nxmx_writer import EventNXmxFileWriter, NXmxFileWriter from nexgen.nxs_write.write_utils import find_number_of_images from nexgen.tools.data_writer import generate_event_files, generate_image_files -from nexgen.tools.vds_w_tools import define_vds_dtype_from_bit_depth +from nexgen.tools.vds_tools import define_vds_dtype_from_bit_depth from nexgen.utils import ( get_filename_template, get_iso_timestamp, diff --git a/src/nexgen/tools/vds_tools/__init__.py b/src/nexgen/tools/vds_tools/__init__.py index 9a0fd0fa..0bf30c65 100644 --- a/src/nexgen/tools/vds_tools/__init__.py +++ b/src/nexgen/tools/vds_tools/__init__.py @@ -1,3 +1,13 @@ -from nexgen.tools.vds_tools.utils import VdsMapping, VdsSettings, find_datasets_in_file +from nexgen.tools.vds_tools.utils import ( + VdsMapping, + VdsSettings, + define_vds_dtype_from_bit_depth, + find_datasets_in_file, +) -__all__ = ["VdsMapping", "VdsSettings", "find_datasets_in_file"] +__all__ = [ + "VdsMapping", + "VdsSettings", + "find_datasets_in_file", + "define_vds_dtype_from_bit_depth", +] diff --git a/src/nexgen/tools/vds_tools/utils.py b/src/nexgen/tools/vds_tools/utils.py index dbff00de..9dc9df76 100644 --- a/src/nexgen/tools/vds_tools/utils.py +++ b/src/nexgen/tools/vds_tools/utils.py @@ -3,6 +3,7 @@ import h5py import numpy as np +from numpy.typing import DTypeLike from pydantic import BaseModel, ConfigDict PydanticDTypeLike: TypeAlias = np.dtype[Any] | type | str | None @@ -25,7 +26,8 @@ class VdsSettings(BaseModel): def find_datasets_in_file(nxdata: h5py.Group) -> list: """ - Look for the source datasets in the NeXus file. Assumes that the source datasets are always h5py.ExternalLink. + Look for the source datasets in the NeXus file. + Assumes that the source datasets are always h5py.ExternalLink. Args: nxdata (h5py.Group): Group where the data should be linked. @@ -36,7 +38,6 @@ def find_datasets_in_file(nxdata: h5py.Group) -> list: Returns: dsets (list): The source datasets. """ - # FIXME for now this assumes that the source datasets are always links dsets = [] for k in nxdata.keys(): if isinstance(nxdata.get(k, getlink=True), h5py.ExternalLink): @@ -46,3 +47,13 @@ def find_datasets_in_file(nxdata: h5py.Group) -> list: f"No External Link datasets found in NeXus file under {nxdata.name}" ) return dsets + + +def define_vds_dtype_from_bit_depth(bit_depth: int) -> DTypeLike: + """Define dtype of VDS based on the passed bit depth.""" + if bit_depth == 32: + return np.uint32 + elif bit_depth == 8: + return np.uint8 + else: + return np.uint16 diff --git a/src/nexgen/tools/vds_w_tools.py b/src/nexgen/tools/vds_w_tools.py index ae432212..1a0b1af4 100644 --- a/src/nexgen/tools/vds_w_tools.py +++ b/src/nexgen/tools/vds_w_tools.py @@ -23,16 +23,6 @@ vds_logger = logging.getLogger("nexgen.VDSWriter") -def define_vds_dtype_from_bit_depth(bit_depth: int) -> DTypeLike: - """Define dtype of VDS based on the passed bit depth.""" - if bit_depth == 32: - return np.uint32 - elif bit_depth == 8: - return np.uint8 - else: - return np.uint16 - - @dataclass class Dataset: name: str diff --git a/tests/tools/test_VDS_tools.py b/tests/tools/test_VDS_tools.py index 334933e3..981e50cf 100644 --- a/tests/tools/test_VDS_tools.py +++ b/tests/tools/test_VDS_tools.py @@ -8,22 +8,12 @@ from nexgen.tools.vds_w_tools import ( Dataset, create_virtual_layout, - define_vds_dtype_from_bit_depth, image_vds_writer, jungfrau_vds_writer, split_datasets, ) -@pytest.mark.parametrize( - "bit_depth, expected_dtype", [(8, np.uint8), (16, np.uint16), (32, np.uint32)] -) -def test_vds_dtype_from_input(bit_depth, expected_dtype): - d = define_vds_dtype_from_bit_depth(bit_depth) - - assert d == expected_dtype - - def test_when_get_frames_and_shape_less_than_1000_then_correct(): sshape = split_datasets(["test1"], (500, 10, 10)) assert sshape == [Dataset("test1", (500, 10, 10), 0, 500)] diff --git a/tests/tools/vds/test_utils.py b/tests/tools/vds/test_utils.py index 77c41a60..28c38f60 100644 --- a/tests/tools/vds/test_utils.py +++ b/tests/tools/vds/test_utils.py @@ -1,9 +1,13 @@ import tempfile import h5py +import numpy as np import pytest -from nexgen.tools.vds_tools.utils import find_datasets_in_file +from nexgen.tools.vds_tools.utils import ( + define_vds_dtype_from_bit_depth, + find_datasets_in_file, +) def test_find_datasets_in_file(nexus_file_with_single_dataset): @@ -26,3 +30,12 @@ def test_find_datasets_fails_if_no_links_in_file(): test_nexus_file["/entry/data/data_0001"] = [0, 0, 0] with pytest.raises(KeyError): find_datasets_in_file(test_nexus_file) + + +@pytest.mark.parametrize( + "bit_depth, expected_dtype", [(8, np.uint8), (16, np.uint16), (32, np.uint32)] +) +def test_vds_dtype_from_input(bit_depth, expected_dtype): + d = define_vds_dtype_from_bit_depth(bit_depth) + + assert d == expected_dtype From 3f75805e026f78cbd6eb0cad60b272e780b0a78a Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Thu, 9 Jul 2026 11:59:35 +0100 Subject: [PATCH 2/7] Move utilities for blocked vds to new location --- src/nexgen/nxs_write/ed_nxmx_writer.py | 3 +- src/nexgen/nxs_write/nxmx_writer.py | 2 +- src/nexgen/tools/vds_tools/blocked_mapping.py | 216 ++++++++++++++++++ src/nexgen/tools/vds_tools/utils.py | 2 +- src/nexgen/tools/vds_w_tools.py | 205 ----------------- tests/tools/test_VDS_tools.py | 94 -------- tests/tools/vds/test_blocked_vds.py | 101 ++++++++ 7 files changed, 321 insertions(+), 302 deletions(-) create mode 100644 src/nexgen/tools/vds_tools/blocked_mapping.py create mode 100644 tests/tools/vds/test_blocked_vds.py diff --git a/src/nexgen/nxs_write/ed_nxmx_writer.py b/src/nexgen/nxs_write/ed_nxmx_writer.py index 58fe2e2f..4820e2c5 100644 --- a/src/nexgen/nxs_write/ed_nxmx_writer.py +++ b/src/nexgen/nxs_write/ed_nxmx_writer.py @@ -14,7 +14,8 @@ from numpy.typing import DTypeLike from ..nxs_utils import Attenuator, Beam, Detector, Goniometer, Source -from ..tools.vds_w_tools import image_vds_writer, vds_file_writer +from ..tools.vds_tools.blocked_mapping import image_vds_writer +from ..tools.vds_w_tools import vds_file_writer from ..utils import coord2mcstas from .nxclass_writers import ( write_NXcoordinate_system_set, diff --git a/src/nexgen/nxs_write/nxmx_writer.py b/src/nexgen/nxs_write/nxmx_writer.py index 7a22c2c0..1aa40bfd 100644 --- a/src/nexgen/nxs_write/nxmx_writer.py +++ b/src/nexgen/nxs_write/nxmx_writer.py @@ -17,9 +17,9 @@ from ..nxs_utils.goniometer import Goniometer from ..nxs_utils.sample import Sample from ..nxs_utils.source import Attenuator, Beam, Source +from ..tools.vds_tools.blocked_mapping import image_vds_writer from ..tools.vds_w_tools import ( clean_unused_links, - image_vds_writer, jungfrau_vds_writer, ) from ..utils import ( diff --git a/src/nexgen/tools/vds_tools/blocked_mapping.py b/src/nexgen/tools/vds_tools/blocked_mapping.py new file mode 100644 index 00000000..d22be08b --- /dev/null +++ b/src/nexgen/tools/vds_tools/blocked_mapping.py @@ -0,0 +1,216 @@ +import logging +import operator +from functools import reduce +from typing import Sequence + +import h5py +import numpy as np +from numpy.typing import DTypeLike +from pydantic.dataclasses import dataclass + +from nexgen.tools.vds_tools.utils import find_datasets_in_file +from nexgen.utils import MAX_FRAMES_PER_DATASET + +block_vds_logger = logging.getLogger("nexgen.tools.vds_tools.block_mapping") + + +@dataclass +class Dataset: + name: str + + # The full shape of the source, regardless of start index + source_shape: Sequence[int] + + # The start index that we should start copying from + start_index: int = 0 + + # The point where we should stop copying. Defaults to maximum for an Eiger + stop_index: int = MAX_FRAMES_PER_DATASET + + # The shape of the destination, including the start_index + dest_shape: Sequence[int] | None = None + + def __post_init__(self): + self.dest_shape = ( + self.source_shape[0] - self.start_index, + *self.source_shape[1:], + ) + if ( + self.stop_index < MAX_FRAMES_PER_DATASET + or self.stop_index < self.source_shape[0] + ): + self.dest_shape = ( + self.dest_shape[0] - (self.source_shape[0] - self.stop_index), + *self.source_shape[1:], + ) + + def __add__(self, x): + """Returns a dataset that has the same start index and shape as if the two were appended to each other.""" + return Dataset( + "", + source_shape=( + self.source_shape[0] + x.source_shape[0], + *self.source_shape[1:], + ), + start_index=self.start_index + x.start_index, + stop_index=self.stop_index + x.stop_index, + ) + + +def split_datasets( + dsets: list[str], + data_shape: tuple[int, int, int], + start_idx: int = 0, + vds_shape: tuple[int, int, int] = None, +) -> list[Dataset]: + """ + Splits the full data shape and start index up into values per dataset, + given that each dataset has a maximum size. + + Args: + dsets (Dataset): The input datasets. + data_shape (tuple[int, int, int]): Shape of the data, usually defined as (num_frames, *image_size). + start_idx (int, optional): The start point for the source data. Defaults to 0. + vds_shape(tuple, optional): Desired shape of the VDS, usually defined as (num_frames, *image_size). \ + The number of frames must be smaller or equal to the one in full_data_shape. Defaults to None. + + Raises: + ValueError: If the passed start index value is higher than the dataset lenght. + ValueError: It the passed start index value is negative. + + Returns: + list[Dataset]: A list of datasets. + """ + if start_idx > data_shape[0]: + raise ValueError( + f"Start index {start_idx} must be less than full dataset length {data_shape[0]}" + ) + if start_idx < 0: + raise ValueError("Start index must be positive") + + if not isinstance(data_shape[0], int): + block_vds_logger.warning("Datashape not passed as int, will attempt to cast") + + if not isinstance(start_idx, int): + block_vds_logger.warning( + "VDS start index not passed as int, will attempt to cast" + ) + + if vds_shape and not isinstance(vds_shape[0], int): + block_vds_logger.warning( + "VDS start index not passed as int, will attempt to cast" + ) + + if vds_shape is None: + block_vds_logger.debug( + "VDS shape not chosen, it will be calculated from the full data shape and the chosen start index." + ) + vds_shape = (data_shape[0] - start_idx, *data_shape[1:]) + + full_frames = int(data_shape[0]) + end_cut_frames = int(full_frames - vds_shape[0]) - int(start_idx) + + result = [] + for dset_name in dsets: + dset = Dataset( + name=dset_name, + source_shape=(min(MAX_FRAMES_PER_DATASET, full_frames), *data_shape[1:]), + start_index=min(MAX_FRAMES_PER_DATASET, max(int(start_idx), 0)), + stop_index=min(MAX_FRAMES_PER_DATASET, (full_frames - end_cut_frames)), + ) + # if start index == 1000 then that source dataset is not used and we should + # not pass it on to use as a source for the VDS + # Same goes if all datasets from last files are not used + if dset.start_index != MAX_FRAMES_PER_DATASET: + if dset.stop_index > 0 and dset.stop_index <= MAX_FRAMES_PER_DATASET: + result.append(dset) + start_idx -= MAX_FRAMES_PER_DATASET + full_frames -= MAX_FRAMES_PER_DATASET + + return result + + +def create_virtual_layout(datasets: list[Dataset], data_type: DTypeLike): + """ + Create a virtual layout and populate it based on the provided data. + + Args: + datasets (list[Dataset]): A list of datasets that are to be merged. + data_type (DTypeLike): The type of the input data. + + Returns: + layout (h5py.VirtualLayout): Virtual layout. + """ + full_dataset: Dataset = reduce(operator.add, datasets) + layout = h5py.VirtualLayout(shape=full_dataset.dest_shape, dtype=data_type) + + dest_start = 0 + for dataset in datasets: + if dataset.stop_index == dataset.source_shape[0]: + dest_end = dest_start + dataset.source_shape[0] - dataset.start_index + else: + dest_end = dest_start + dataset.dest_shape[0] + + vsource = h5py.VirtualSource( + ".", "/entry/data/" + dataset.name, shape=dataset.source_shape + ) + + layout[dest_start:dest_end, :, :] = vsource[ + dataset.start_index : dataset.stop_index, :, : + ] + dest_start = dest_end + + return layout + + +def image_vds_writer( + nxsfile: h5py.File, + full_data_shape: tuple | list, + start_index: int = 0, + vds_shape: tuple | list | None = None, + data_type: DTypeLike = np.uint16, + entry_key: str = "data", +): + """ + Virtual DataSet writer function for image data. + + Args: + nxsfile (h5py.File): Handle to NeXus file being written. + full_data_shape (tuple | list): Shape of the full dataset, usually defined as (num_frames, *image_size). + start_index(int): The start point for the source data. Defaults to 0. + vds_shape(tuple, optional): Desired shape of the VDS, usually defined as (num_frames, *image_size). \ + The number of frames must be smaller or equal to the one in full_data_shape. Defaults to None. + data_type (DTypeLike, optional): The type of the input data. Defaults to np.uint16. + entry_key (str, optional): Entry key for the Virtual DataSet name. Defaults to data. + """ + block_vds_logger.debug("Start creating VDS ...") + # Where the vds will go + nxdata = nxsfile["/entry/data"] + dset_names = find_datasets_in_file(nxdata) + + vds_shape = ( + tuple(vds_shape) + if vds_shape is not None + else (full_data_shape[0] - start_index, *full_data_shape[1:]) + ) + + # Hack for datasets with no maximum number of frames (eg. Singla) + if len(dset_names) == 1 and full_data_shape[0] > MAX_FRAMES_PER_DATASET: + # nxdata[dset_names[0]].shape[0] > MAX_FRAMES_PER_DATASET + # .maxshape[0] is None + datasets = [ + Dataset( + name=dset_names[0], + source_shape=full_data_shape, + start_index=start_index, + stop_index=full_data_shape[0], + ) + ] + else: + datasets = split_datasets(dset_names, full_data_shape, start_index, vds_shape) + + layout = create_virtual_layout(datasets, data_type) + + # Writea Virtual Dataset in NeXus file + nxdata.create_virtual_dataset(entry_key, layout, fillvalue=-1) + block_vds_logger.debug("VDS correctly written to NeXus file.") diff --git a/src/nexgen/tools/vds_tools/utils.py b/src/nexgen/tools/vds_tools/utils.py index 9dc9df76..1d75a775 100644 --- a/src/nexgen/tools/vds_tools/utils.py +++ b/src/nexgen/tools/vds_tools/utils.py @@ -24,7 +24,7 @@ class VdsSettings(BaseModel): vds_mapping: VdsMapping = VdsMapping.BLOCKED -def find_datasets_in_file(nxdata: h5py.Group) -> list: +def find_datasets_in_file(nxdata: h5py.Group) -> list[str]: """ Look for the source datasets in the NeXus file. Assumes that the source datasets are always h5py.ExternalLink. diff --git a/src/nexgen/tools/vds_w_tools.py b/src/nexgen/tools/vds_w_tools.py index 1a0b1af4..b7ac222b 100644 --- a/src/nexgen/tools/vds_w_tools.py +++ b/src/nexgen/tools/vds_w_tools.py @@ -2,225 +2,20 @@ Tools to write Virtual DataSets """ -from __future__ import annotations - import logging -import operator -from functools import reduce from pathlib import Path -from typing import Sequence import h5py import numpy as np from numpy.typing import DTypeLike -from pydantic.dataclasses import dataclass from nexgen.tools.vds_tools import find_datasets_in_file -from ..utils import MAX_FRAMES_PER_DATASET from .constants import jungfrau_fill_value, jungfrau_gap_size, jungfrau_mod_size vds_logger = logging.getLogger("nexgen.VDSWriter") -@dataclass -class Dataset: - name: str - - # The full shape of the source, regardless of start index - source_shape: Sequence[int] - - # The start index that we should start copying from - start_index: int = 0 - - # The point where we should stop copying. Defaults to maximum for an Eiger - stop_index: int = MAX_FRAMES_PER_DATASET - - # The shape of the destination, including the start_index - dest_shape: Sequence[int] | None = None - - def __post_init__(self): - self.dest_shape = ( - self.source_shape[0] - self.start_index, - *self.source_shape[1:], - ) - if ( - self.stop_index < MAX_FRAMES_PER_DATASET - or self.stop_index < self.source_shape[0] - ): - self.dest_shape = ( - self.dest_shape[0] - (self.source_shape[0] - self.stop_index), - *self.source_shape[1:], - ) - - def __add__(self, x): - """Returns a dataset that has the same start index and shape as if the two were appended to each other.""" - return Dataset( - "", - source_shape=( - self.source_shape[0] + x.source_shape[0], - *self.source_shape[1:], - ), - start_index=self.start_index + x.start_index, - stop_index=self.stop_index + x.stop_index, - ) - - -def split_datasets( - dsets, - data_shape: tuple[int, int, int], - start_idx: int = 0, - vds_shape: tuple[int, int, int] = None, -) -> list[Dataset]: - """ - Splits the full data shape and start index up into values per dataset, - given that each dataset has a maximum size. - - Args: - dsets (Dataset): The input datasets. - data_shape (tuple[int, int, int]): Shape of the data, usually defined as (num_frames, *image_size). - start_idx (int, optional): The start point for the source data. Defaults to 0. - vds_shape(tuple, optional): Desired shape of the VDS, usually defined as (num_frames, *image_size). \ - The number of frames must be smaller or equal to the one in full_data_shape. Defaults to None. - - Raises: - ValueError: If the passed start index value is higher than the dataset lenght. - ValueError: It the passed start index value is negative. - - Returns: - list[Dataset]: A list of datasets. - """ - if start_idx > data_shape[0]: - raise ValueError( - f"Start index {start_idx} must be less than full dataset length {data_shape[0]}" - ) - if start_idx < 0: - raise ValueError("Start index must be positive") - - if not isinstance(data_shape[0], int): - vds_logger.warning("Datashape not passed as int, will attempt to cast") - - if not isinstance(start_idx, int): - vds_logger.warning("VDS start index not passed as int, will attempt to cast") - - if vds_shape and not isinstance(vds_shape[0], int): - vds_logger.warning("VDS start index not passed as int, will attempt to cast") - - if vds_shape is None: - vds_logger.debug( - "VDS shape not chosen, it will be calculated from the full data shape and the chosen start index." - ) - vds_shape = (data_shape[0] - start_idx, *data_shape[1:]) - - full_frames = int(data_shape[0]) - end_cut_frames = int(full_frames - vds_shape[0]) - int(start_idx) - - result = [] - for dset_name in dsets: - dset = Dataset( - name=dset_name, - source_shape=(min(MAX_FRAMES_PER_DATASET, full_frames), *data_shape[1:]), - start_index=min(MAX_FRAMES_PER_DATASET, max(int(start_idx), 0)), - stop_index=min(MAX_FRAMES_PER_DATASET, (full_frames - end_cut_frames)), - ) - # if start index == 1000 then that source dataset is not used and we should - # not pass it on to use as a source for the VDS - # Same goes if all datasets from last files are not used - if dset.start_index != MAX_FRAMES_PER_DATASET: - if dset.stop_index > 0 and dset.stop_index <= MAX_FRAMES_PER_DATASET: - result.append(dset) - start_idx -= MAX_FRAMES_PER_DATASET - full_frames -= MAX_FRAMES_PER_DATASET - - return result - - -def create_virtual_layout(datasets: list[Dataset], data_type: DTypeLike): - """ - Create a virtual layout and populate it based on the provided data. - - Args: - datasets (list[Dataset]): A list of datasets that are to be merged. - data_type (DTypeLike): The type of the input data. - - Returns: - layout (h5py.VirtualLayout): Virtual layout. - """ - full_dataset: Dataset = reduce(operator.add, datasets) - layout = h5py.VirtualLayout(shape=full_dataset.dest_shape, dtype=data_type) - - dest_start = 0 - for dataset in datasets: - if dataset.stop_index == dataset.source_shape[0]: - dest_end = dest_start + dataset.source_shape[0] - dataset.start_index - else: - dest_end = dest_start + dataset.dest_shape[0] - - vsource = h5py.VirtualSource( - ".", "/entry/data/" + dataset.name, shape=dataset.source_shape - ) - - layout[dest_start:dest_end, :, :] = vsource[ - dataset.start_index : dataset.stop_index, :, : - ] - dest_start = dest_end - - return layout - - -def image_vds_writer( - nxsfile: h5py.File, - full_data_shape: tuple | list, - start_index: int = 0, - vds_shape: tuple | list | None = None, - data_type: DTypeLike = np.uint16, - entry_key: str = "data", -): - """ - Virtual DataSet writer function for image data. - - Args: - nxsfile (h5py.File): Handle to NeXus file being written. - full_data_shape (tuple | list): Shape of the full dataset, usually defined as (num_frames, *image_size). - start_index(int): The start point for the source data. Defaults to 0. - vds_shape(tuple, optional): Desired shape of the VDS, usually defined as (num_frames, *image_size). \ - The number of frames must be smaller or equal to the one in full_data_shape. Defaults to None. - data_type (DTypeLike, optional): The type of the input data. Defaults to np.uint16. - entry_key (str, optional): Entry key for the Virtual DataSet name. Defaults to data. - """ - vds_logger.debug("Start creating VDS ...") - # Where the vds will go - nxdata = nxsfile["/entry/data"] - dset_names = find_datasets_in_file(nxdata) - - vds_shape = ( - tuple(vds_shape) - if vds_shape is not None - else (full_data_shape[0] - start_index, *full_data_shape[1:]) - ) - - # Hack for datasets with no maximum number of frames (eg. Singla) - if len(dset_names) == 1 and full_data_shape[0] > MAX_FRAMES_PER_DATASET: - # nxdata[dset_names[0]].shape[0] > MAX_FRAMES_PER_DATASET - # .maxshape[0] is None - datasets = [ - Dataset( - name=dset_names[0], - source_shape=full_data_shape, - start_index=start_index, - stop_index=full_data_shape[0], - ) - ] - else: - datasets = split_datasets(dset_names, full_data_shape, start_index, vds_shape) - - layout = create_virtual_layout(datasets, data_type) - - # Writea Virtual Dataset in NeXus file - nxdata.create_virtual_dataset(entry_key, layout, fillvalue=-1) - vds_logger.debug("VDS correctly written to NeXus file.") - - def jungfrau_vds_writer( nxsfile: h5py.File, vds_shape: tuple | list, diff --git a/tests/tools/test_VDS_tools.py b/tests/tools/test_VDS_tools.py index 981e50cf..8313cc51 100644 --- a/tests/tools/test_VDS_tools.py +++ b/tests/tools/test_VDS_tools.py @@ -2,106 +2,12 @@ from unittest.mock import MagicMock import h5py -import numpy as np -import pytest from nexgen.tools.vds_w_tools import ( - Dataset, - create_virtual_layout, - image_vds_writer, jungfrau_vds_writer, - split_datasets, ) -def test_when_get_frames_and_shape_less_than_1000_then_correct(): - sshape = split_datasets(["test1"], (500, 10, 10)) - assert sshape == [Dataset("test1", (500, 10, 10), 0, 500)] - - create_virtual_layout(sshape, np.uint16) - - -def test_when_get_frames_and_shape_greater_than_1000_then_correct(): - sshape = split_datasets(["test1", "test2"], (1300, 10, 10)) - assert sshape == [ - Dataset("test1", (1000, 10, 10), 0, 1000), - Dataset("test2", (300, 10, 10), 0, 300), - ] - - create_virtual_layout(sshape, np.uint16) - - -def test_when_get_frames_and_shape_less_than_1000_non_zero_then_correct(): - sshape = split_datasets(["test1"], (500, 10, 10), 200, (300, 10, 10)) - assert sshape == [Dataset("test1", (500, 10, 10), 200, 500)] - - create_virtual_layout(sshape, np.uint16) - - -def test_when_get_frames_and_shape_greater_than_1000_non_zero_then_correct(): - sshape = split_datasets(["test1", "test2"], (1500, 10, 10), 200) - assert sshape == [ - Dataset("test1", (1000, 10, 10), 200, 1000), - Dataset("test2", (500, 10, 10), 0, 500), - ] - - create_virtual_layout(sshape, np.uint16) - - -def test_when_get_frames_and_shape_greater_than_1000_non_zero_greater_than_1000_then_correct(): - sshape = split_datasets(["test1", "test2"], (1500, 10, 10), 1200) - assert sshape == [ - Dataset("test2", (500, 10, 10), 200, 500), - ] - - create_virtual_layout(sshape, np.uint16) - - -def test_when_get_frames_and_shape_much_greater_than_1000_non_zero_greater_than_1000_then_correct(): - returned = split_datasets( - ["test1", "test2", "test3", "test4"], (3100, 10, 10), 1100 - ) - assert returned == [ - Dataset("test2", (1000, 10, 10), 100, 1000), - Dataset("test3", (1000, 10, 10), 0, 1000), - Dataset("test4", (100, 10, 10), 0, 100), - ] - - create_virtual_layout(returned, np.uint16) - - -def test_when_start_idx_higher_than_full_then_exception_raised(): - with pytest.raises(ValueError): - split_datasets(["test1", "test2"], (1100, 10, 10), 3100) - - -def test_when_start_idx_negative_then_exception_raised(): - with pytest.raises(ValueError): - split_datasets(["test1"], (1100, 10, 10), -100) - - -def test_when_float_shape_passed_to_vds_writer_then_no_exception( - nexus_file_with_single_dataset, -): - image_vds_writer(nexus_file_with_single_dataset, (500.0, 10.0, 10.0), start_index=0) - - -def test_when_float_start_index_passed_to_vds_writer_then_no_exception( - nexus_file_with_single_dataset, -): - image_vds_writer( - nexus_file_with_single_dataset, (500.0, 10.0, 10.0), start_index=0.0 - ) - - -def test_given_file_with_no_dataset_external_links_then_exception_is_sensible(): - test_hdf_file = tempfile.TemporaryFile() - test_nexus_file = h5py.File(test_hdf_file, "w") - test_nexus_file["/entry/data/data_0001"] = MagicMock() - with pytest.raises(KeyError): - image_vds_writer(test_nexus_file, (1000, 10, 10)) - - def test_jungfrau_vds_writer_with_external_dsets(): test_hdf_file = tempfile.TemporaryFile() test_nexus_file = h5py.File(test_hdf_file, "w") diff --git a/tests/tools/vds/test_blocked_vds.py b/tests/tools/vds/test_blocked_vds.py new file mode 100644 index 00000000..d51fb3d5 --- /dev/null +++ b/tests/tools/vds/test_blocked_vds.py @@ -0,0 +1,101 @@ +import tempfile +from unittest.mock import MagicMock + +import h5py +import numpy as np +import pytest + +from nexgen.tools.vds_tools.blocked_mapping import ( + Dataset, + create_virtual_layout, + image_vds_writer, + split_datasets, +) + + +def test_when_get_frames_and_shape_less_than_1000_then_correct(): + sshape = split_datasets(["test1"], (500, 10, 10)) + assert sshape == [Dataset("test1", (500, 10, 10), 0, 500)] + + create_virtual_layout(sshape, np.uint16) + + +def test_when_get_frames_and_shape_greater_than_1000_then_correct(): + sshape = split_datasets(["test1", "test2"], (1300, 10, 10)) + assert sshape == [ + Dataset("test1", (1000, 10, 10), 0, 1000), + Dataset("test2", (300, 10, 10), 0, 300), + ] + + create_virtual_layout(sshape, np.uint16) + + +def test_when_get_frames_and_shape_less_than_1000_non_zero_then_correct(): + sshape = split_datasets(["test1"], (500, 10, 10), 200, (300, 10, 10)) + assert sshape == [Dataset("test1", (500, 10, 10), 200, 500)] + + create_virtual_layout(sshape, np.uint16) + + +def test_when_get_frames_and_shape_greater_than_1000_non_zero_then_correct(): + sshape = split_datasets(["test1", "test2"], (1500, 10, 10), 200) + assert sshape == [ + Dataset("test1", (1000, 10, 10), 200, 1000), + Dataset("test2", (500, 10, 10), 0, 500), + ] + + create_virtual_layout(sshape, np.uint16) + + +def test_when_get_frames_and_shape_greater_than_1000_non_zero_greater_than_1000_then_correct(): + sshape = split_datasets(["test1", "test2"], (1500, 10, 10), 1200) + assert sshape == [ + Dataset("test2", (500, 10, 10), 200, 500), + ] + + create_virtual_layout(sshape, np.uint16) + + +def test_when_get_frames_and_shape_much_greater_than_1000_non_zero_greater_than_1000_then_correct(): + returned = split_datasets( + ["test1", "test2", "test3", "test4"], (3100, 10, 10), 1100 + ) + assert returned == [ + Dataset("test2", (1000, 10, 10), 100, 1000), + Dataset("test3", (1000, 10, 10), 0, 1000), + Dataset("test4", (100, 10, 10), 0, 100), + ] + + create_virtual_layout(returned, np.uint16) + + +def test_when_start_idx_higher_than_full_then_exception_raised(): + with pytest.raises(ValueError): + split_datasets(["test1", "test2"], (1100, 10, 10), 3100) + + +def test_when_start_idx_negative_then_exception_raised(): + with pytest.raises(ValueError): + split_datasets(["test1"], (1100, 10, 10), -100) + + +def test_when_float_shape_passed_to_vds_writer_then_no_exception( + nexus_file_with_single_dataset, +): + image_vds_writer(nexus_file_with_single_dataset, (500.0, 10.0, 10.0), start_index=0) + + +def test_when_float_start_index_passed_to_vds_writer_then_no_exception( + nexus_file_with_single_dataset, +): + image_vds_writer( + nexus_file_with_single_dataset, (500.0, 10.0, 10.0), start_index=0.0 + ) + + +def test_given_file_with_no_dataset_external_links_then_exception_is_sensible(): + test_hdf_file = tempfile.TemporaryFile() + test_nexus_file = h5py.File(test_hdf_file, "w") + test_nexus_file["/entry/data/data_0001"] = MagicMock() + with pytest.raises(KeyError): + image_vds_writer(test_nexus_file, (1000, 10, 10)) From 32443e970225d5621fd763f259340eaf66f4b3b9 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Thu, 9 Jul 2026 13:20:34 +0100 Subject: [PATCH 3/7] Move clean links to utils --- src/nexgen/nxs_write/nxmx_writer.py | 2 +- src/nexgen/tools/vds_tools/__init__.py | 2 ++ src/nexgen/tools/vds_tools/utils.py | 41 ++++++++++++++++++++++++++ src/nexgen/tools/vds_w_tools.py | 37 ----------------------- 4 files changed, 44 insertions(+), 38 deletions(-) diff --git a/src/nexgen/nxs_write/nxmx_writer.py b/src/nexgen/nxs_write/nxmx_writer.py index 1aa40bfd..88985fc2 100644 --- a/src/nexgen/nxs_write/nxmx_writer.py +++ b/src/nexgen/nxs_write/nxmx_writer.py @@ -17,9 +17,9 @@ from ..nxs_utils.goniometer import Goniometer from ..nxs_utils.sample import Sample from ..nxs_utils.source import Attenuator, Beam, Source +from ..tools.vds_tools import clean_unused_links from ..tools.vds_tools.blocked_mapping import image_vds_writer from ..tools.vds_w_tools import ( - clean_unused_links, jungfrau_vds_writer, ) from ..utils import ( diff --git a/src/nexgen/tools/vds_tools/__init__.py b/src/nexgen/tools/vds_tools/__init__.py index 0bf30c65..cb3ec0ae 100644 --- a/src/nexgen/tools/vds_tools/__init__.py +++ b/src/nexgen/tools/vds_tools/__init__.py @@ -1,6 +1,7 @@ from nexgen.tools.vds_tools.utils import ( VdsMapping, VdsSettings, + clean_unused_links, define_vds_dtype_from_bit_depth, find_datasets_in_file, ) @@ -10,4 +11,5 @@ "VdsSettings", "find_datasets_in_file", "define_vds_dtype_from_bit_depth", + "clean_unused_links", ] diff --git a/src/nexgen/tools/vds_tools/utils.py b/src/nexgen/tools/vds_tools/utils.py index 1d75a775..89991976 100644 --- a/src/nexgen/tools/vds_tools/utils.py +++ b/src/nexgen/tools/vds_tools/utils.py @@ -1,3 +1,4 @@ +import logging from enum import StrEnum from typing import Any, TypeAlias @@ -8,6 +9,8 @@ PydanticDTypeLike: TypeAlias = np.dtype[Any] | type | str | None +vds_logger = logging.getLogger("nexgen.tools.vds_tools") + class VdsMapping(StrEnum): BLOCKED = "blocked" # default, usual one @@ -43,6 +46,7 @@ def find_datasets_in_file(nxdata: h5py.Group) -> list[str]: if isinstance(nxdata.get(k, getlink=True), h5py.ExternalLink): dsets.append(k) if not dsets: + vds_logger.error("No extrnale link datasets found.") raise KeyError( f"No External Link datasets found in NeXus file under {nxdata.name}" ) @@ -57,3 +61,40 @@ def define_vds_dtype_from_bit_depth(bit_depth: int) -> DTypeLike: return np.uint8 else: return np.uint16 + + +def clean_unused_links( + nxsfile: h5py.File, + vds_shape: tuple | list, + start_index: int = 0, +): + """ + Remove links to external data not used in VDS. + + Args: + nxsfile (h5py.File): Handle to NeXus file being written. + vds_shape (tuple | list): Actual shape of the VDS dataset, usually defined as (num_frames, *image_size). + start_index(int): The start point for the source data. Defaults to 0. + """ + vds_logger.debug("Cleaning links unused in VDS ...") + # Location of the VDS + nxdata = nxsfile["/entry/data"] + dataset_names = find_datasets_in_file(nxdata) + if len(dataset_names) == 1: + vds_logger.debug("Only one linked file, no need to remove it.") + return + datasets = [nxdata[name] for name in dataset_names] + dataset_lengths = [d.shape[0] for d in datasets] + if sum(dataset_lengths) == vds_shape[0]: + vds_logger.debug("All links are used in VDS, no need to remove any.") + return + for i, _ in enumerate(datasets): + # unlink datasets before the start of VDS + if sum(dataset_lengths[0 : i + 1]) < start_index: + vds_logger.debug(f"Removing {dataset_names[i]} link.") + del nxdata[dataset_names[i]] + # unlink datasets after the end of VDS + if sum(dataset_lengths[0:i]) > start_index + vds_shape[0]: + vds_logger.debug(f"Removing {dataset_names[i]} link.") + del nxdata[dataset_names[i]] + vds_logger.debug("Links unused in VDS removed from NeXus file.") diff --git a/src/nexgen/tools/vds_w_tools.py b/src/nexgen/tools/vds_w_tools.py index b7ac222b..6337aaa6 100644 --- a/src/nexgen/tools/vds_w_tools.py +++ b/src/nexgen/tools/vds_w_tools.py @@ -96,40 +96,3 @@ def vds_file_writer( vds.create_virtual_dataset("data", layout, fillvalue=-1) nxdata["data"] = h5py.ExternalLink(vds_filename.name, "data") vds_logger.debug(f"{vds_filename} written and link added to NeXus file.") - - -def clean_unused_links( - nxsfile: h5py.File, - vds_shape: tuple | list, - start_index: int = 0, -): - """ - Remove links to external data not used in VDS. - - Args: - nxsfile (h5py.File): Handle to NeXus file being written. - vds_shape (tuple | list): Actual shape of the VDS dataset, usually defined as (num_frames, *image_size). - start_index(int): The start point for the source data. Defaults to 0. - """ - vds_logger.debug("Cleaning links unused in VDS ...") - # Location of the VDS - nxdata = nxsfile["/entry/data"] - dataset_names = find_datasets_in_file(nxdata) - if len(dataset_names) == 1: - vds_logger.debug("Only one linked file, no need to remove it.") - return - datasets = [nxdata[name] for name in dataset_names] - dataset_lengths = [d.shape[0] for d in datasets] - if sum(dataset_lengths) == vds_shape[0]: - vds_logger.debug("All links are used in VDS, no need to remove any.") - return - for i, _ in enumerate(datasets): - # unlink datasets before the start of VDS - if sum(dataset_lengths[0 : i + 1]) < start_index: - vds_logger.debug(f"Removing {dataset_names[i]} link.") - del nxdata[dataset_names[i]] - # unlink datasets after the end of VDS - if sum(dataset_lengths[0:i]) > start_index + vds_shape[0]: - vds_logger.debug(f"Removing {dataset_names[i]} link.") - del nxdata[dataset_names[i]] - vds_logger.debug("Links unused in VDS removed from NeXus file.") From 97ab24e9a67a995097128f06951c0dcaaed6dc54 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Thu, 9 Jul 2026 16:26:23 +0100 Subject: [PATCH 4/7] Finish moving files and functions around --- src/nexgen/nxs_write/nxmx_writer.py | 4 +- src/nexgen/tools/vds_tools/tiled_mapping.py | 53 +++++++++++++++++++ .../{vds_w_tools.py => vds_tools/vds_file.py} | 38 ------------- .../test_tiled_mapping.py} | 2 +- 4 files changed, 55 insertions(+), 42 deletions(-) create mode 100644 src/nexgen/tools/vds_tools/tiled_mapping.py rename src/nexgen/tools/{vds_w_tools.py => vds_tools/vds_file.py} (61%) rename tests/tools/{test_VDS_tools.py => vds/test_tiled_mapping.py} (95%) diff --git a/src/nexgen/nxs_write/nxmx_writer.py b/src/nexgen/nxs_write/nxmx_writer.py index 88985fc2..be7a5d9e 100644 --- a/src/nexgen/nxs_write/nxmx_writer.py +++ b/src/nexgen/nxs_write/nxmx_writer.py @@ -19,9 +19,7 @@ from ..nxs_utils.source import Attenuator, Beam, Source from ..tools.vds_tools import clean_unused_links from ..tools.vds_tools.blocked_mapping import image_vds_writer -from ..tools.vds_w_tools import ( - jungfrau_vds_writer, -) +from ..tools.vds_tools.tiled_mapping import jungfrau_vds_writer from ..utils import ( MAX_FRAMES_PER_DATASET, MAX_SUFFIX_DIGITS, diff --git a/src/nexgen/tools/vds_tools/tiled_mapping.py b/src/nexgen/tools/vds_tools/tiled_mapping.py new file mode 100644 index 00000000..9a398a80 --- /dev/null +++ b/src/nexgen/tools/vds_tools/tiled_mapping.py @@ -0,0 +1,53 @@ +"""Create a Virtual DataSet with a strided mapping, ie mappaing every {n} frames from each file + +For now only the JF1M functionality has been added, the full vds capabilities will be added later. +""" + +import logging + +import h5py +import numpy as np +from numpy.typing import DTypeLike + +from nexgen.tools.constants import ( + jungfrau_fill_value, + jungfrau_gap_size, + jungfrau_mod_size, +) +from nexgen.tools.vds_tools.utils import find_datasets_in_file + +tiled_vds_logger = logging.getLogger("nexgen.tools.vds_tools.utils") + + +def jungfrau_vds_writer( + nxsfile: h5py.File, + vds_shape: tuple | list, + data_type: DTypeLike = np.uint16, + source_dsets: list[str] | None = None, +): + """Write VDS for Jungfrau 1M use case, with a tiled layout.""" + external_dsets = True + entry_key = "data" + frames = vds_shape[0] + + nxdata = nxsfile["/entry/data"] + if not source_dsets: + source_dsets = find_datasets_in_file(nxdata) + external_dsets = False + + sources = [] + for dset in source_dsets: + source_path = dset if external_dsets is True else "." + source_name = entry_key if external_dsets is True else f"/entry/data/{dset}" + source = h5py.VirtualSource( + source_path, source_name, shape=(frames, *jungfrau_mod_size) + ) + sources.append(source) + + layout = h5py.VirtualLayout(shape=vds_shape, dtype=data_type) + # The first one is the upper one + s0 = jungfrau_mod_size[0] + jungfrau_gap_size[0] + layout[:, : jungfrau_mod_size[0], :] = sources[1][:, :, :] + layout[:, s0:, :] = sources[0][:, :, :] + + nxdata.create_virtual_dataset(entry_key, layout, fillvalue=jungfrau_fill_value) diff --git a/src/nexgen/tools/vds_w_tools.py b/src/nexgen/tools/vds_tools/vds_file.py similarity index 61% rename from src/nexgen/tools/vds_w_tools.py rename to src/nexgen/tools/vds_tools/vds_file.py index 6337aaa6..6949952f 100644 --- a/src/nexgen/tools/vds_w_tools.py +++ b/src/nexgen/tools/vds_tools/vds_file.py @@ -9,47 +9,9 @@ import numpy as np from numpy.typing import DTypeLike -from nexgen.tools.vds_tools import find_datasets_in_file - -from .constants import jungfrau_fill_value, jungfrau_gap_size, jungfrau_mod_size - vds_logger = logging.getLogger("nexgen.VDSWriter") -def jungfrau_vds_writer( - nxsfile: h5py.File, - vds_shape: tuple | list, - data_type: DTypeLike = np.uint16, - source_dsets: list[str] | None = None, -): - """Write VDS for Jungfrau 1M use case, with a tiled layout.""" - external_dsets = True - entry_key = "data" - frames = vds_shape[0] - - nxdata = nxsfile["/entry/data"] - if not source_dsets: - source_dsets = find_datasets_in_file(nxdata) - external_dsets = False - - sources = [] - for dset in source_dsets: - source_path = dset if external_dsets is True else "." - source_name = entry_key if external_dsets is True else f"/entry/data/{dset}" - source = h5py.VirtualSource( - source_path, source_name, shape=(frames, *jungfrau_mod_size) - ) - sources.append(source) - - layout = h5py.VirtualLayout(shape=vds_shape, dtype=data_type) - # The first one is the upper one - s0 = jungfrau_mod_size[0] + jungfrau_gap_size[0] - layout[:, : jungfrau_mod_size[0], :] = sources[1][:, :, :] - layout[:, s0:, :] = sources[0][:, :, :] - - nxdata.create_virtual_dataset(entry_key, layout, fillvalue=jungfrau_fill_value) - - def vds_file_writer( nxsfile: h5py.File, datafiles: list[Path], diff --git a/tests/tools/test_VDS_tools.py b/tests/tools/vds/test_tiled_mapping.py similarity index 95% rename from tests/tools/test_VDS_tools.py rename to tests/tools/vds/test_tiled_mapping.py index 8313cc51..8250d8f4 100644 --- a/tests/tools/test_VDS_tools.py +++ b/tests/tools/vds/test_tiled_mapping.py @@ -3,7 +3,7 @@ import h5py -from nexgen.tools.vds_w_tools import ( +from nexgen.tools.vds_tools.tiled_mapping import ( jungfrau_vds_writer, ) From 7eaaa66ae8837be0591234ee7f7577f9a98d7317 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Thu, 9 Jul 2026 16:27:00 +0100 Subject: [PATCH 5/7] And maybe save everything --- src/nexgen/nxs_write/ed_nxmx_writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nexgen/nxs_write/ed_nxmx_writer.py b/src/nexgen/nxs_write/ed_nxmx_writer.py index 4820e2c5..ba6d3eb8 100644 --- a/src/nexgen/nxs_write/ed_nxmx_writer.py +++ b/src/nexgen/nxs_write/ed_nxmx_writer.py @@ -15,7 +15,7 @@ from ..nxs_utils import Attenuator, Beam, Detector, Goniometer, Source from ..tools.vds_tools.blocked_mapping import image_vds_writer -from ..tools.vds_w_tools import vds_file_writer +from ..tools.vds_tools.vds_file import vds_file_writer from ..utils import coord2mcstas from .nxclass_writers import ( write_NXcoordinate_system_set, From 00169424b02a3db89748110a67ac1d75dcc6000f Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Thu, 9 Jul 2026 17:30:04 +0100 Subject: [PATCH 6/7] Add strided mapping to the vds writer --- src/nexgen/nxs_write/nxmx_writer.py | 122 +++++++++++++++++----------- 1 file changed, 76 insertions(+), 46 deletions(-) diff --git a/src/nexgen/nxs_write/nxmx_writer.py b/src/nexgen/nxs_write/nxmx_writer.py index be7a5d9e..e0d1ac55 100644 --- a/src/nexgen/nxs_write/nxmx_writer.py +++ b/src/nexgen/nxs_write/nxmx_writer.py @@ -2,8 +2,6 @@ A writer for NXmx format NeXus Files. """ -from __future__ import annotations - import logging import math from datetime import datetime @@ -13,6 +11,9 @@ import numpy as np from numpy.typing import DTypeLike +from nexgen.tools.vds_tools.strided_mapping import write_strided_vds +from nexgen.tools.vds_tools.utils import VdsMapping + from ..nxs_utils.detector import Detector from ..nxs_utils.goniometer import Goniometer from ..nxs_utils.sample import Sample @@ -243,69 +244,98 @@ def write_vds( vds_offset: int = 0, vds_shape: tuple[int, int, int] = None, vds_dtype: DTypeLike = np.uint16, + vds_mapping: VdsMapping = VdsMapping.BLOCKED, + stride: int = 2, clean_up: bool = False, ): """Write a Virtual Dataset. - This method adds a VDS under /entry/data/data in the NeXus file, linking to either the full datasets or the subset defined by \ + This method adds a VDS under /entry/data/data in the NeXus file, linking to either the full datasets or the subset defined by vds_offset (used as start index) and vds_shape. WARNING. Only use clean up if the data collection is finished and all the files have already been written. Args: - vds_offset (int, optional): Start index for the vds writer. Defaults to 0. - vds_shape (tuple[int,int,int], optional): Shape of the data which will be linked in the VDS. If not passed, it will be defined as \ + vds_offset (int, optional): Start index for the vds writer. For the strided mapping this will be where the source starts. + Defaults to 0. + vds_shape (tuple[int,int,int], optional): Shape of the data which will be linked in the VDS. If not passed, it will be defined as (tot_num_imgs - start_idx, *image_size). Defaults to None. vds_dtype (DTypeLike, optional): The type of the input data. Defaults to np.uint16. + vds_mapping (VdsMapping, optional): Mapping to apply to build the vds. Defaults to "blocked". + stride (int, optional): Every how many images to map the strided vds. Defaults to 2 (map to every other image). clean_up(bool, optional): Clean up unused links in vds. Defaults to False. """ - if not vds_shape: - vds_shape = ( - self.tot_num_imgs - vds_offset, - *self.detector.detector_params.image_size, - ) - if self.goniometer.get_number_of_scan_points() != vds_shape[0]: - vds_shape = ( - self.goniometer.get_number_of_scan_points(), - *vds_shape[1:], - ) - nxmx_logger.warning( - "The number of scan points doesn't match the calculated vds_shape. \ - Resetting it to match the number of frames indicated by the scan." - ) - - nxmx_logger.debug(f"VDS shape set to {vds_shape}.") - - with h5py.File(self.filename, "r+") as nxs: - # For a coming ticket - for now write a separate file - # Here will be better to have a match-case for VDS mapping. Default is the same as blocked. - if "jungfrau" in self.detector.detector_params.description.lower(): + if "jungfrau" in self.detector.detector_params.description.lower(): + with h5py.File(self.filename, "r+") as nxs: jungfrau_vds_writer( nxs, (self.tot_num_imgs, *self.detector.detector_params.image_size), data_type=vds_dtype, ) - else: - image_vds_writer( - nxs, - (self.tot_num_imgs, *self.detector.detector_params.image_size), - start_index=vds_offset, - vds_shape=vds_shape, - data_type=vds_dtype, + nxmx_logger.info("VDS for JF1M written.") + return + + match vds_mapping: + case VdsMapping.BLOCKED: + nxmx_logger.debug("Writing 'standard' blocked VDS") + if not vds_shape: + vds_shape = ( + self.tot_num_imgs - vds_offset, + *self.detector.detector_params.image_size, + ) + if self.goniometer.get_number_of_scan_points() != vds_shape[0]: + vds_shape = ( + self.goniometer.get_number_of_scan_points(), + *vds_shape[1:], + ) + nxmx_logger.warning( + "The number of scan points doesn't match the calculated vds_shape. \ + Resetting it to match the number of frames indicated by the scan." + ) + + nxmx_logger.debug(f"VDS shape set to {vds_shape}.") + + with h5py.File(self.filename, "r+") as nxs: + image_vds_writer( + nxs, + (self.tot_num_imgs, *self.detector.detector_params.image_size), + start_index=vds_offset, + vds_shape=vds_shape, + data_type=vds_dtype, + ) + + if clean_up is True: + nxmx_logger.warning("Starting clean up of unused links.") + clean_unused_links( + nxs, + vds_shape=vds_shape, + start_index=vds_offset, + ) + + # If number of frames in the VDS is lower than the total, nimages in NXcollection should be overwritten to match this + if vds_shape[0] < self.tot_num_imgs: + del nxs["/entry/instrument/detector/detectorSpecific/nimages"] + nxs[ + "/entry/instrument/detector/detectorSpecific" + ].create_dataset("nimages", data=vds_shape[0]) + nxmx_logger.info(f"VDS correctly written to file {self.filename}") + case VdsMapping.STRIDED: + nxmx_logger.info( + f"Writing strided nexus file with start idx {vds_offset} and stride {stride}." ) - if clean_up is True: - nxmx_logger.warning("Starting clean up of unused links.") - clean_unused_links( - nxs, - vds_shape=vds_shape, - start_index=vds_offset, - ) - - # If number of frames in the VDS is lower than the total, nimages in NXcollection should be overwritten to match this - if vds_shape[0] < self.tot_num_imgs: - del nxs["/entry/instrument/detector/detectorSpecific/nimages"] - nxs["/entry/instrument/detector/detectorSpecific"].create_dataset( - "nimages", data=vds_shape[0] + with h5py.File(self.filename, "r+") as nxs: + write_strided_vds( + nxs, + (self.tot_num_imgs, *self.detector.detector_params.image_size), + vds_offset, + stride, + vds_dtype, + ) + nxmx_logger.info(f"VDS correctly written to file {self.filename}") + case VdsMapping.TILED: + nxmx_logger.error( + "Tiled VDS mapping not implemented yet, will not write vds." ) + return class EventNXmxFileWriter(NXmxFileWriter): From 27cb071ba0be6d574665ee878cae5f9336f64ce0 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Thu, 9 Jul 2026 17:48:20 +0100 Subject: [PATCH 7/7] Add mapping to eiger writer for I19) --- src/nexgen/beamlines/i19_2/eiger.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nexgen/beamlines/i19_2/eiger.py b/src/nexgen/beamlines/i19_2/eiger.py index ff4be507..c509be35 100644 --- a/src/nexgen/beamlines/i19_2/eiger.py +++ b/src/nexgen/beamlines/i19_2/eiger.py @@ -291,7 +291,8 @@ def start_writer( vds_offset=vds_settings.vds_offset, vds_shape=vds_settings.vds_shape, vds_dtype=vds_settings.vds_dtype, - ) # TODO add mapping + vds_mapping=vds_settings.vds_mapping, + ) if parameters.timestamps[1]: NXmx_writer.update_timestamps(parameters.timestamps[1], "end_time") if notes: