diff --git a/.gitignore b/.gitignore index 6ab2335f..2492cef4 100644 --- a/.gitignore +++ b/.gitignore @@ -566,3 +566,8 @@ cython_debug/ # setuptools-scm generated version file src/nyx/python/nyxus/_version.py .planning/ + +# Feature tables written by the test suites into the tree +NyxusFeatures.arrow +NyxusFeatures.parquet +NyxusFeatures.csv diff --git a/CMakeLists.txt b/CMakeLists.txt index 71318bc1..082352b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -127,6 +127,10 @@ set(SOURCE src/nyx/raw_tiff.h # sources src/nyx/3rdparty/dsyevj3.cpp + # native OME metadata parsing + unified format detection + src/nyx/ome/format_detect.cpp + src/nyx/ome/ome_tiff_meta.cpp + src/nyx/ome/ome_zarr_meta.cpp src/nyx/features/basic_morphology.cpp src/nyx/features/caliper_feret.cpp src/nyx/features/caliper_martin.cpp @@ -154,6 +158,7 @@ set(SOURCE src/nyx/features/hexagonality_polygonality.cpp src/nyx/features/image_matrix.cpp src/nyx/features/image_matrix_nontriv.cpp + src/nyx/features/voxel_cloud_nontriv.cpp src/nyx/features/2d_geomoments.cpp src/nyx/features/2d_geomoments_basic.cpp src/nyx/features/2d_geomoments_basic_nt.cpp @@ -223,6 +228,7 @@ set(SOURCE src/nyx/phase2_25d.cpp src/nyx/phase2_3d.cpp src/nyx/phase3.cpp + src/nyx/phase3_3d.cpp src/nyx/pixel_feed.cpp src/nyx/raw_image_loader.cpp src/nyx/reduce_trivial_rois.cpp @@ -275,7 +281,25 @@ if(USE_Z5) include_directories (${ZLIB_INCLUDE_DIR}) add_definitions(-DHAVE_ZLIB) # telling NIFTI code of zlib availability, otherwise it'll support only uncompressed NIFTI endif() - + + # zstd: the DEFAULT codec of Zarr v3 (OME-Zarr 0.5). z5's ZstdCompressor is gated by + # WITH_ZSTD; without it, zstd-compressed v3 chunks fail to decode. Optional: if zstd + # isn't found we still read uncompressed / blosc / zlib v3 stores. Defining WITH_ZSTD + # makes a z5 header include , so the header must be found as well as the + # library -- a build environment can carry libzstd while its headers sit outside the + # dependency prefix, and defining the macro on the strength of the library alone turns + # that into a fatal include error. + find_library(ZSTD_LIBRARY NAMES zstd libzstd) + find_path(ZSTD_INCLUDE_DIR NAMES zstd.h) + if(ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR) + add_definitions(-DWITH_ZSTD) + list(APPEND Nyxus_LIBRARIES ${ZSTD_LIBRARY}) + include_directories(${ZSTD_INCLUDE_DIR}) + message(STATUS "zstd found (${ZSTD_LIBRARY}) -> Zarr v3 zstd codec enabled") + else() + message(STATUS "zstd not fully available (lib='${ZSTD_LIBRARY}' include='${ZSTD_INCLUDE_DIR}') -> Zarr v3 zstd-compressed chunks will be unsupported") + endif() + if(nlohmann_json_FOUND) include_directories(${nlohmann_json_INCLUDE_DIR}) endif() diff --git a/README.md b/README.md index 0d0322d2..4d715976 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Nyxus is a feature-rich, optimized, Python/C++ application capable of analyzing Nyxus can be used via Python or command line and is available in containerized form for reproducible execution. Nyxus computes over 450 combined intensity, texture, and morphological features at the ROI or whole image level with more in development. Key features that make Nyxus unique among other image feature extraction applications is its ability to operate at any scale, its highly validated algorithms, and its modular nature that makes the addition of new features straightforward. -Currently, Nyxus can read 2D image data from OME-TIFF, OME-Zarr, and DICOM 2D Grayscale images. Nyxus also reads compressed and uncompressed NIFTI 3D files. Nyxus Python API supports featurizing in-memory 2D image data represented by NumPy arrays. +Currently, Nyxus can read 2D image data from OME-TIFF, OME-Zarr, and DICOM 2D Grayscale images. Nyxus reads 3D volumes from compressed and uncompressed NIFTI files, and natively from OME-TIFF and OME-Zarr (NGFF 0.4 and 0.5), including 5-dimensional (X, Y, Z, C, T) datasets whose axis order is taken from the file's own metadata rather than assumed. Nyxus Python API supports featurizing in-memory 2D image data represented by NumPy arrays. The docs can be found at [Read the Docs](https://nyxus.readthedocs.io/en/latest/). @@ -34,7 +34,7 @@ conda install nyxus -c conda-forge ## Usage -The library provides class `Nyxus` for 2-dimensional TIFF, OME.TIFF, OME.ZARR, and DICOM slides and intensity-mask slide pairs, and class `Nyxus3D` for NIFTI volumes and intensity-volume volume pairs. Additionally to a single-file representation, a volume can be represented by its z-slices residing in separate 2-dimensional TIFF or ONE.TIFF slides (so called "layout A"). Slides and volumes can be featurized as all the files in a directory filtered with a file pattern passed specified. Alternatively, explicit file name pairs and pair lists can be featurized. Alternatively, 2D and 3D NumPy arrays can be featurized. +The library provides class `Nyxus` for 2-dimensional TIFF, OME.TIFF, OME.ZARR, and DICOM slides and intensity-mask slide pairs, and class `Nyxus3D` for NIFTI, OME-TIFF, and OME-Zarr volumes and intensity-volume volume pairs. Additionally to a single-file representation, a volume can be represented by its z-slices residing in separate 2-dimensional TIFF or ONE.TIFF slides (so called "layout A"). Slides and volumes can be featurized as all the files in a directory filtered with a file pattern passed specified. Alternatively, explicit file name pairs and pair lists can be featurized. Alternatively, 2D and 3D NumPy arrays can be featurized. ### 2D usage @@ -190,6 +190,25 @@ features2 = nyx.featurize_directory (idir, mdir, file_pattern=".*\.nii\.gz") ``` Note on CT datasets: voxel intensities recorded in the Hounsfield units are automatically read by Nyxus as 0-based values by adding the minimum value of an original file (typically -1024). +Volumes are also read natively from OME-TIFF and OME-Zarr, including 5-dimensional (X, Y, Z, C, T) datasets. The axis order comes from the file's own metadata, so non-canonical layouts are addressed correctly rather than assumed to be `TCZYX`: + +```python +import nyxus +nyx = nyxus.Nyxus3D (["*3D_ALL*"]) + +features = nyx.featurize_directory (idir, mdir, file_pattern=".*\.ome\.zarr") +``` + +Each channel and timeframe of a multi-channel or time-series volume is featurized as a volume of its own, and the resulting rows carry `c_index` and `t_index` columns identifying which one they came from. A single-channel mask is reused across all intensity channels. + +Note on physical voxel spacing: OME-TIFF and OME-Zarr files may declare a physical size per axis. Passing `use_physical_spacing=True` makes Nyxus use each volume's declared spacing as its voxel anisotropy, ratio-normalized so that the smallest axis is 1: + +```python +nyx = nyxus.Nyxus3D (["*3D_ALL*"], use_physical_spacing=True) +``` + +It is off by default, so the default cubic-voxel behavior is unchanged, and explicitly specified `anisotropy_x/y/z` values take precedence over the declared spacing. The spacing actually applied is reported in the `phys_x`, `phys_y`, `phys_z`, and `phys_unit` output columns, canonicalized to micrometers so that files declaring their sizes in different units stay numerically comparable. + Featurizing explicitly specified volume files is straightforward, too: ```python @@ -817,6 +836,12 @@ Applying a 1.5 x 2.0 x 2.5 anisotropy correction to all the volumes in a compres --anisox=1.5 --anisoy=2 --filePattern="*\.ome\.tiff" --features=*ALL* --resultFname=features1 --outputType=singlecsv --intDir=/data/plate123/int --segDir=/data/plate123/seg --outDir=/output/plate123 ``` +Instead of stating the anisotropy explicitly, it can be derived from the physical voxel spacing that OME-TIFF and OME-Zarr volumes declare in their own metadata: + +``` +--use-physical-spacing=true --dim=3 --filePattern=".*\.ome\.zarr" --features=*3D_ALL* --resultFname=3d-features --outputType=singlecsv --intDir=/data/patient123/intensity --segDir=/data/patient123/masks --outDir=/output/patient123 +``` + ## Nested ROIs Hierarchical ROI analysis in a form of finding ROIs nested geometrically as nested AABBs and aggregating features of child ROIs within corresponding parent is available as an optional extra step after the feature extraction of the whole image set is finished. To enable this step, all the command line options '--hsig', '--hpar', '--hchi', and '--hag' need to have non-blank valid values. diff --git a/src/nyx/abs_tile_loader.h b/src/nyx/abs_tile_loader.h index c4f509c6..aa6544b2 100644 --- a/src/nyx/abs_tile_loader.h +++ b/src/nyx/abs_tile_loader.h @@ -32,12 +32,16 @@ class AbstractTileLoader /// @param tile Tile to load /// @param indexRowGlobalTile Tile's row index in the file to load /// @param indexColGlobalTile Tile's col index in the file to load - /// @param indexLayerGlobalTile Tile's layer index in the file to load + /// @param indexLayerGlobalTile Tile's layer (Z) index in the file to load + /// @param indexChannel Channel (C) index to load; 0 for single-channel data + /// @param indexTimeframe Time (T) index to load; 0 for non-time-series data /// @param level Tile's pyramidal level in the file to load virtual void loadTileFromFile(std::shared_ptr> tile, size_t indexRowGlobalTile, size_t indexColGlobalTile, size_t indexLayerGlobalTile, + size_t indexChannel, + size_t indexTimeframe, size_t level) = 0; /// \brief Getter to full Height @@ -70,6 +74,14 @@ class AbstractTileLoader return 1; } + /// \brief Physical voxel spacing along X/Y/Z (default 1.0 == uncalibrated). + /// OME loaders return the parsed PhysicalSize*; everything else stays 1.0. + [[nodiscard]] virtual double physicalSizeX() const { return 1.0; } + [[nodiscard]] virtual double physicalSizeY() const { return 1.0; } + [[nodiscard]] virtual double physicalSizeZ() const { return 1.0; } + /// \brief Physical-size unit string (e.g. "micrometer"); empty if unknown. + [[nodiscard]] virtual std::string physicalSizeUnit() const { return std::string(); } + /// \brief Getter to tile Width /// @param level tile's level considered /// \return Tile Width diff --git a/src/nyx/cli_option_constants.h b/src/nyx/cli_option_constants.h index 266e9875..da2bc546 100644 --- a/src/nyx/cli_option_constants.h +++ b/src/nyx/cli_option_constants.h @@ -69,6 +69,7 @@ #define clo_AGGREGATE "--aggr" // -> raw_aggregate #define clo_ANNOTATE "--annot" // -> raw_annotate #define clo_ANNOT_SEP "--annotsep" // -> raw_anno_separator +#define clo_USE_PHYSICAL_SPACING "--use-physical-spacing" // -> rawUsePhysicalSpacing (true/false); opt-in OME PhysicalSize* voxel spacing // Valid values of 'OUTPUTTYPE' #define clo_OT_SEPCSV "separatecsv" diff --git a/src/nyx/constants.h b/src/nyx/constants.h index f833602f..91fc0e3c 100644 --- a/src/nyx/constants.h +++ b/src/nyx/constants.h @@ -1,4 +1,5 @@ #pragma once constexpr double DEFAULT_T_INDEX = 0.0; +constexpr double DEFAULT_C_INDEX = 0.0; // FIX (IO): default channel index for single-channel / non-OME input constexpr int DEFAULT_NUM_HISTO_BINS = 24; diff --git a/src/nyx/environment.cpp b/src/nyx/environment.cpp index f6f419da..40c4631b 100644 --- a/src/nyx/environment.cpp +++ b/src/nyx/environment.cpp @@ -473,6 +473,7 @@ bool Environment::parse_cmdline(int argc, char** argv) || find_string_argument(i, clo_FPIMAGE_MIN, fpimageOptions.raw_min_intensity) || find_string_argument(i, clo_FPIMAGE_MAX, fpimageOptions.raw_max_intensity) || find_string_argument(i, clo_PRESERVE_HU, fpimageOptions.raw_preserve_hu) // CT/HU mode + || find_string_argument(i, clo_USE_PHYSICAL_SPACING, rawUsePhysicalSpacing) // FIX (IO): opt-in physical voxel spacing || find_string_argument(i, clo_RESULTFNAME, nyxus_result_fname) || find_string_argument(i, clo_CLI_DIM, raw_dim) @@ -799,6 +800,14 @@ bool Environment::parse_cmdline(int argc, char** argv) } } + //==== Parse opt-in physical-spacing flag (FIX (IO); default false) + if (!rawUsePhysicalSpacing.empty()) + { + std::string v = rawUsePhysicalSpacing; + std::transform(v.begin(), v.end(), v.begin(), ::tolower); + use_physical_spacing_ = (v == "true" || v == "t" || v == "1" || v == "on" || v == "yes"); + } + //==== Parse exclusive-inclusive timing #ifdef CHECKTIMING if (!rawExclusiveTiming.empty()) diff --git a/src/nyx/environment.h b/src/nyx/environment.h index a07e3352..d65a7625 100644 --- a/src/nyx/environment.h +++ b/src/nyx/environment.h @@ -2,6 +2,7 @@ #include #include +#include // FIX: csv_paths_written registry (see below) #include #include "arrow_output_stream.h" @@ -80,6 +81,18 @@ class Environment: public BasicEnvironment std::string rawOutpType; //= ""; // Valid values: "separatecsv", "singlecsv", "arrow", "parquet" bool separateCsv; //= true; + // FIX: CSV output paths already written during this run. A slide now yields one row per + // (channel, timeframe), and the CSV sinks are called once per plane -- but in separatecsv + // mode they opened the same path with "w" every time, so each plane truncated the previous + // and only the LAST one survived. The first write to a path truncates and renders the + // header; later writes append. Held per-Environment rather than in a function-static so a + // long-lived Python session starts each run clean instead of appending to a stale file. + std::unordered_set csv_paths_written; + + // True the first time `path` is written in this run (and records it). + bool csv_claim_first_write (const std::string& path) { return csv_paths_written.insert(path).second; } + void reset_csv_output_state() { csv_paths_written.clear(); } + Nyxus::SaveOption saveOption; // x- and y- resolution in pixels per centimeter @@ -87,6 +100,14 @@ class Environment: public BasicEnvironment float xyRes; //= 0.0, float pixelSizeUm; //= 0.0; + // FIX (IO): opt-in physical-voxel-spacing calibration. When set, the 3D pipeline uses + // each slide's OME PhysicalSize* (ratio-normalized) as voxel spacing (anisotropy). + // Off by default -> today's cube-voxel behavior is unchanged. Spacing is still emitted + // as phys_x/y/z + phys_unit output columns regardless of this flag. + std::string rawUsePhysicalSpacing; //= ""; + bool use_physical_spacing_ = false; + bool use_physical_spacing() const { return use_physical_spacing_; } + int get_pixel_distance(); void set_pixel_distance(int pixelDistance); size_t get_ram_limit(); diff --git a/src/nyx/feature_method.cpp b/src/nyx/feature_method.cpp index c5740309..0fb2d580 100644 --- a/src/nyx/feature_method.cpp +++ b/src/nyx/feature_method.cpp @@ -43,7 +43,10 @@ void FeatureMethod::add_dependencies(const std::initializer_listosized_calculate (r, s, ldr); this->save_value (r.fvals); diff --git a/src/nyx/feature_method.h b/src/nyx/feature_method.h index 16aac5b9..88c8a28b 100644 --- a/src/nyx/feature_method.h +++ b/src/nyx/feature_method.h @@ -5,6 +5,8 @@ #include "image_loader.h" #include "features/image_matrix_nontriv.h" +class Dataset; // forward decl for the Dataset-aware osized_scan_whole_image overload + /// @brief Abstract class encapsulating basic feature functionality e.g. dependency on other features, dependency on helper objects, state (calculated or pending), etc. class FeatureMethod { @@ -25,7 +27,10 @@ class FeatureMethod virtual void parallel_process(std::vector& roi_labels, std::unordered_map & roiData, int n_threads) {} //=== Oversized ROI - virtual void osized_scan_whole_image (LR& roi, const Fsettings& s, ImageLoader& ldr); + // Out-of-core dispatch. Features that need slide props (intensity, intensity-histogram) + // override this to reach their Dataset-taking osized_calculate; the default ignores the + // Dataset and calls the Dataset-less osized_calculate, so every other feature is unaffected. + virtual void osized_scan_whole_image (LR& roi, const Fsettings& s, const Dataset& ds, ImageLoader& ldr); virtual void osized_add_online_pixel (size_t x, size_t y, uint32_t intensity) = 0; // Called each time the ROI pixel is being scanned in the raster order virtual void osized_reduce() final {}; // Get rid of this method in all derived diff --git a/src/nyx/features/3d_glcm.cpp b/src/nyx/features/3d_glcm.cpp index 08ebb27f..b8f8912b 100644 --- a/src/nyx/features/3d_glcm.cpp +++ b/src/nyx/features/3d_glcm.cpp @@ -179,7 +179,14 @@ void D3_GLCM_feature::save_value(std::vector>& fvals) void D3_GLCM_feature::extract_texture_features_at_angle(int dx, int dy, int dz, const SimpleCube& binned_greys, PixIntens min_val, PixIntens max_val, int n_greys, bool ibsi, double soft_nan) { calculateCoocMatAtAngle (P_matrix, dx, dy, dz, binned_greys, min_val, max_val, n_greys, ibsi); + finalize_angle (soft_nan); +} +// Computes this angle's GLCM feature values from the already-built P_matrix + sum_p + I. Split out +// of extract_texture_features_at_angle so the out-of-core path (which builds P_matrix by streaming a +// 2-plane window instead of scanning the in-RAM cube) reuses the exact same feature math. +void D3_GLCM_feature::finalize_angle (double soft_nan) +{ // Blank cooc-matrix? -- no point to use it, assign each feature value '0' and return. if (sum_p == 0) { diff --git a/src/nyx/features/3d_glcm.h b/src/nyx/features/3d_glcm.h index 82c7290e..980b938d 100644 --- a/src/nyx/features/3d_glcm.h +++ b/src/nyx/features/3d_glcm.h @@ -145,6 +145,8 @@ class D3_GLCM_feature : public FeatureMethod, public TextureFeature bool normalize); void extract_texture_features_at_angle (int dx, int dy, int dz, const SimpleCube & grays, PixIntens min_val, PixIntens max_val, int n_greys, bool ibsi, double soft_nan); + // Compute this angle's features from the already-built P_matrix (shared by in-core + out-of-core) + void finalize_angle (double soft_nan); void calculateCoocMatAtAngle( // out diff --git a/src/nyx/features/3d_glcm_nontriv.cpp b/src/nyx/features/3d_glcm_nontriv.cpp index a0fb45fb..8fed07a4 100644 --- a/src/nyx/features/3d_glcm_nontriv.cpp +++ b/src/nyx/features/3d_glcm_nontriv.cpp @@ -1,11 +1,216 @@ -#include "../environment.h" -#include "3d_glcm.h" -#include "image_matrix_nontriv.h" - -using namespace Nyxus; - -void D3_GLCM_feature::osized_calculate (LR& r, const Fsettings& s, ImageLoader&) -{ - calculate (r, s); -} - +#include +#include +#include +#include "../environment.h" +#include "3d_glcm.h" +#include "image_matrix_nontriv.h" + +using namespace Nyxus; + +// The 13 direction shifts, identical to the in-core path (3d_glcm.cpp) +namespace { + struct Sh { int dx, dy, dz; }; + const Sh SHIFTS[13] = { + {1, 1, 1}, {1, 1, 0}, {1, 1, -1}, {1, 0, 1}, {1, 0, 0}, {1, 0, -1}, + {1, -1, 1}, {1, -1, 0}, {1, -1, -1}, {0, 1, 1}, {0, 1, 0}, {0, 1, -1}, {0, 0, 1} + }; +} + +// Out-of-core 3D GLCM. Instead of scanning the in-RAM grey-binned cube, this builds the 13 +// co-occurrence matrices by streaming the disk-backed voxel cloud one Z-plane at a time through a +// sliding window of (offset+1) dense grey-binned planes (co-occurrence counts are additive and the +// 13 directions only reach +/- offset in Z). The per-direction feature math is then the shared +// finalize_angle(), so the values are identical to calculate(). +void D3_GLCM_feature::osized_calculate (LR& r, const Fsettings& s, ImageLoader&) +{ + clear_result_buffers(); + + const size_t nvox = r.raw_voxels_NT.size(); + + // grey-binning mode, mirroring calculate(): IBSI forces the "no binning" (==0) mode + int greyInfo = STNGS_IBSI(s) ? 0 : STNGS_GLCM_GREYDEPTH(s); + bool ibsi = STNGS_IBSI(s); + double soft_nan = STNGS_NAN(s); + PixIntens mn = r.aux_min, mx = r.aux_max; + + const int W = (int) r.aabb.get_width(), + H = (int) r.aabb.get_height(), + Dz = (int) r.aabb.get_z_depth(); + const StatsInt xmin = r.aabb.get_xmin(), + ymin = r.aabb.get_ymin(), + zmin = r.aabb.get_zmin(); + + // Background cells of the in-core cube are grey-binned too (0 -> 1 for matlab, 0 otherwise); + // reproduce that so a dense plane matches a Z-slice of the binned cube exactly. + const PixIntens bg = TextureFeature::bin_pixel (0, mn, mx, greyInfo); + + // --- pass 1: the global set of grey levels (I) + matrix dimension, as in calculateCoocMatAtAngle, + // which builds I from the unique values of the WHOLE binned cube (mask + background) for the + // radiomics branch -- so a background bin (e.g. matlab binning maps raw-0 to bin 1) must be + // included too when the ROI bbox actually has background voxels. + const bool hasBackground = nvox < (size_t) r.aabb.get_width() * r.aabb.get_height() * r.aabb.get_z_depth(); + std::set uniq; + PixIntens maxbin = 0; + if (hasBackground && bg != 0) + { + uniq.insert (bg); + maxbin = bg; + } + for (size_t i = 0; i < nvox; i++) + { + Pixel3 v = r.raw_voxels_NT[i]; + PixIntens b = TextureFeature::bin_pixel (v.inten, mn, mx, greyInfo); + if (b > maxbin) maxbin = b; + if (b != 0) uniq.insert (b); + } + + I.clear(); + if (radiomics_grey_binning(greyInfo)) + I.assign (uniq.begin(), uniq.end()); // std::set is already sorted ascending + else if (matlab_grey_binning(greyInfo)) + { + int n = greyInfo; + I.resize (n); + for (int i = 0; i < n; i++) I[i] = i + 1; + } + else + { + int n = (int) maxbin; // IBSI: levels 1..max + I.resize (n); + for (int i = 0; i < n; i++) I[i] = i + 1; + } + const int Ng = (int) I.size(); + + // Nothing to featurize (empty/blank): emit 13 blank angles, matching the in-core path's + // per-direction soft-NaN pushes so the angled vectors and their averages line up. + if (Ng == 0 || nvox == 0) + { + P_matrix.allocate (1, 1); + std::fill (P_matrix.begin(), P_matrix.end(), 0.0); + for (int k = 0; k < 13; k++) + { + sum_p = 0; + finalize_angle (soft_nan); + } + return; + } + + // --- pass 2: accumulate the 13 co-occurrence matrices over a sliding Z window + std::vector> mats (13); + for (int k = 0; k < 13; k++) + { + mats[k].allocate (Ng, Ng); + std::fill (mats[k].begin(), mats[k].end(), 0.0); + } + + const int off = D3_GLCM_feature::offset; + const bool sym = D3_GLCM_feature::symmetric_glcm; + + // O(1) grey-level -> matrix row, replacing the per-pair binary search in the hot add_pair loop + // (radiomics is the only binning branch that indexes I by value; the others use pi-1 directly). + // rowLUT[v] == lower_bound(I, v) - I.begin() for every possible binned level, so the row is + // identical to the search it replaces. + const bool radiomics = radiomics_grey_binning(greyInfo); + std::vector rowLUT; + if (radiomics) + { + rowLUT.assign ((size_t) maxbin + 1, 0); + for (PixIntens v = 0; v <= maxbin; v++) + rowLUT[v] = (int) (std::lower_bound (I.begin(), I.end(), v) - I.begin()); + } + + // Count one (base, neighbor) grey-level pair into matrix M, mirroring calculateCoocMatAtAngle: + // GLCM.xy(idx(neighbor), idx(base)), plus the symmetric transpose for radiomics/ibsi/symmetric. + auto add_pair = [&](SimpleMatrix& M, PixIntens lvl_base, PixIntens lvl_nbr) + { + if (ibsi_grey_binning(greyInfo)) + if (lvl_nbr == 0 || lvl_base == 0) + return; + int a = (int) lvl_nbr, b = (int) lvl_base; + if (radiomics) + { + if (a == 0 || b == 0) + return; + a = rowLUT[a]; + b = rowLUT[b]; + } + else { a -= 1; b -= 1; } + M.xy (a, b) += 1.0; + if (sym || radiomics || ibsi_grey_binning(greyInfo)) + M.xy (b, a) += 1.0; + }; + + // Ring of (off+1) dense grey-binned planes, indexed by local z modulo (off+1) + const int ring = off + 1; + std::vector> planes (ring); + std::vector slab; + + for (int lz = 0; lz < Dz; lz++) + { + std::vector& cur = planes[lz % ring]; + cur.assign ((size_t) W * H, bg); + r.raw_voxels_NT.read_slab ((size_t)(zmin + lz), slab); + for (const auto& v : slab) + { + int lx = (int) v.x - (int) xmin, ly = (int) v.y - (int) ymin; + if (lx >= 0 && lx < W && ly >= 0 && ly < H) + cur[(size_t) ly * W + lx] = TextureFeature::bin_pixel (v.inten, mn, mx, greyInfo); + } + + const std::vector* farp = (lz >= off) ? &planes[(lz - off) % ring] : nullptr; + + for (int k = 0; k < 13; k++) + { + const int dx = SHIFTS[k].dx * off, dy = SHIFTS[k].dy * off, dz = SHIFTS[k].dz * off; + SimpleMatrix& M = mats[k]; + + if (dz == 0) + { + // base and neighbor both on the current plane + for (int y = 0; y < H; y++) + for (int x = 0; x < W; x++) + { + int nx = x + dx, ny = y + dy; + if (nx >= 0 && nx < W && ny >= 0 && ny < H) + add_pair (M, cur[(size_t) y * W + x], cur[(size_t) ny * W + nx]); + } + } + else if (dz < 0) + { + // base on current plane, neighbor 'off' planes below (the far plane in the ring) + if (!farp) continue; + const std::vector& far = *farp; + for (int y = 0; y < H; y++) + for (int x = 0; x < W; x++) + { + int nx = x + dx, ny = y + dy; + if (nx >= 0 && nx < W && ny >= 0 && ny < H) + add_pair (M, cur[(size_t) y * W + x], far[(size_t) ny * W + nx]); + } + } + else + { + // dz>0: base on the far plane (local z-off), neighbor on the current plane + if (!farp) continue; + const std::vector& far = *farp; + for (int y = 0; y < H; y++) + for (int x = 0; x < W; x++) + { + int nx = x + dx, ny = y + dy; + if (nx >= 0 && nx < W && ny >= 0 && ny < H) + add_pair (M, far[(size_t) y * W + x], cur[(size_t) ny * W + nx]); + } + } + } + } + + // --- per-direction feature values from the accumulated matrices (shared finalize). + // Move each matrix into P_matrix rather than copy it -- mats[k] is not reused afterwards. + for (int k = 0; k < 13; k++) + { + P_matrix = std::move (mats[k]); + sum_p = 0; + for (double a : P_matrix) sum_p += a; + finalize_angle (soft_nan); + } +} diff --git a/src/nyx/features/3d_gldm.cpp b/src/nyx/features/3d_gldm.cpp index 663b2d4f..bdcf56bf 100644 --- a/src/nyx/features/3d_gldm.cpp +++ b/src/nyx/features/3d_gldm.cpp @@ -1,8 +1,10 @@ #include #include #include +#include #include #include +#include #include "3d_gldm.h" #include "../environment.h" @@ -246,7 +248,152 @@ void D3_GLDM_feature::osized_add_online_pixel(size_t x, size_t y, uint32_t inten void D3_GLDM_feature::osized_calculate(LR& r, const Fsettings& s, ImageLoader&) { - calculate (r, s); + // Out-of-core GLDM. Streams the disk-backed voxel cloud through a 3-plane sliding window of + // dense grey-binned planes (the 26-neighbourhood spans z-1..z+1) and fills the dependence + // matrix P directly per voxel (no O(volume) zone list). Feature math (calc_*) is shared with + // the in-core path, so values are identical. + clear_buffers(); + + // intercept blank ROIs (same guard as calculate()) + if (r.aux_min == r.aux_max) + { + fv_SDE = fv_LDE = fv_GLN = fv_DN = fv_DNN = fv_GLV = fv_DV = fv_DE = + fv_LGLE = fv_HGLE = fv_SDLGLE = fv_SDHGLE = fv_LDLGLE = fv_LDHGLE = STNGS_NAN(s); + return; + } + + int greyInfo = STNGS_IBSI(s) ? 0 : STNGS_GLDM_GREYDEPTH(s); + PixIntens mn = r.aux_min, mx = r.aux_max; + const PixIntens bg = TextureFeature::bin_pixel (0, mn, mx, greyInfo); + const PixIntens zeroI = matlab_grey_binning(greyInfo) ? 1 : 0; + + const int W = (int) r.aabb.get_width(), + H = (int) r.aabb.get_height(), + Dz = (int) r.aabb.get_z_depth(); + const StatsInt xmin = r.aabb.get_xmin(), ymin = r.aabb.get_ymin(), zmin = r.aabb.get_zmin(); + + // --- grey levels I + matrix dimension (mirrors calculate(), which builds I from the WHOLE + // binned cube incl. background -- e.g. matlab binning maps raw-0 background to bin 1, and that + // bin must appear in I too, matching D3_GLDM_feature::calculate()'s unordered_set(D.begin(),D.end())). + std::set uniq; + PixIntens maxbin = 0; + const size_t nvox = r.raw_voxels_NT.size(); + const bool hasBackground = nvox < (size_t) W * H * Dz; + if (hasBackground && bg != 0) + { + uniq.insert (bg); + maxbin = bg; + } + for (size_t i = 0; i < nvox; i++) + { + PixIntens b = TextureFeature::bin_pixel (r.raw_voxels_NT[i].inten, mn, mx, greyInfo); + if (b > maxbin) maxbin = b; + if (b != 0) uniq.insert (b); + } + I.clear(); + if (ibsi_grey_binning(greyInfo)) + { + int n = (int) maxbin; + I.resize (n); + for (int i = 0; i < n; i++) I[i] = i + 1; + } + else + I.assign (uniq.begin(), uniq.end()); // std::set already sorted + Ng = (greyInfo == 0) ? (I.empty() ? 0 : (int) *std::max_element(I.begin(), I.end())) : (int) I.size(); + + Nd = nsh + 1; + P.allocate (Nd + 1, Ng + 1); + std::fill (P.begin(), P.end(), 0); + int max_Nd = 0; + + // O(1) grey-level -> matrix row, replacing the per-voxel binary search in the scan below. + // rowLUT[v] == lower_bound(I, v) - I.begin() for every possible binned level, so the row is + // identical to the search it replaces (used only by the non-IBSI branch; IBSI uses pi-1). + std::vector rowLUT ((size_t) maxbin + 1, 0); + for (PixIntens v = 0; v <= maxbin; v++) + rowLUT[v] = (int) (std::lower_bound (I.begin(), I.end(), v) - I.begin()); + + // --- 3-plane sliding window of dense grey-binned planes (bg-initialised so background matches + // the in-core cube), indexed by local z modulo 3 + std::vector> ring (3); + int ringZ[3] = { -1, -1, -1 }; + std::vector slab; + auto load = [&](int lz) + { + if (lz < 0 || lz >= Dz) return; + int slot = lz % 3; + if (ringZ[slot] == lz) return; + std::vector& pl = ring[slot]; + pl.assign ((size_t) W * H, bg); + r.raw_voxels_NT.read_slab ((size_t)(zmin + lz), slab); + for (const auto& v : slab) + { + int lx = (int) v.x - (int) xmin, ly = (int) v.y - (int) ymin; + if (lx >= 0 && lx < W && ly >= 0 && ly < H) + pl[(size_t) ly * W + lx] = TextureFeature::bin_pixel (v.inten, mn, mx, greyInfo); + } + ringZ[slot] = lz; + }; + + const bool ibsi = STNGS_IBSI(s); + for (int c = 0; c < Dz; c++) + { + load (c - 1); load (c); load (c + 1); + const std::vector& cur = ring[c % 3]; + + for (int y = 0; y < H; y++) + for (int x = 0; x < W; x++) + { + PixIntens pi = cur[(size_t) y * W + x]; + if (pi == zeroI) + continue; + + int nd = 1; + for (int i = 0; i < nsh; i++) + { + int nz = c + shifts[i].dz, ny = y + shifts[i].dy, nx = x + shifts[i].dx; + if (nz < 0 || nz >= Dz || ny < 0 || ny >= H || nx < 0 || nx >= W) + continue; + if (pi == ring[nz % 3][(size_t) ny * W + nx]) + nd++; + } + + int row = ibsi ? (int) pi - 1 : rowLUT[pi]; + int col = nd - 1; + P.xy (col, row)++; + max_Nd = (std::max) (max_Nd, nd); + } + } + + if (greyInfo) + Nd = max_Nd; + + Nz = 0; + for (auto p : P) + Nz += p; + + if (Nz == 0) + { + fv_SDE = fv_LDE = fv_GLN = fv_DN = fv_DNN = fv_GLV = fv_DV = fv_DE = + fv_LGLE = fv_HGLE = fv_SDLGLE = fv_SDHGLE = fv_LDLGLE = fv_LDHGLE = STNGS_NAN(s); + } + else + { + fv_SDE = calc_SDE(); + fv_LDE = calc_LDE(); + fv_GLN = calc_GLN(); + fv_DN = calc_DN(); + fv_DNN = calc_DNN(); + fv_GLV = calc_GLV(); + fv_DV = calc_DV(); + fv_DE = calc_DE(); + fv_LGLE = calc_LGLE(); + fv_HGLE = calc_HGLE(); + fv_SDLGLE = calc_SDLGLE(); + fv_SDHGLE = calc_SDHGLE(); + fv_LDLGLE = calc_LDLGLE(); + fv_LDHGLE = calc_LDHGLE(); + } } void D3_GLDM_feature::save_value(std::vector>& fvals) diff --git a/src/nyx/features/3d_gldzm.cpp b/src/nyx/features/3d_gldzm.cpp index 6fda0553..34b092a4 100644 --- a/src/nyx/features/3d_gldzm.cpp +++ b/src/nyx/features/3d_gldzm.cpp @@ -1,8 +1,13 @@ +#include #include +#include #include +#include +#include #include "../environment.h" #include "3d_gldzm.h" #include "image_cube.h" +#include "streaming_ccl.h" int D3_GLDZM_feature::n_levels = 0; @@ -571,7 +576,187 @@ void D3_GLDZM_feature::save_value (std::vector> & fvals) void D3_GLDZM_feature::osized_add_online_pixel (size_t x, size_t y, uint32_t intensity) {} +// Out-of-core 3D GLDZM. prepare_GLDZM_matrix_kit()'s flood fill is 6-connectivity (face-adjacent +// only -- East/West/North/South/Up/Down, no diagonals) tracking each zone's MINIMUM per-voxel +// distance to the ROI border (zone SIZE is also tracked in-core but never actually consumed by +// calc_gldzm_matrix, so this streaming version doesn't bother). Same streaming two-pass CCL +// approach as GLSZM (docs/13-3d-ooc-glszm-gldzm-design.md), but with only 3 causal neighbours +// (West, North, straight-down-Z) since 6-connectivity halves to 3, and a MIN-combining +// LabelUnionFind instead of a size-summing one. +// +// dist2border() is 2D-only (scans left/right/up/down within the CURRENT Z-plane only to the +// nearest literal-0 voxel or the plane edge) -- a limitation/convention of this implementation, +// replicated exactly, not "fixed". It needs only the current plane's dense buffer, already +// resident in the 2-plane window, so no extra memory cost versus GLSZM. +// +// Background-skip convention is GLDZM's OWN (a THIRD distinct rule, different from both GLRLM's +// unconditional zeroI=0 and GLSZM's matlab-aware zeroI): skip a voxel ONLY when +// ibsi_grey_binning(greyInfo) is true AND its binned value is literal 0 -- for radiomics/matlab +// binning, background is never skipped and forms its own zone(s), exactly mirroring +// prepare_GLDZM_matrix_kit's `if (ibsi_grey_binning(greyInfo)) if (inten==0) continue;`. void D3_GLDZM_feature::osized_calculate (LR& r, const Fsettings& s, ImageLoader&) { - calculate (r, s); + clear_buffers(); + + if (r.aux_min == r.aux_max) + { + f_SDE = f_LDE = f_LGLZE = f_HGLZE = f_SDLGLE = f_SDHGLE = f_LDLGLE = f_LDHGLE = + f_GLNU = f_GLNUN = f_ZDNU = f_ZDNUN = f_ZP = f_GLM = f_GLV = f_ZDM = f_ZDV = f_ZDE = f_GLE = STNGS_NAN(s); + return; + } + + // grey-binning selection mirrors prepare_GLDZM_matrix_kit exactly: no GLDZM-specific greydepth + // setting exists, so it starts from the general STNGS_NGREYS(s), can be overridden by the static + // D3_GLDZM_feature::n_levels, then forced to 0 (IBSI/no-rescale) when STNGS_IBSI(s) is set. + auto greyInfo = STNGS_NGREYS(s); + auto greyInfo_localFeature = D3_GLDZM_feature::n_levels; + if (greyInfo_localFeature != 0 && greyInfo != greyInfo_localFeature) + greyInfo = greyInfo_localFeature; + if (STNGS_IBSI(s)) + greyInfo = 0; + + PixIntens mn = r.aux_min, mx = r.aux_max; + const PixIntens bg = TextureFeature::bin_pixel (0, mn, mx, greyInfo); + const bool skipZero = ibsi_grey_binning (greyInfo); // GLDZM's own rule: only skip in ibsi mode + + const int W = (int) r.aabb.get_width(), + H = (int) r.aabb.get_height(), + Dz = (int) r.aabb.get_z_depth(); + const StatsInt xmin = r.aabb.get_xmin(), ymin = r.aabb.get_ymin(), zmin = r.aabb.get_zmin(); + const size_t nvox = r.raw_voxels_NT.size(); + const bool hasBackground = nvox < (size_t) W * H * Dz; + + // --- grey levels (mirrors prepare_GLDZM_matrix_kit: unique values of the WHOLE binned cube, + // incl. background, minus literal 0 -- or a dense 1..max linspace for ibsi_grey_binning) + std::vector I; + if (ibsi_grey_binning (greyInfo)) + { + PixIntens maxbin = 0; + for (size_t i = 0; i < nvox; i++) + { + PixIntens b = TextureFeature::bin_pixel (r.raw_voxels_NT[i].inten, mn, mx, greyInfo); + if (b > maxbin) maxbin = b; + } + I.resize (maxbin); + for (int i = 0; i < (int) maxbin; i++) I[i] = i + 1; + } + else + { + std::set uniq; + if (hasBackground && bg != 0) + uniq.insert (bg); + for (size_t i = 0; i < nvox; i++) + { + PixIntens b = TextureFeature::bin_pixel (r.raw_voxels_NT[i].inten, mn, mx, greyInfo); + if (b != 0) uniq.insert (b); + } + I.assign (uniq.begin(), uniq.end()); + } + + // --- streaming 6-connectivity CCL over a 2-plane window, tracking min distance-to-border + LabelUnionFind uf; + std::vector prevPlane, curPlane; + std::vector prevLabel, curLabel; + std::vector slab; + + // 2D-only distance to the nearest literal-0 voxel or the plane edge, within 'plane' -- matches + // dist2border() exactly (1-based, min of 4 directions, clamped to >=1). + auto dist2border_plane = [&](const std::vector& plane, int x, int y) -> long long + { + int dist2l = 0; + for (int x0 = x - 1; x0 >= 0; x0--) + if (plane[(size_t) y * W + x0] == 0 || x0 == 0) { dist2l = x - x0; break; } + int dist2r = 0; + for (int x0 = x + 1; x0 < W; x0++) + if (plane[(size_t) y * W + x0] == 0 || x0 == W - 1) { dist2r = x0 - x; break; } + int dist2t = 0; + for (int y0 = y - 1; y0 >= 0; y0--) + if (plane[(size_t) y0 * W + x] == 0 || y0 == 0) { dist2t = y - y0; break; } + int dist2b = 0; + for (int y0 = y + 1; y0 < H; y0++) + if (plane[(size_t) y0 * W + x] == 0 || y0 == H - 1) { dist2b = y0 - y; break; } + dist2l++; dist2r++; dist2t++; dist2b++; + long long d = (std::min) ((std::min) (dist2l, dist2r), (std::min) (dist2t, dist2b)); + if (d == 0) d = 1; + return d; + }; + + for (int z = 0; z < Dz; z++) + { + curPlane.assign ((size_t) W * H, bg); + r.raw_voxels_NT.read_slab ((size_t)(zmin + z), slab); + for (const auto& v : slab) + { + int lx = (int) v.x - (int) xmin, ly = (int) v.y - (int) ymin; + if (lx >= 0 && lx < W && ly >= 0 && ly < H) + curPlane[(size_t) ly * W + lx] = TextureFeature::bin_pixel (v.inten, mn, mx, greyInfo); + } + curLabel.assign ((size_t) W * H, -1); + + for (int y = 0; y < H; y++) + for (int x = 0; x < W; x++) + { + PixIntens pi = curPlane[(size_t) y * W + x]; + if (skipZero && pi == 0) + continue; + + long long d = dist2border_plane (curPlane, x, y); + + int found = -1; + if (x > 0 && curPlane[(size_t) y * W + (x - 1)] == pi) + { + int lbl = curLabel[(size_t) y * W + (x - 1)]; + if (lbl >= 0) found = lbl; + } + if (y > 0 && curPlane[(size_t) (y - 1) * W + x] == pi) + { + int lbl = curLabel[(size_t) (y - 1) * W + x]; + if (lbl >= 0) found = (found < 0) ? lbl : uf.union_min (found, lbl); + } + if (z > 0 && prevPlane[(size_t) y * W + x] == pi) + { + int lbl = prevLabel[(size_t) y * W + x]; + if (lbl >= 0) found = (found < 0) ? lbl : uf.union_min (found, lbl); + } + + if (found < 0) + found = uf.make_label (pi, d); + else + uf.update_min (found, d); + curLabel[(size_t) y * W + x] = found; + } + + prevPlane.swap (curPlane); + prevLabel.swap (curLabel); + } + + // --- one zone per distinct root: (intensity, min distance) + std::vector> Zones; + std::vector seenRoot (uf.num_labels(), 0); + for (size_t i = 0; i < uf.num_labels(); i++) + { + int rt = uf.find ((int) i); + if (seenRoot[rt]) continue; + seenRoot[rt] = 1; + Zones.push_back ({ uf.intensity_of (rt), (int) uf.metric (rt) }); + } + + int Ng = (int) I.size(); + int Nd = 0; + for (auto& zo : Zones) + Nd = (std::max) (Nd, zo.second); + + SimpleMatrix GLDZM; + GLDZM.allocate (Nd, Ng); // Ng rows, Nd columns -- matches calc_gldzm_matrix's layout + GLDZM.fill (0); + for (auto& zo : Zones) + { + int row = (int)(std::lower_bound (I.begin(), I.end(), zo.first) - I.begin()); + int col = zo.second - 1; + GLDZM.yx (row, col)++; + } + + std::vector Mx, Md; + calc_row_and_column_sum_vectors (Mx, Md, GLDZM, Ng, Nd, I); + calc_features (Mx, Md, GLDZM, I, r.aux_area); } diff --git a/src/nyx/features/3d_glrlm_nontriv.cpp b/src/nyx/features/3d_glrlm_nontriv.cpp index b4191d05..bc5e48a0 100644 --- a/src/nyx/features/3d_glrlm_nontriv.cpp +++ b/src/nyx/features/3d_glrlm_nontriv.cpp @@ -1,11 +1,242 @@ -#include "../environment.h" -#include "3d_glrlm.h" -#include "image_matrix_nontriv.h" - -using namespace Nyxus; - -void D3_GLRLM_feature::osized_calculate (LR& r, const Fsettings& s, ImageLoader&) -{ - calculate (r, s); -} - +#include +#include +#include +#include +#include "../environment.h" +#include "3d_glrlm.h" +#include "image_matrix_nontriv.h" + +using namespace Nyxus; + +// Same 13 canonical direction shifts as 3d_glrlm.cpp (dz in {0,1} only: half of the 26-connected +// directions, avoiding double-counting an axis and its opposite). AngleShift order is {dz,dy,dx}. +static const AngleShift SHIFTS13[] = +{ + {1, 1, 1}, {1, 1, 0}, {1, 1, -1}, {1, 0, 1}, {1, 0, 0}, {1, 0, -1}, + {1, -1, 1}, {1, -1, 0}, {1, -1, -1}, {0, 1, 1}, {0, 1, 0}, {0, 1, -1}, {0, 0, 1} +}; + +// Out-of-core 3D GLRLM. gather_rl_zones() greedily walks each run forward along its direction, +// consuming voxels via an in-RAM VISITED marker -- workable in-core (whole cube available) but not +// as-is out-of-core. Two cases, driven by the fact all 13 canonical directions have dz in {0,1}: +// +// - dz==0 (3 directions): the run never leaves its Z-plane, so gather_rl_zones() is called +// UNCHANGED on a depth-1 SimpleCube built from one streamed dense plane -- byte-identical to the +// in-core algorithm, just fed one plane at a time. +// - dz==1 (10 directions): a run advances exactly one (dz,dy,dx) step per Z-plane, so it visits at +// most one voxel per plane. This lets a "carry" array (indexed by the CURRENT plane's (y,x)) +// track each in-progress run's length across a 2-plane window, finalizing (recording into the +// grey/length histogram) a run the moment its continuation check fails, instead of needing the +// whole volume. Peak memory is O(plane area), not O(volume). +// +// Grey-level LUT and histogram-fill both reuse calc_SRE()/etc unchanged once a full SimpleMatrix +// is assembled per direction, so values are identical to calculate(). +void D3_GLRLM_feature::osized_calculate (LR& r, const Fsettings& s, ImageLoader&) +{ + n_angles_ = (int) (sizeof(SHIFTS13) / sizeof(AngleShift)); + clear_buffers(); + + PixIntens minI = r.aux_min, maxI = r.aux_max; + if (minI == maxI) + { + double w = STNGS_NAN(s); + angled_SRE.assign (n_angles_, w); angled_LRE.assign (n_angles_, w); + angled_GLN.assign (n_angles_, w); angled_GLNN.assign (n_angles_, w); + angled_RLN.assign (n_angles_, w); angled_RLNN.assign (n_angles_, w); + angled_RP.assign (n_angles_, w); angled_GLV.assign (n_angles_, w); + angled_RV.assign (n_angles_, w); angled_RE.assign (n_angles_, w); + angled_LGLRE.assign (n_angles_, w); angled_HGLRE.assign (n_angles_, w); + angled_SRLGLE.assign (n_angles_, w); angled_SRHGLE.assign (n_angles_, w); + angled_LRLGLE.assign (n_angles_, w); angled_LRHGLE.assign (n_angles_, w); + return; + } + + int greyInfo = STNGS_IBSI(s) ? 0 : STNGS_GLRLM_GREYDEPTH(s); + const bool ibsi = STNGS_IBSI(s); + const PixIntens bg = TextureFeature::bin_pixel (0, minI, maxI, greyInfo); + + const int W = (int) r.aabb.get_width(), + H = (int) r.aabb.get_height(), + Dz = (int) r.aabb.get_z_depth(); + const StatsInt xmin = r.aabb.get_xmin(), ymin = r.aabb.get_ymin(), zmin = r.aabb.get_zmin(); + const size_t nvox = r.raw_voxels_NT.size(); + const bool hasBackground = nvox < (size_t) W * H * Dz; + + // --- grey levels I (mirrors calculate(): unique values of the WHOLE binned cube, incl. + // background, minus 0 -- for IBSI a 1..max linspace instead). Ng == I.size() in both cases. + // NOTE: the in-core path's I-construction branches on ibsi_grey_binning(greyInfo) (the numeric + // "no rescale" binning mode), NOT the STNGS_IBSI(s) compliance flag used below for row lookup -- + // under default settings greyInfo==0 (ibsi_grey_binning true) while STNGS_IBSI(s) is false, so + // conflating the two picks the wrong (sparse unique-set) branch here. + std::vector I; + if (ibsi_grey_binning (greyInfo)) + { + PixIntens maxbin = 0; + for (size_t i = 0; i < nvox; i++) + { + PixIntens b = TextureFeature::bin_pixel (r.raw_voxels_NT[i].inten, minI, maxI, greyInfo); + if (b > maxbin) maxbin = b; + } + I.resize (maxbin); + for (int i = 0; i < (int) maxbin; i++) I[i] = i + 1; + } + else + { + std::set uniq; + if (hasBackground && bg != 0) + uniq.insert (bg); + for (size_t i = 0; i < nvox; i++) + { + PixIntens b = TextureFeature::bin_pixel (r.raw_voxels_NT[i].inten, minI, maxI, greyInfo); + if (b != 0) uniq.insert (b); + } + I.assign (uniq.begin(), uniq.end()); + } + const int Ng = (int) I.size(); + const size_t Np = nvox; // matches r.raw_pixels_3D.size() in the in-core path + + auto row_of = [&](PixIntens pi) -> int + { + return ibsi ? (int) pi - 1 : (int)(std::lower_bound (I.begin(), I.end(), pi) - I.begin()); + }; + + // --- stream one direction at a time + for (const AngleShift& ash : SHIFTS13) + { + std::vector> counts (Ng > 0 ? Ng : 1); // counts[row][length-1] = run count + + auto finalize = [&](PixIntens pi, int length) + { + if (length <= 0 || Ng == 0) return; + int row = row_of (pi); + if (row < 0 || row >= Ng) return; + if ((int) counts[row].size() < length) + counts[row].resize (length, 0); + counts[row][length - 1]++; + }; + + if (ash.dz == 0) + { + // In-plane run: reuse the in-core algorithm verbatim on a depth-1 sub-cube per plane. + std::vector plane; + std::vector slab; + for (int z = 0; z < Dz; z++) + { + plane.assign ((size_t) W * H, bg); + r.raw_voxels_NT.read_slab ((size_t)(zmin + z), slab); + for (const auto& v : slab) + { + int lx = (int) v.x - (int) xmin, ly = (int) v.y - (int) ymin; + if (lx >= 0 && lx < W && ly >= 0 && ly < H) + plane[(size_t) ly * W + lx] = TextureFeature::bin_pixel (v.inten, minI, maxI, greyInfo); + } + + SimpleCube D1 (plane, W, H, 1); + std::vector> zones; + D3_GLRLM_feature::gather_rl_zones (zones, ash, D1, /*zeroI=*/ 0); + for (auto& zo : zones) + finalize (zo.first, zo.second); + } + } + else + { + // Cross-plane run (dz==1): carry each in-progress run's length across a 2-plane window, + // indexed by the CURRENT plane's (y,x). A run at (y,x) in plane z has its predecessor at + // (y-dy, x-dx) in plane z-1; if intensities match, the run continues (length+1), else the + // PREDECESSOR's run just ended and is finalized. Whatever remains unconsumed after the + // last plane is finalized once more after the loop. + std::vector prevPlane, curPlane; + std::vector prevCarry, curCarry; + std::vector slab; + + for (int z = 0; z < Dz; z++) + { + curPlane.assign ((size_t) W * H, bg); + r.raw_voxels_NT.read_slab ((size_t)(zmin + z), slab); + for (const auto& v : slab) + { + int lx = (int) v.x - (int) xmin, ly = (int) v.y - (int) ymin; + if (lx >= 0 && lx < W && ly >= 0 && ly < H) + curPlane[(size_t) ly * W + lx] = TextureFeature::bin_pixel (v.inten, minI, maxI, greyInfo); + } + curCarry.assign ((size_t) W * H, 0); + + std::vector consumedPrev; + if (z > 0) + consumedPrev.assign ((size_t) W * H, 0); + + for (int y = 0; y < H; y++) + for (int x = 0; x < W; x++) + { + PixIntens pi = curPlane[(size_t) y * W + x]; + if (pi == 0) // background/skip: matches gather_rl_zones's zeroI==0 check + continue; + + int py = y - ash.dy, px = x - ash.dx; + int length = 1; + if (z > 0 && py >= 0 && py < H && px >= 0 && px < W + && prevPlane[(size_t) py * W + px] == pi) + { + length = prevCarry[(size_t) py * W + px] + 1; + consumedPrev[(size_t) py * W + px] = 1; + } + curCarry[(size_t) y * W + x] = length; + } + + if (z > 0) + { + for (int y = 0; y < H; y++) + for (int x = 0; x < W; x++) + { + size_t idx = (size_t) y * W + x; + if (prevCarry[idx] > 0 && !consumedPrev[idx]) + finalize (prevPlane[idx], prevCarry[idx]); + } + } + + prevPlane.swap (curPlane); + prevCarry.swap (curCarry); + } + // Finalize whatever is still active after the last plane + for (int y = 0; y < H; y++) + for (int x = 0; x < W; x++) + { + size_t idx = (size_t) y * W + x; + if (prevCarry[idx] > 0) + finalize (prevPlane[idx], prevCarry[idx]); + } + } + + // --- assemble the direction's matrix from counts and reuse the shared feature math + int Nr = 0; + for (auto& row : counts) + Nr = (std::max) (Nr, (int) row.size()); + + P_matrix P; + P.allocate (Nr, Ng); + std::fill (P.begin(), P.end(), 0); + for (int row = 0; row < Ng; row++) + for (int col = 0; col < (int) counts[row].size(); col++) + P.xy (col, row) = counts[row][col]; + + double sum = 0; + for (auto p : P) sum += p; + + angled_SRE.push_back (calc_SRE (P, sum)); + angled_LRE.push_back (calc_LRE (P, sum)); + angled_GLN.push_back (calc_GLN (P, sum)); + angled_GLNN.push_back (calc_GLNN (P, sum)); + angled_RLN.push_back (calc_RLN (P, sum)); + angled_RLNN.push_back (calc_RLNN (P, sum)); + angled_RP.push_back (Np > 0 ? sum / double(Np) : STNGS_NAN(s)); + angled_GLV.push_back (calc_GLV (P, I, sum)); + angled_RV.push_back (calc_RV (P, sum)); + angled_RE.push_back (calc_RE (P, sum)); + angled_LGLRE.push_back (calc_LGLRE (P, I, sum)); + angled_HGLRE.push_back (calc_HGLRE (P, I, sum)); + angled_SRLGLE.push_back (calc_SRLGLE (P, I, sum)); + angled_SRHGLE.push_back (calc_SRHGLE (P, I, sum)); + angled_LRLGLE.push_back (calc_LRLGLE (P, I, sum)); + angled_LRHGLE.push_back (calc_LRHGLE (P, I, sum)); + } +} diff --git a/src/nyx/features/3d_glszm.cpp b/src/nyx/features/3d_glszm.cpp index 3b740e2b..69f43e19 100644 --- a/src/nyx/features/3d_glszm.cpp +++ b/src/nyx/features/3d_glszm.cpp @@ -1,9 +1,11 @@ #include #include #include +#include #include #include #include "3d_glszm.h" +#include "streaming_ccl.h" #include "../environment.h" #include "../helpers/timing.h" @@ -36,9 +38,197 @@ D3_GLSZM_feature::D3_GLSZM_feature() : FeatureMethod("D3_GLSZM_feature") void D3_GLSZM_feature::osized_add_online_pixel(size_t x, size_t y, uint32_t intensity) {} // Not supporting +// Out-of-core 3D GLSZM. gather_size_zones() is a full 26-connectivity flood fill that greedily +// walks a zone (with explicit backtracking via 'history') and marks voxels VISITED -- workable +// in-core but assumes the whole cube is resident, since a zone can wander in any direction and +// therefore its final size isn't known until the whole volume has been explored. +// +// This streams a standard two-pass ("raster scan") connected-component labeling: process Z-planes +// in ascending order, and within each plane voxels in raster (row,col) order, checking only +// CAUSAL neighbours -- those already fully labeled by the scan order. For 26-connectivity that is +// 4 same-plane neighbours (West, North, North-West, North-East) plus all 9 neighbours in the +// previous (fully resolved) plane -- 13 of the 26, exactly the causal half. The symmetric "future" +// neighbours are not missed: they get discovered from the OTHER voxel's perspective once the scan +// reaches it. A growable union-find (LabelUnionFind, streaming_ccl.h) merges labels that turn out +// to belong to the same zone and accumulates each zone's voxel count (summed on union), so the +// final size is known the instant the volume finishes streaming -- no second voxel pass needed. +// Peak memory is a 2-plane label window (O(area)) plus the label table (bounded by zone count, +// worst case O(volume) for pathological data -- the same class of bound already accepted for +// GLRLM's per-row run-length histograms). void D3_GLSZM_feature::osized_calculate (LR& r, const Fsettings& s, ImageLoader&) { - calculate (r, s); + clear_buffers(); + + if (r.aux_min == r.aux_max) + { + invalidate (STNGS_NAN(s)); + return; + } + + int greyInfo = STNGS_GLSZM_GREYDEPTH(s); + if (STNGS_IBSI(s)) greyInfo = 0; + PixIntens mn = r.aux_min, mx = r.aux_max; + const PixIntens bg = TextureFeature::bin_pixel (0, mn, mx, greyInfo); + // Background/skip value: matlab-aware, matching THIS feature's own convention (calculate()'s + // zeroI) -- NOT GLRLM's hardcoded 0, which is a different feature's own quirk. + const PixIntens zeroI = matlab_grey_binning (greyInfo) ? 1 : 0; + + const int W = (int) r.aabb.get_width(), + H = (int) r.aabb.get_height(), + Dz = (int) r.aabb.get_z_depth(); + const StatsInt xmin = r.aabb.get_xmin(), ymin = r.aabb.get_ymin(), zmin = r.aabb.get_zmin(); + const size_t nvox = r.raw_voxels_NT.size(); + const bool hasBackground = nvox < (size_t) W * H * Dz; + + // --- grey levels I (mirrors calculate(): ibsi_grey_binning(greyInfo) numeric check for + // construction -- NOT the STNGS_IBSI(s) compliance flag, which calculate() reserves for row + // lookup below; conflating the two was the exact bug found and fixed in GLRLM). + I.clear(); + if (ibsi_grey_binning (greyInfo)) + { + PixIntens maxbin = 0; + for (size_t i = 0; i < nvox; i++) + { + PixIntens b = TextureFeature::bin_pixel (r.raw_voxels_NT[i].inten, mn, mx, greyInfo); + if (b > maxbin) maxbin = b; + } + I.resize (maxbin); + for (int i = 0; i < (int) maxbin; i++) I[i] = i + 1; + } + else + { + std::set uniq; + if (hasBackground && bg != 0) + uniq.insert (bg); + for (size_t i = 0; i < nvox; i++) + { + PixIntens b = TextureFeature::bin_pixel (r.raw_voxels_NT[i].inten, mn, mx, greyInfo); + if (b != 0) uniq.insert (b); + } + I.assign (uniq.begin(), uniq.end()); + } + const bool ibsiRow = STNGS_IBSI(s); + auto row_of = [&](PixIntens pi) -> int + { + return ibsiRow ? (int) pi - 1 : (int)(std::lower_bound (I.begin(), I.end(), pi) - I.begin()); + }; + + // --- streaming 26-connectivity CCL over a 2-plane window + LabelUnionFind uf; + std::vector prevPlane, curPlane; + std::vector prevLabel, curLabel; + std::vector slab; + + static const int SAME_PLANE_CAUSAL[4][2] = { {0,-1}, {-1,0}, {-1,-1}, {-1,1} }; // dy,dx + + for (int z = 0; z < Dz; z++) + { + curPlane.assign ((size_t) W * H, bg); + r.raw_voxels_NT.read_slab ((size_t)(zmin + z), slab); + for (const auto& v : slab) + { + int lx = (int) v.x - (int) xmin, ly = (int) v.y - (int) ymin; + if (lx >= 0 && lx < W && ly >= 0 && ly < H) + curPlane[(size_t) ly * W + lx] = TextureFeature::bin_pixel (v.inten, mn, mx, greyInfo); + } + curLabel.assign ((size_t) W * H, -1); + + for (int y = 0; y < H; y++) + for (int x = 0; x < W; x++) + { + PixIntens pi = curPlane[(size_t) y * W + x]; + if (pi == zeroI) + continue; + + int found = -1; + for (auto& d : SAME_PLANE_CAUSAL) + { + int ny = y + d[0], nx = x + d[1]; + if (ny < 0 || ny >= H || nx < 0 || nx >= W) continue; + if (curPlane[(size_t) ny * W + nx] != pi) continue; + int lbl = curLabel[(size_t) ny * W + nx]; + if (lbl < 0) continue; + found = (found < 0) ? lbl : uf.union_sum (found, lbl); + } + if (z > 0) + { + for (int dy = -1; dy <= 1; dy++) + for (int dx = -1; dx <= 1; dx++) + { + int ny = y + dy, nx = x + dx; + if (ny < 0 || ny >= H || nx < 0 || nx >= W) continue; + if (prevPlane[(size_t) ny * W + nx] != pi) continue; + int lbl = prevLabel[(size_t) ny * W + nx]; + if (lbl < 0) continue; + found = (found < 0) ? lbl : uf.union_sum (found, lbl); + } + } + + if (found < 0) + found = uf.make_label (pi, 1); + else + uf.add_sum (found, 1); + curLabel[(size_t) y * W + x] = found; + } + + prevPlane.swap (curPlane); + prevLabel.swap (curLabel); + } + + // --- one zone per distinct root + std::vector> Zones; + std::vector seenRoot (uf.num_labels(), 0); + for (size_t i = 0; i < uf.num_labels(); i++) + { + int rt = uf.find ((int) i); + if (seenRoot[rt]) continue; + seenRoot[rt] = 1; + Zones.push_back ({ uf.intensity_of (rt), (int) uf.metric (rt) }); + } + + int maxZoneArea = 0; + for (auto& zo : Zones) + maxZoneArea = (std::max) (maxZoneArea, zo.second); + + Ng = (int) I.size(); + Ns = maxZoneArea; + Nz = (int) Zones.size(); + Np = (int) nvox; + + P.allocate (Ns, Ng); + P.fill (0); + for (auto& zo : Zones) + { + int row = row_of (zo.first); + int col = zo.second - 1; + P.xy (col, row)++; + } + + sum_p = 0; + for (auto a : P) sum_p += a; + if (sum_p == 0) + { + invalidate (STNGS_NAN(s)); + return; + } + + calc_sums_of_P(); + fv_SAE = calc_SAE(); + fv_LAE = calc_LAE(); + fv_GLN = calc_GLN(); + fv_GLNN = calc_GLNN(); + fv_SZN = calc_SZN(); + fv_SZNN = calc_SZNN(); + fv_ZP = calc_ZP(); + fv_GLV = calc_GLV(); + fv_ZV = calc_ZV(); + fv_ZE = calc_ZE(); + fv_LGLZE = calc_LGLZE(); + fv_HGLZE = calc_HGLZE(); + fv_SALGLE = calc_SALGLE(); + fv_SAHGLE = calc_SAHGLE(); + fv_LALGLE = calc_LALGLE(); + fv_LAHGLE = calc_LAHGLE(); } /*static*/ void D3_GLSZM_feature::gather_size_zones (std::vector> & Zones, SimpleCube & D, PixIntens zeroI) diff --git a/src/nyx/features/3d_intensity.cpp b/src/nyx/features/3d_intensity.cpp index 2777609f..042e7b0b 100644 --- a/src/nyx/features/3d_intensity.cpp +++ b/src/nyx/features/3d_intensity.cpp @@ -203,21 +203,32 @@ void D3_VoxelIntensityFeatures::osized_calculate (LR & r, const Fsettings & s, c val_COVERED_IMAGE_INTENSITY_RANGE = 1; double n = r.aux_area; + const size_t nvox = r.raw_voxels_NT.size(); - // --MEAN, ENERGY, CENTROID_XY + // Stream voxels off disk (raw_voxels_NT) rather than holding the cube (raw_pixels_3D) or + // reading the 2D z-less cloud (raw_pixels_NT). Each disk pass decodes the whole cloud, so the + // per-voxel scalars are gathered in as few passes as their data dependencies allow (mean must + // be known before dispersion; the percentiles before median-abs-dev / robust mean). + + // --- Pass 1: raw sums + online moments (MEAN, ENERGY, CENTROID, INTEGRATED_INTENSITY, and the + // Moments4 accumulator for SKEWNESS/KURTOSIS -- folded in here instead of its own later pass) double mean_ = 0.0; double energy = 0.0; double cen_x = 0.0, cen_y = 0.0, + cen_z = 0.0, integInten = 0.0; - for (size_t i = 0; i < r.raw_pixels_NT.size(); i++) + Moments4 mom; + for (size_t i = 0; i < nvox; i++) { - Pixel2 px = r.raw_pixels_NT[i]; + Pixel3 px = r.raw_voxels_NT[i]; mean_ += px.inten; energy += px.inten * px.inten; cen_x += px.x; cen_y += px.y; + cen_z += px.z; integInten += px.inten; + mom.add(px.inten); } mean_ /= n; val_MEAN = mean_; @@ -225,12 +236,12 @@ void D3_VoxelIntensityFeatures::osized_calculate (LR & r, const Fsettings & s, c val_ROOT_MEAN_SQUARED = sqrt(val_ENERGY / n); val_INTEGRATED_INTENSITY = integInten; - // --MAD, VARIANCE, STDDEV, COV + // --- Pass 2: dispersion (MAD, VARIANCE, STDDEV, COV) -- needs mean_ double mad = 0.0, var = 0.0; - for (auto& px : r.raw_pixels_3D) + for (size_t i = 0; i < nvox; i++) { - double diff = px.inten - mean_; + double diff = r.raw_voxels_NT[i].inten - mean_; mad += std::abs(diff); var += diff * diff; } @@ -244,14 +255,27 @@ void D3_VoxelIntensityFeatures::osized_calculate (LR & r, const Fsettings & s, c // --Standard error val_STANDARD_ERROR = val_STANDARD_DEVIATION / sqrt(n); - //==== Do not calculate features of all-blank intensities (to avoid NANs) + //==== Do not calculate features of all-blank intensities (to avoid NANs). Kept before the + // Moments4-derived and percentile-derived features so a blank ROI leaves them at their + // defaults exactly as before (the accumulator is filled in pass 1 but not consumed until here). if (r.aux_min == 0 && r.aux_max == 0) return; + // Skewness / Kurtosis from the pass-1 accumulator + val_SKEWNESS = mom.skewness(); + val_KURTOSIS = mom.kurtosis(); + // Excess kurtosis. NOTE: this is NOT val_KURTOSIS-3 -- Moments4::excess_kurtosis() has its own + // independent zero-variance (M2==0) guard returning exactly 0.0, whereas kurtosis() ALSO + // returns exactly 0.0 under that same guard; subtracting 3 from the latter wrongly yields -3.0 + // for a degenerate (constant-intensity) ROI instead of 0.0. Only diverges from kurtosis()-3 in + // that degenerate case (for n>4 the two formulas are algebraically identical), which is why + // every non-degenerate fixture matched before this was caught by a blank-ROI OOC test. + val_EXCESS_KURTOSIS = mom.excess_kurtosis(); + // P10, 25, 75, 90, IQR, QCOD, RMAD, entropy, uniformity int n_greybins = STNGS_NGREYS(s); TrivialHistogram H; - H.initialize (n_greybins, r.aux_min, r.aux_max, r.raw_pixels_NT); + H.initialize (n_greybins, r.aux_min, r.aux_max, r.raw_voxels_NT); auto [median_, mode_, p01_, p10_, p25_, p75_, p90_, p99_, iqr_, rmad_, entropy_, uniformity_] = H.get_stats(); val_MEDIAN = median_; val_P01 = p01_; @@ -267,51 +291,41 @@ void D3_VoxelIntensityFeatures::osized_calculate (LR & r, const Fsettings & s, c val_MODE = mode_; val_UNIFORMITY = uniformity_; - // Median absolute deviation - double medad = 0.0; - for (auto& px : r.raw_pixels_3D) - medad += std::abs(px.inten - median_); - val_MEDIAN_ABSOLUTE_DEVIATION = medad / n; + // --Uniformity calculated as PIU, percent image uniformity - see "A comparison of five standard methods for evaluating image + // intensity uniformity in partially parallel imaging MRI" [https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3745492/] + // and https://aapm.onlinelibrary.wiley.com/doi/abs/10.1118/1.2241606 + double piu = (1.0 - double(r.aux_max - r.aux_min) / double(r.aux_max + r.aux_min)) * 100.0; + val_UNIFORMITY_PIU = piu; - // FIX 3ROBUST_MEAN: same omission in the out-of-core path (defaulted to 0) -> compute the - // mean of voxels within the [P10,P90] robust window, matching the 2D implementation. + // --- Pass 3: the percentile-dependent scalars in one sweep -- median absolute deviation, + // robust mean (mean of voxels in the [P10,P90] window, matching the 2D implementation), and + // the hyperskewness/hyperflatness sum-of-powers (the in-core calculate()'s explicit + // definition; Moments4::hyperskewness()/hyperflatness() use a different definition and + // diverge). Each accumulator sums in the same voxel order as when these were three separate + // passes, so the values are unchanged. + double medad = 0.0; double robustMean = 0.0; size_t robustCount = 0; - for (auto& px : r.raw_pixels_3D) - if (px.inten >= p10_ && px.inten <= p90_) + double sumPow5 = 0, sumPow6 = 0; + for (size_t i = 0; i < nvox; i++) + { + double inten = double(r.raw_voxels_NT[i].inten); + medad += std::abs(inten - median_); + if (inten >= p10_ && inten <= p90_) { - robustMean += px.inten; + robustMean += inten; robustCount++; } - val_ROBUST_MEAN = robustCount ? robustMean / double(robustCount) : 0.0; - - // --Uniformity calculated as PIU, percent image uniformity - see "A comparison of five standard methods for evaluating image - // intensity uniformity in partially parallel imaging MRI" [https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3745492/] - // and https://aapm.onlinelibrary.wiley.com/doi/abs/10.1118/1.2241606 - double piu = (1.0 - double(r.aux_max - r.aux_min) / double(r.aux_max + r.aux_min)) * 100.0; - val_UNIFORMITY_PIU = piu; - - // Skewness - Moments4 mom; - for (size_t i = 0; i < r.raw_pixels_NT.size(); i++) - { - Pixel2 px = r.raw_pixels_NT[i]; - mom.add(px.inten); + double diff = inten - mean_; + sumPow5 += std::pow(diff, 5.); + sumPow6 += std::pow(diff, 6.); } - - val_SKEWNESS = mom.skewness(); - - // Kurtosis - val_KURTOSIS = mom.kurtosis(); - - // Excess kurtosis - val_EXCESS_KURTOSIS = val_KURTOSIS - 3; - - // Hyperskewness hs = E[x-mean].^5 / std(x).^5 - val_HYPERSKEWNESS = mom.hyperskewness(); - - // Hyperflatness hf = E[x-mean].^6 / std(x).^6 - val_HYPERFLATNESS = mom.hyperflatness(); + val_MEDIAN_ABSOLUTE_DEVIATION = medad / n; + val_ROBUST_MEAN = robustCount ? robustMean / double(robustCount) : 0.0; + double denomHS = (n * std::pow(val_STANDARD_DEVIATION, 5.)); + val_HYPERSKEWNESS = denomHS == 0. ? 0. : sumPow5 / denomHS; + double denomHF = (n * std::pow(val_STANDARD_DEVIATION, 6.)); + val_HYPERFLATNESS = denomHF == 0. ? 0. : sumPow6 / denomHF; } void D3_VoxelIntensityFeatures::save_value(std::vector>& fvals) @@ -367,10 +381,10 @@ void D3_VoxelIntensityFeatures::reduce (size_t start, size_t end, std::vector>& feature_vals); static void reduce (size_t start, size_t end, std::vector* ptrLabels, std::unordered_map * ptrLabelData, const Fsettings & s, const Dataset & ds); - static void extract (LR& r, const Fsettings& s); + // FIX: extract() must carry the Dataset, because calculate() needs it for + // COVERED_IMAGE_INTENSITY_RANGE (slide extrema via ds.dataset_props[r.slide_idx]). + // The old 2-arg extract called the 2-arg calculate, which is a stub that throws + // "illegal call" -- that broke ALL 3D whole-volume featurization (the segmented path + // goes through reduce(), which passes the Dataset, so it never hit this). + static void extract (LR& r, const Fsettings& s, const Dataset& ds); void cleanup_instance(); // list dependencies of this class diff --git a/src/nyx/features/3d_ngldm.cpp b/src/nyx/features/3d_ngldm.cpp index 52532f0d..db1092fe 100644 --- a/src/nyx/features/3d_ngldm.cpp +++ b/src/nyx/features/3d_ngldm.cpp @@ -1,6 +1,10 @@ #include +#include +#include +#include #include "3d_ngldm.h" #include "../environment.h" +#include "../helpers/helpers.h" using namespace Nyxus; @@ -405,5 +409,112 @@ void D3_NGLDM_feature::osized_add_online_pixel(size_t x, size_t y, uint32_t inte void D3_NGLDM_feature::osized_calculate (LR& r, const Fsettings& s, ImageLoader&) { - calculate (r, s); + // Out-of-core NGLDM. Streams the disk-backed voxel cloud through a 3-plane sliding window of + // dense grey-binned planes and fills the NGLD-matrix directly. Binning, interior-only scan, + // 24-neighbourhood and background handling mirror calc_ngld_matrix exactly; the totals and + // feature math (calc_rowwise_and_columnwise_totals / calc_features) are shared. + clear_buffers(); + + if (r.aux_min == r.aux_max) + { + f_LDE = f_HDE = f_LGLCE = f_HGLCE = f_LDLGLE = f_LDHGLE = f_HDLGLE = f_HDHGLE = + f_GLNU = f_GLNUN = f_DCNU = f_DCNUN = f_GLCM = f_GLV = f_DCM = f_DCP = f_DCV = + f_DCENT = f_DCENE = STNGS_NAN(s); + return; + } + + const int nGrays = STNGS_NGREYS(s); + const bool ibsi = STNGS_IBSI(s); + const PixIntens range = r.aux_max; // binning basis is [0, aux_max], as in calc_ngld_matrix + + const int W = (int) r.aabb.get_width(), + H = (int) r.aabb.get_height(), + Dz = (int) r.aabb.get_z_depth(); + const StatsInt xmin = r.aabb.get_xmin(), ymin = r.aabb.get_ymin(), zmin = r.aabb.get_zmin(); + + // Background grey level of the dense cube (raw 0 binned). Present in the LUT only if the ROI + // bbox actually contains a non-mask (background) voxel, matching unique(aux_image_cube). + const size_t nvox = r.raw_voxels_NT.size(); + const bool hasBackground = nvox < (size_t) W * H * Dz; + const PixIntens bg = Nyxus::to_grayscale (0, 0, range, nGrays, ibsi); + + // --- grey-level LUT: unique binned values over the whole cube (mask voxels + maybe background) + std::set uniq; + if (hasBackground) + uniq.insert (bg); + for (size_t i = 0; i < nvox; i++) + uniq.insert (Nyxus::to_grayscale (r.raw_voxels_NT[i].inten, 0, range, nGrays, ibsi)); + std::vector grey_levels_LUT (uniq.begin(), uniq.end()); // std::set already sorted + int Ng = (int) grey_levels_LUT.size(); + + const int maxNr = nsh + 1; + SimpleMatrix NGLDM; + NGLDM.allocate (maxNr, Ng); + NGLDM.fill (0); + int max_dep = 0; + + // O(1) grey-level -> matrix row, replacing the per-voxel binary search in the scan below. + // rowLUT[v] == lower_bound(grey_levels_LUT, v) for every possible binned level (the max is + // the last, sorted, entry), so the row is identical to the search it replaces. + std::vector rowLUT; + if (Ng > 0) + { + const PixIntens maxbin = grey_levels_LUT.back(); + rowLUT.assign ((size_t) maxbin + 1, 0); + for (PixIntens v = 0; v <= maxbin; v++) + rowLUT[v] = (int) (std::lower_bound (grey_levels_LUT.begin(), grey_levels_LUT.end(), v) - grey_levels_LUT.begin()); + } + + // --- 3-plane sliding window of dense grey-binned planes (bg-initialised) + std::vector> ring (3); + int ringZ[3] = { -1, -1, -1 }; + std::vector slab; + auto load = [&](int lz) + { + if (lz < 0 || lz >= Dz) return; + int slot = lz % 3; + if (ringZ[slot] == lz) return; + std::vector& pl = ring[slot]; + pl.assign ((size_t) W * H, bg); + r.raw_voxels_NT.read_slab ((size_t)(zmin + lz), slab); + for (const auto& v : slab) + { + int lx = (int) v.x - (int) xmin, ly = (int) v.y - (int) ymin; + if (lx >= 0 && lx < W && ly >= 0 && ly < H) + pl[(size_t) ly * W + lx] = Nyxus::to_grayscale (v.inten, 0, range, nGrays, ibsi); + } + ringZ[slot] = lz; + }; + + // Interior voxels only (skip the 1-voxel margin so every voxel has all neighbours), as in + // calc_ngld_matrix. Background is NOT skipped. + for (int c = 1; c < Dz - 1; c++) + { + load (c - 1); load (c); load (c + 1); + const std::vector& cur = ring[c % 3]; + + for (int y = 1; y < H - 1; y++) + for (int x = 1; x < W - 1; x++) + { + PixIntens cpi = cur[(size_t) y * W + x]; + int row = rowLUT[cpi]; + + int n_matches = 0; + for (int i = 0; i < nsh; i++) + { + int nz = c + shifts[i].dz, ny = y + shifts[i].dy, nx = x + shifts[i].dx; + if (nz < 0 || nz >= Dz || ny < 0 || ny >= H || nx < 0 || nx >= W) + continue; + if (cpi == ring[nz % 3][(size_t) ny * W + nx]) + n_matches++; + } + NGLDM.yx (row, n_matches)++; + max_dep = (std::max) (max_dep, n_matches); + } + } + int Nr = max_dep + 1; + + std::vector Sg, Sr; + calc_rowwise_and_columnwise_totals (Sg, Sr, NGLDM, Ng, Nr); + calc_features (Sg, Sr, NGLDM, Nr, grey_levels_LUT, r.aux_area); } diff --git a/src/nyx/features/3d_ngtdm.cpp b/src/nyx/features/3d_ngtdm.cpp index cf6dce21..ef25e39f 100644 --- a/src/nyx/features/3d_ngtdm.cpp +++ b/src/nyx/features/3d_ngtdm.cpp @@ -274,144 +274,142 @@ void D3_NGTDM_feature::osized_add_online_pixel(size_t x, size_t y, uint32_t inte void D3_NGTDM_feature::osized_calculate (LR& r, const Fsettings& s, ImageLoader&) { - // Clear variables + // Out-of-core NGTDM. Streams the disk-backed voxel cloud through a (2*radius+1)-plane sliding + // window of dense grey-binned planes, reproducing calculate() exactly: unique-over-cube grey + // levels (incl background), the "+1 if min level is 0" shift, the radius neighbourhood average + // (which includes background neighbours), then N/S/P and the shared feature math (calc_*). clear_buffers(); + I.clear(); - // Check if the ROI is degenerate (equal intensity) - if (r.aux_min == r.aux_max) - { - bad_roi_data = true; - return; - } - - // Prepare ROI's intensity range for normalize_I() - PixIntens piRange = r.aux_max - r.aux_min; + int greyInfo = STNGS_IBSI(s) ? 0 : STNGS_NGTDM_GREYDEPTH(s); + bool ibsi = STNGS_IBSI(s); + PixIntens mn = r.aux_min, mx = r.aux_max; + const PixIntens bg = TextureFeature::bin_pixel (0, mn, mx, greyInfo); - // Make a list of intensity clusters (zones) - using AveNeighborhoodInte = std::pair; // Pairs of (intensity, average intensity of all 8 neighbors) - std::vector Z; + const int W = (int) r.aabb.get_width(), + H = (int) r.aabb.get_height(), + Dz = (int) r.aabb.get_z_depth(); + const StatsInt xmin = r.aabb.get_xmin(), ymin = r.aabb.get_ymin(), zmin = r.aabb.get_zmin(); - // While scanning clusters, learn unique intensities + // --- unique binned levels over the whole cube (mask + background if the bbox has any) + const size_t nvox = r.raw_voxels_NT.size(); + const bool hasBackground = nvox < (size_t) W * H * Dz; std::unordered_set U; + if (hasBackground) + U.insert (bg); + PixIntens maxbin = hasBackground ? bg : 0; + for (size_t i = 0; i < nvox; i++) + { + PixIntens b = TextureFeature::bin_pixel (r.raw_voxels_NT[i].inten, mn, mx, greyInfo); + U.insert (b); + if (b > maxbin) maxbin = b; + } + Ngp = (int) U.size(); - // ROI image - WriteImageMatrix_nontriv D("D3_NGTDM_feature_osized_calculate_D", r.label); - D.allocate_from_cloud(r.raw_pixels_NT, r.aabb, false); + // --- grey levels I: IBSI uses a linspace [0, max]; otherwise the unique set. Then sort. + if (ibsi) + for (PixIntens i = 0; i <= maxbin; i++) + I.push_back (i); + else + I.assign (U.begin(), U.end()); + std::sort (I.begin(), I.end()); - // Gather zones - unsigned int nGrays = STNGS_NGTDM_GREYDEPTH(s); // former theEnvironment.get_coarse_gray_depth() - for (int row = 0; row < D.get_height(); row++) - for (int col = 0; col < D.get_width(); col++) - { - // Find a non-blank pixel - PixIntens pi = Nyxus::to_grayscale(D.yx(row, col), r.aux_min, piRange, nGrays, STNGS_IBSI(s)); - if (pi == 0) - continue; + // --- "+1 if min level is 0" shift (applied to I and every voxel value) + const bool shift = (!I.empty() && I[0] == 0); + if (shift) + for (auto& x : I) x += 1; + const PixIntens bgv = bg + (shift ? 1 : 0); // background value in the shifted planes + const PixIntens zeroI = matlab_grey_binning(greyInfo) ? 1 : 0; - // Update unique intensities - U.insert(pi); + // is binned data informative? + if (I.size() < 2) + { + _coarseness = _contrast = _busyness = _complexity = _strength = STNGS_NAN(s); + return; + } - // Evaluate the neighborhood - double neigsI = 0; + const int rad = STNGS_NGTDM_RADIUS(s); + Ng = (int) I.size(); + N.assign (Ng, 0); + S.assign (Ng, 0.0); + P.assign (Ng, 0.0); + Nvp = 0; - int nd = 0; // Number of dependencies + // O(1) grey-level -> matrix row, replacing the per-voxel binary search in the scan below. + // rowLUT[v] == lower_bound(I, v) - I.begin() for every possible (shifted) binned level (the + // max is I's last, sorted, entry), so the row is identical to the search it replaces. + std::vector rowLUT ((size_t) I.back() + 1, 0); + for (PixIntens v = 0; v <= I.back(); v++) + rowLUT[v] = (int) (std::lower_bound (I.begin(), I.end(), v) - I.begin()); + + // --- (2*rad+1)-plane sliding window of dense grey-binned planes (shift baked in) + const int ringN = 2 * rad + 1; + std::vector> ring (ringN); + std::vector ringZ (ringN, -1); + std::vector slab; + auto load = [&](int lz) + { + if (lz < 0 || lz >= Dz) return; + int slot = lz % ringN; + if (ringZ[slot] == lz) return; + std::vector& pl = ring[slot]; + pl.assign ((size_t) W * H, bgv); + r.raw_voxels_NT.read_slab ((size_t)(zmin + lz), slab); + for (const auto& v : slab) + { + int lx = (int) v.x - (int) xmin, ly = (int) v.y - (int) ymin; + if (lx >= 0 && lx < W && ly >= 0 && ly < H) + pl[(size_t) ly * W + lx] = TextureFeature::bin_pixel (v.inten, mn, mx, greyInfo) + (shift ? 1 : 0); + } + ringZ[slot] = lz; + }; - if (D.safe(row - 1, col) && D.yx(row - 1, col) != 0) // North - { - neigsI += Nyxus::to_grayscale(D.yx(row - 1, col), r.aux_min, piRange, nGrays, STNGS_IBSI(s)); - nd++; - } + for (int c = 0; c < Dz; c++) + { + for (int lz = c - rad; lz <= c + rad; lz++) + load (lz); + const std::vector& cur = ring[c % ringN]; - if (D.safe(row - 1, col + 1) && D.yx(row - 1, col + 1) != 0) // North-East + for (int y = 0; y < H; y++) + for (int x = 0; x < W; x++) { - neigsI += Nyxus::to_grayscale(D.yx(row - 1, col + 1), r.aux_min, piRange, nGrays, STNGS_IBSI(s)); - nd++; - } + PixIntens pi = cur[(size_t) y * W + x]; + if (pi == zeroI) // skip background/off-mask voxels + continue; - if (D.safe(row, col + 1) && D.yx(row, col + 1) != 0) // East - { - neigsI += Nyxus::to_grayscale(D.yx(row, col + 1), r.aux_min, piRange, nGrays, STNGS_IBSI(s)); - nd++; - } - if (D.safe(row + 1, col + 1) && D.yx(row + 1, col + 1) != 0) // South-East - { - neigsI += Nyxus::to_grayscale(D.yx(row + 1, col + 1), r.aux_min, piRange, nGrays, STNGS_IBSI(s)); - nd++; - } - if (D.safe(row + 1, col) && D.yx(row + 1, col) != 0) // South - { - neigsI += Nyxus::to_grayscale(D.yx(row + 1, col), r.aux_min, piRange, nGrays, STNGS_IBSI(s)); - nd++; - } - if (D.safe(row + 1, col - 1) && D.yx(row + 1, col - 1) != 0) // South-West - { - neigsI += Nyxus::to_grayscale(D.yx(row + 1, col - 1), r.aux_min, piRange, nGrays, STNGS_IBSI(s)); - nd++; - } - if (D.safe(row, col - 1) && D.yx(row, col - 1) != 0) // West - { - neigsI += Nyxus::to_grayscale(D.yx(row, col - 1), r.aux_min, piRange, nGrays, STNGS_IBSI(s)); - nd++; - } - if (D.safe(row - 1, col - 1) && D.yx(row - 1, col - 1) != 0) // North-West - { - neigsI += Nyxus::to_grayscale(D.yx(row - 1, col - 1), r.aux_min, piRange, nGrays, STNGS_IBSI(s)); - nd++; - } + double neigsI = 0; + int nd = 0; + for (int dz = -rad; dz <= rad; dz++) + for (int dy = -rad; dy <= rad; dy++) + for (int dx = -rad; dx <= rad; dx++) + { + if (dz == 0 && dy == 0 && dx == 0) + continue; + int nz = c + dz, ny = y + dy, nx = x + dx; + if (nz < 0 || nz >= Dz || ny < 0 || ny >= H || nx < 0 || nx >= W) + continue; + neigsI += ring[nz % ringN][(size_t) ny * W + nx]; + nd++; + } - // Save the intensity's average neighborhood intensity - if (nd > 0) - { - neigsI /= nd; - AveNeighborhoodInte z = { pi, neigsI }; - Z.push_back(z); + if (nd > 0) + { + double aveNeigI = neigsI / nd; + int row = rowLUT[pi]; + N[row]++; + S[row] += std::abs ((double) I[row] - aveNeigI); + if (aveNeigI > 0.0) + Nvp++; + } } - } - - // Fill the matrix - - Ng = (int) U.size(); - Ngp = (int) U.size(); - - // --allocate the matrix - P.resize (Ng, 0); - S.resize (Ng, 0); - N.resize (Ng, 0); - - // --Set to vector to be able to know each intensity's index - std::vector I(U.begin(), U.end()); - std::sort(I.begin(), I.end()); // Optional - - // --Calculate N and S - for (auto& z : Z) - { - // row - auto iter = std::find(I.begin(), I.end(), z.first); - int row = (STNGS_IBSI(s)) ? - z.first : int(iter - I.begin()); - // col - int col = (int)z.second; // 1-based - // increment - N[row]++; - // --S - PixIntens pi = row; - double aveNeigI = z.second; - S[row] += std::abs(pi - aveNeigI); - // --Nvp - if (aveNeigI > 0.0) - Nvp++; } - // --Calculate Nvc (sum of N) Nvc = 0; - for (int i = 0; i < N.size(); i++) + for (size_t i = 0; i < N.size(); i++) Nvc += N[i]; + for (size_t i = 0; i < N.size(); i++) + P[i] = (Nvc > 0) ? (double) N[i] / Nvc : 0.0; - // --Calculate P - for (int i = 0; i < N.size(); i++) - P[i] = (double)N[i] / Nvc; - - // Calculate features _coarseness = calc_Coarseness(); _contrast = calc_Contrast(); _busyness = calc_Busyness(); diff --git a/src/nyx/features/3d_surface.cpp b/src/nyx/features/3d_surface.cpp index ddc30bf1..7ca04855 100644 --- a/src/nyx/features/3d_surface.cpp +++ b/src/nyx/features/3d_surface.cpp @@ -290,6 +290,17 @@ void D3_SurfaceFeature::build_surface (LR & r) } } + build_hull (P); +} + +// Build the convex-hull facet complex from a contour point cloud. Shared by the in-core +// build_surface and the out-of-core osized_calculate (which collects the same contour points +// from the disk-backed voxel cloud), so both produce an identical hull. +void D3_SurfaceFeature::build_hull (const std::vector>& P) +{ + constexpr std::size_t dim = 3; + using Points = std::vector>; + const auto eps = 1e-10f; quick_hull qh{ dim, eps }; qh.add_points(std::cbegin(P), std::cend(P)); @@ -352,16 +363,16 @@ void D3_SurfaceFeature::calculate (LR& r, const Fsettings& s) } // volume - - // (fast approximation based on cubic lattice packaging of balls) - double ball_r3 = 1. / 8., // r^3, after the anisotropy correction, lattice is expected to be cubic - sumPackedV = 0.0; - for (const auto& vox : r.raw_pixels_3D) - sumPackedV += 4. / 3. * M_PI * ball_r3; + + // (fast approximation based on cubic lattice packaging of balls). The per-voxel packed + // volume is a constant, so the whole sum is voxel_count * that constant (the out-of-core + // path already computes it closed-form; match it here instead of an O(n) accumulation). + double ball_r3 = 1. / 8.; // r^3, after the anisotropy correction, lattice is expected to be cubic + double sumPackedV = double(r.raw_pixels_3D.size()) * (4. / 3. * M_PI * ball_r3); fval_VOXEL_VOLUME = sumPackedV / 0.5236; // packaging density at kissing number = 4 (cubic lattice) // surface - + // -- order z-planes' indices std::vector zindices; for (auto& plane : r.zplanes) @@ -517,7 +528,180 @@ void D3_SurfaceFeature::calculate (LR& r, const Fsettings& s) void D3_SurfaceFeature::osized_add_online_pixel(size_t x, size_t y, uint32_t intensity) {} -void D3_SurfaceFeature::osized_calculate (LR& r, const Fsettings& s, ImageLoader& ldr) {} +void D3_SurfaceFeature::osized_calculate (LR& r, const Fsettings& s, ImageLoader& ldr) +{ + // Out-of-core surface: stream the disk-backed voxel cloud (raw_voxels_NT) one Z-plane at a + // time instead of holding the whole cube (raw_pixels_3D). Per plane we extract its contour and + // tally exposed-face adjacencies; covariance is accumulated online. Peak memory is the + // collected boundary points (~surface area) plus two Z-plane occupancy bitmaps -- bounded by + // area, not volume. Values mirror the in-core calculate(). + size_t n = r.raw_voxels_NT.size(); + if (n == 0) + { + cleanup_instance(); + return; + } + + if (STNGS_SINGLEROI(s)) + { + auto w = r.aabb.get_width(), + h = r.aabb.get_height(), + d = r.aabb.get_z_depth(); + + fval_AREA = 2 * (w*h + h*d + w*d); + fval_VOLUME_CONVEXHULL = fval_VOXEL_VOLUME = fval_MESH_VOLUME = w * h * d; + fval_AREA_2_VOLUME = fval_AREA / fval_VOXEL_VOLUME; + fval_COMPACTNESS1 = fval_VOXEL_VOLUME / std::sqrt(M_PI * fval_AREA * fval_AREA * fval_AREA); + fval_COMPACTNESS2 = 36. * M_PI * fval_VOXEL_VOLUME * fval_VOXEL_VOLUME / (fval_AREA * fval_AREA * fval_AREA); + fval_SPHERICAL_DISPROPORTION = fval_AREA / std::pow(36. * M_PI * fval_VOXEL_VOLUME * fval_VOXEL_VOLUME, 1. / 3.); + fval_SPHERICITY = std::pow(36. * M_PI * fval_VOXEL_VOLUME * fval_VOXEL_VOLUME, 1. / 3.) / fval_AREA; + fval_MAJOR_AXIS_LEN = fval_MINOR_AXIS_LEN = fval_LEAST_AXIS_LEN = fval_ELONGATION = fval_FLATNESS = 0; + return; + } + + // -- VOXEL_VOLUME: same cubic-lattice ball packing, a function of voxel count only + double sumPackedV = double(n) * (4. / 3. * M_PI * (1. / 8.)); + fval_VOXEL_VOLUME = sumPackedV / 0.5236; + + const int W = (int) r.aabb.get_width(), + H = (int) r.aabb.get_height(), + minx = (int) r.aabb.get_xmin(), + miny = (int) r.aabb.get_ymin(); + const int verb = STNGS_VERBOSLVL(s); + + // Collected boundary points for the convex hull (bounded by surface area), plus the running + // centroid of those points (matches the in-core centroid over contour voxels). + std::vector> P; + double cx = 0, cy = 0, cz = 0; + size_t hullCloudLen = 0; + + // Surface area as exposed faces = 6*N - 2*(adjacent voxel pairs). x/y adjacencies are tallied + // within a plane; z adjacencies between a plane and the previous one -- a 2-plane window. + double adjacencies = 0.0; + std::vector prevOcc; // occupancy bitmap of plane (z-1) over the [W x H] ROI bbox + long long prevZ = -2; // z index of prevOcc; -2 = none + + // Online covariance sums over all voxels + double sx = 0, sy = 0, sz = 0, sxx = 0, syy = 0, szz = 0, sxy = 0, sxz = 0, syz = 0; + + const size_t depth = r.raw_voxels_NT.depth(); + std::vector slab; + for (size_t z = 0; z < depth; z++) + { + r.raw_voxels_NT.read_slab (z, slab); + if (slab.empty()) + continue; + + // -- contour of this plane (build over a plane-local cloud; the returned indices point + // into 'slab', and the resulting boundary POINTS/order match the in-core path) + std::vector planeIdx (slab.size()); + for (size_t i = 0; i < slab.size(); i++) + planeIdx[i] = i; + std::vector K; + Nyxus::build_contour_imp (K, slab, planeIdx, (int) z, W, H, minx, miny, verb); + for (auto ik : K) + { + const auto& v = slab[ik]; + P.push_back (std::array({ (float) v.x, (float) v.y, (float) v.z })); + cx += v.x; cy += v.y; cz += v.z; + hullCloudLen++; + } + + // -- this plane's occupancy bitmap + online covariance sums + std::vector curOcc ((size_t) W * H, 0); + for (const auto& v : slab) + { + int lx = (int) v.x - minx, ly = (int) v.y - miny; + if (lx >= 0 && lx < W && ly >= 0 && ly < H) + curOcc[(size_t) ly * W + lx] = 1; + + double dx = (double) v.x, dy = (double) v.y, dz = (double) v.z; + sx += dx; sy += dy; sz += dz; + sxx += dx * dx; syy += dy * dy; szz += dz * dz; + sxy += dx * dy; sxz += dx * dz; syz += dy * dz; + } + + // -- adjacencies. +x / +y within this plane (each unordered in-plane pair once) + for (const auto& v : slab) + { + int lx = (int) v.x - minx, ly = (int) v.y - miny; + if (lx + 1 < W && ly >= 0 && ly < H && curOcc[(size_t) ly * W + (lx + 1)]) + adjacencies += 1.0; + if (ly + 1 < H && lx >= 0 && lx < W && curOcc[(size_t) (ly + 1) * W + lx]) + adjacencies += 1.0; + // +z: pair with the previous plane at the same (x,y) (each unordered z pair once) + if (prevZ == (long long) z - 1 && lx >= 0 && lx < W && ly >= 0 && ly < H + && prevOcc[(size_t) ly * W + lx]) + adjacencies += 1.0; + } + + prevOcc.swap (curOcc); + prevZ = (long long) z; + } + + fval_AREA = 6.0 * double(n) - 2.0 * adjacencies; + + // -- convex hull from the collected boundary points + build_hull (P); + + // -- convex hull volume via signed tetrahedra from the contour centroid + if (hullCloudLen) + { + cx /= double(hullCloudLen); + cy /= double(hullCloudLen); + cz /= double(hullCloudLen); + } + fval_VOLUME_CONVEXHULL = 0; + for (const auto& sx3 : hull_complex) + { + double d = Nyxus::det4( + sx3.a[0], sx3.a[1], sx3.a[2], 1, + sx3.b[0], sx3.b[1], sx3.b[2], 1, + sx3.c[0], sx3.c[1], sx3.c[2], 1, + cx, cy, cz, 1); + fval_VOLUME_CONVEXHULL += d / 6; + } + + // -- volume-area ratio features + fval_MESH_VOLUME = fval_VOLUME_CONVEXHULL; + fval_AREA_2_VOLUME = fval_AREA / fval_VOXEL_VOLUME; + fval_COMPACTNESS1 = fval_VOXEL_VOLUME / std::sqrt(M_PI * fval_AREA * fval_AREA * fval_AREA); + fval_COMPACTNESS2 = 36. * M_PI * fval_VOXEL_VOLUME * fval_VOXEL_VOLUME / (fval_AREA * fval_AREA * fval_AREA); + fval_SPHERICAL_DISPROPORTION = fval_AREA / std::pow(36. * M_PI * fval_VOXEL_VOLUME * fval_VOXEL_VOLUME, 1. / 3.); + fval_SPHERICITY = std::pow(36. * M_PI * fval_VOXEL_VOLUME * fval_VOXEL_VOLUME, 1. / 3.) / fval_AREA; + + // -- PCA axis lengths from the covariance matrix (sample covariance, matching calc_covariance: + // Cov(a,b) = (Sum a*b - N*mean_a*mean_b) / (N-1)) + double K[3][3]; + if (n > 1) + { + double mx = sx / double(n), my = sy / double(n), mz = sz / double(n); + double nf = double(n) - 1.0; + K[0][0] = (sxx - double(n) * mx * mx) / nf; + K[1][1] = (syy - double(n) * my * my) / nf; + K[2][2] = (szz - double(n) * mz * mz) / nf; + K[0][1] = K[1][0] = (sxy - double(n) * mx * my) / nf; + K[0][2] = K[2][0] = (sxz - double(n) * mx * mz) / nf; + K[1][2] = K[2][1] = (syz - double(n) * my * mz) / nf; + } + else + K[0][0] = K[0][1] = K[0][2] = K[1][0] = K[1][1] = K[1][2] = K[2][0] = K[2][1] = K[2][2] = 0; + + double L[3]; + if (Nyxus::calc_eigvals(L, K)) + { + // calc_eigvals returns L sorted DESCENDING (L[0] largest): MAJOR<-L[0], MINOR<-L[1], + // LEAST<-L[2]; ELONGATION=MINOR/MAJOR=sqrt(L[1]/L[0]), FLATNESS=LEAST/MAJOR=sqrt(L[2]/L[0]). + // Matches MIRP/IBSI and the in-RAM calculate() path. + fval_MAJOR_AXIS_LEN = 4.0 * sqrt(L[0]); + fval_MINOR_AXIS_LEN = 4.0 * sqrt(L[1]); + fval_LEAST_AXIS_LEN = 4.0 * sqrt(L[2]); + fval_ELONGATION = sqrt(L[1] / L[0]); + fval_FLATNESS = sqrt(L[2] / L[0]); + } + else + fval_MAJOR_AXIS_LEN = fval_MINOR_AXIS_LEN = fval_LEAST_AXIS_LEN = fval_ELONGATION = fval_FLATNESS = 0.0; +} void D3_SurfaceFeature::save_value(std::vector>& fvals) { diff --git a/src/nyx/features/3d_surface.h b/src/nyx/features/3d_surface.h index be1042f4..8410e8bf 100644 --- a/src/nyx/features/3d_surface.h +++ b/src/nyx/features/3d_surface.h @@ -1,6 +1,8 @@ #pragma once #pragma once +#include +#include #include "../dataset.h" #include "../featureset.h" #include "../feature_method.h" @@ -41,13 +43,13 @@ class D3_SurfaceFeature : public FeatureMethod private: - struct Simplex3 - { - float a[3], b[3], c[3]; // layout: x, y, z - Simplex3 (const float* a_, const float* b_, const float* c_) - { - for (int i = 0; i < 3; i++) - { + struct Simplex3 + { + float a[3], b[3], c[3]; // layout: x, y, z + Simplex3 (const float* a_, const float* b_, const float* c_) + { + for (int i = 0; i < 3; i++) + { a[i] = a_[i]; b[i] = b_[i]; c[i] = c_[i]; @@ -56,8 +58,10 @@ class D3_SurfaceFeature : public FeatureMethod }; std::vector hull_complex; - + void build_surface (LR& r); + // Build the hull complex from a contour point cloud (shared by in-core + out-of-core paths) + void build_hull (const std::vector>& P); double fval_AREA, fval_AREA_2_VOLUME, diff --git a/src/nyx/features/histogram.h b/src/nyx/features/histogram.h index e1547edd..50efc997 100644 --- a/src/nyx/features/histogram.h +++ b/src/nyx/features/histogram.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include #include @@ -11,6 +11,7 @@ #include "../helpers/helpers.h" #include "pixel.h" #include "image_matrix_nontriv.h" +#include "voxel_cloud_nontriv.h" using HistoItem = unsigned int; @@ -20,15 +21,20 @@ class TrivialHistogram TrivialHistogram() {} - template - void initialize (int n_cust_bins, HistoItem min_value, HistoItem max_value, const std::vector& raw_data) + // One implementation for every intensity source: the in-RAM std::vector, and the two + // disk-backed clouds (OutOfRamPixelCloud, OutOfRamVoxelCloud). All expose size() and a + // range-for yielding an item with an .inten field, so a single template serves them and the + // in-core / out-of-core histograms stay identical by construction (fixes that had to be + // applied to each copy -- e.g. the p10/p90 robust-MAD -- now live in one place). + template + void initialize (int n_cust_bins, HistoItem min_value, HistoItem max_value, const Src& raw_data) { // safety int n_custBins = std::abs(n_cust_bins); pop_ = raw_data.size(); - // Allocate + // Allocate // -- "percentile" bins100_.reserve(100); for (int i = 0; i < 100 + 1; i++) @@ -43,9 +49,12 @@ class TrivialHistogram maxVal_ = max_value; auto valRange = maxVal_ - minVal_; - // unique values + // unique values (reserve the full population up front: for the disk-backed clouds this + // avoids the log-many reallocations that made the out-of-core pass grow its resident + // footprint one push at a time) + U_.reserve (pop_); for (auto s : raw_data) - U_.push_back (s.inten); + U_.push_back (s.inten); // Build the "percentile" histogram binW100_ = double(valRange) / 100.; @@ -108,94 +117,6 @@ class TrivialHistogram } } - void initialize (int n_cust_bins, HistoItem min_value, HistoItem max_value, const OutOfRamPixelCloud& raw_data) - { - // safety - int n_custBins = std::abs(n_cust_bins); - - pop_ = raw_data.size(); - - // Allocate - // -- "percentile" - bins100_.reserve(100); - for (int i = 0; i < 100 + 1; i++) - bins100_.push_back(0); - // -- "uint8" - bins_cust_.reserve(n_custBins); - for (int i = 0; i < n_custBins + 1; i++) - bins_cust_.push_back(0); - - // Cache min/max - minVal_ = min_value; - maxVal_ = max_value; - auto valRange = maxVal_ - minVal_; - - // unique values - for (auto s : raw_data) - U_.push_back (s.inten); - - // Build the "percentile" histogram - binW100_ = double(valRange) / 100.; - for (auto s : raw_data) - { - HistoItem h = s.inten; - double realIdx = double(h - minVal_) / binW100_; - int idx = std::isnan(realIdx) ? 0 : int(realIdx); - (bins100_[idx])++; - } - - // -- Fix the special last bin - bins100_[100 - 1] += bins100_[100]; - bins100_[100] = 0; - - // Build the "uint8" histogram - binWcust_ = double(valRange) / double(n_custBins-1); - for (auto s : raw_data) - { - HistoItem h = Nyxus::to_grayscale(s.inten, minVal_, valRange, n_custBins); - bins_cust_[h] = bins_cust_[h] + 1; - } - - // -- Fix the special last bin - bins_cust_[n_custBins - 1] += bins_cust_[n_custBins]; - bins_cust_[n_custBins] = 0; - - // Mean calculation - meanVal_ = 0; - for (auto s : raw_data) - meanVal_ += double(s.inten); - meanVal_ /= double(pop_); - - // percentiles - calc_percentiles(); - - // robust mean - bin10ctr_ = bins100_ [10]; - bin90ctr_ = bins100_ [90]; - mean1090val_ = 0.0; - size_t pop1090 = 0; - for (auto pxl : raw_data) - { - double a = double (pxl.inten); - if (a >= bin10ctr_ && a <= bin90ctr_) - { - mean1090val_ += a; - pop1090++; - } - } - rmad_ = 0.0; - if (pop1090) - { - for (auto pxl : raw_data) - { - double a = double(pxl.inten); - if (a >= bin10ctr_ && a <= bin90ctr_) - rmad_ += (std::fabs)(a - mean1090val_); - } - rmad_ /= double(pop1090); - } - } - void initialize_uniques (const std::vector& raw_data) { for (auto h : raw_data) @@ -290,7 +211,7 @@ class TrivialHistogram size_t pop_ = 0; HistoItem minVal_, maxVal_; double meanVal_, binW100_, binWcust_; - double bin10ctr_, bin90ctr_, mean1090val_, rmad_; // robust estimation + double mean1090val_, rmad_; // robust estimation (p10/p90-thresholded) std::vector bins100_, bins_cust_; std::vector U_; double p1_, p10_, p25_, p75_, p90_, p99_; diff --git a/src/nyx/features/intensity.cpp b/src/nyx/features/intensity.cpp index f4f268dd..6aa511e2 100644 --- a/src/nyx/features/intensity.cpp +++ b/src/nyx/features/intensity.cpp @@ -269,8 +269,10 @@ void PixelIntensityFeatures::osized_calculate (LR& r, const Fsettings& stng, con // --MAD, VARIANCE, STDDEV, COV double mad = 0.0, var = 0.0; - for (auto& px : r.raw_pixels) + // out-of-core: dispersion is accumulated from the disk-backed pixel cloud + for (size_t i = 0; i < r.raw_pixels_NT.size(); i++) { + Pixel2 px = r.raw_pixels_NT[i]; double diff = px.inten - mean_; mad += std::abs(diff); var += diff * diff; @@ -323,12 +325,16 @@ void PixelIntensityFeatures::osized_calculate (LR& r, const Fsettings& stng, con // Median absolute deviation double medad = 0.0; - for (auto& px : r.raw_pixels) + // out-of-core: read from the disk-backed pixel cloud + for (size_t i = 0; i < r.raw_pixels_NT.size(); i++) + { + Pixel2 px = r.raw_pixels_NT[i]; medad += std::abs(px.inten - median_); + } val_MEDIAN_ABSOLUTE_DEVIATION = medad / n; - // --Uniformity calculated as PIU, percent image uniformity - see "A comparison of five standard methods for evaluating image - // intensity uniformity in partially parallel imaging MRI" [https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3745492/] + // --Uniformity calculated as PIU, percent image uniformity - see "A comparison of five standard methods for evaluating image + // intensity uniformity in partially parallel imaging MRI" [https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3745492/] // and https://aapm.onlinelibrary.wiley.com/doi/abs/10.1118/1.2241606 double piu = (1.0 - double(r.aux_max - r.aux_min) / double(r.aux_max + r.aux_min)) * 100.0; val_UNIFORMITY_PIU = piu; @@ -349,11 +355,19 @@ void PixelIntensityFeatures::osized_calculate (LR& r, const Fsettings& stng, con // Excess kurtosis val_EXCESS_KURTOSIS = val_KURTOSIS - 3; - // Hyperskewness hs = E[x-mean].^5 / std(x).^5 - val_HYPERSKEWNESS = mom.hyperskewness(); - - // Hyperflatness hf = E[x-mean].^6 / std(x).^6 - val_HYPERFLATNESS = mom.hyperflatness(); + // Hyperskewness / Hyperflatness from explicit central moments over the disk-backed cloud, + // matching the in-core path's sum-of-powers definition rather than the Moments4 variant + double sumPow5 = 0, sumPow6 = 0; + for (size_t i = 0; i < r.raw_pixels_NT.size(); i++) + { + double diff = double(r.raw_pixels_NT[i].inten) - mean_; + sumPow5 += std::pow(diff, 5.); + sumPow6 += std::pow(diff, 6.); + } + double denom5 = n * std::pow(val_STANDARD_DEVIATION, 5.); + val_HYPERSKEWNESS = denom5 == 0. ? 0. : sumPow5 / denom5; + double denom6 = n * std::pow(val_STANDARD_DEVIATION, 6.); + val_HYPERFLATNESS = denom6 == 0. ? 0. : sumPow6 / denom6; } void PixelIntensityFeatures::save_value(std::vector>& fvals) diff --git a/src/nyx/features/intensity.h b/src/nyx/features/intensity.h index 140c61bf..c99035b1 100644 --- a/src/nyx/features/intensity.h +++ b/src/nyx/features/intensity.h @@ -46,6 +46,10 @@ class PixelIntensityFeatures : public FeatureMethod void osized_add_online_pixel(size_t x, size_t y, uint32_t intensity); void osized_calculate (LR & roi, const Fsettings & s, const Dataset & dataset, ImageLoader& ldr); void osized_calculate (LR& roi, const Fsettings& s, ImageLoader& ldr) { throw std::runtime_error("illegal use of PixelIntensityFeatures::osized_calculate()"); } + // Out-of-core dispatch for intensity: route to the Dataset-taking osized_calculate, which + // needs slide props for COVERED_IMAGE_INTENSITY_RANGE. The Dataset-less overload above is a + // guard that throws, so the base osized_scan_whole_image cannot be used for this feature. + void osized_scan_whole_image (LR& roi, const Fsettings& s, const Dataset& ds, ImageLoader& ldr) override { osized_calculate (roi, s, ds, ldr); save_value (roi.fvals); } void save_value (std::vector>& feature_vals); static void reduce (size_t start, size_t end, std::vector* ptrLabels, std::unordered_map * ptrLabelData, const Fsettings & settings, const Dataset & dataset); static void extract (LR & roi, const Fsettings & settings, const Dataset & dataset); diff --git a/src/nyx/features/intensity_histogram.h b/src/nyx/features/intensity_histogram.h index 815fa0f6..f1d08811 100644 --- a/src/nyx/features/intensity_histogram.h +++ b/src/nyx/features/intensity_histogram.h @@ -84,6 +84,10 @@ class IntensityHistogramFeatures : public FeatureMethod void osized_add_online_pixel (size_t x, size_t y, uint32_t intensity); void osized_calculate (LR& roi, const Fsettings& s, const Dataset& dataset, ImageLoader& ldr); void osized_calculate (LR& roi, const Fsettings& s, ImageLoader& ldr) { throw std::runtime_error("illegal use of IntensityHistogramFeatures::osized_calculate()"); } + // Out-of-core dispatch for the intensity histogram: route to the Dataset-taking + // osized_calculate. The Dataset-less overload above is a guard that throws, so the base + // osized_scan_whole_image cannot be used for this feature. + void osized_scan_whole_image (LR& roi, const Fsettings& s, const Dataset& ds, ImageLoader& ldr) override { osized_calculate (roi, s, ds, ldr); save_value (roi.fvals); } void save_value (std::vector>& feature_vals); static void reduce (size_t start, size_t end, std::vector* ptrLabels, std::unordered_map * ptrLabelData, const Fsettings& settings, const Dataset& dataset); static void extract (LR& roi, const Fsettings& settings, const Dataset& dataset); diff --git a/src/nyx/features/streaming_ccl.h b/src/nyx/features/streaming_ccl.h new file mode 100644 index 00000000..9bdb7217 --- /dev/null +++ b/src/nyx/features/streaming_ccl.h @@ -0,0 +1,67 @@ +#pragma once + +#include +#include +#include "pixel.h" + +/// @brief Growable union-find over connected-component labels, shared by the out-of-core +/// GLSZM/GLDZM streaming connected-component labeling (see docs/13-3d-ooc-glszm-gldzm-design.md). +/// +/// A label is created for each newly-discovered zone (make_label) and merged with another when a +/// streaming scan discovers two previously-separate labels actually belong to the same zone +/// (union_sum/union_min). Each label carries a running metric -- the zone's voxel count for GLSZM +/// (combined by SUM on union) or its minimum distance-to-border for GLDZM (combined by MIN) -- so +/// after the whole volume streams once, each root label directly holds its zone's final metric; no +/// second pass over voxels is needed. +class LabelUnionFind +{ +public: + int make_label (PixIntens intensity, long long initial_metric) + { + parent_.push_back ((int) parent_.size()); + rank_.push_back (0); + metric_.push_back (initial_metric); + intensity_.push_back (intensity); + return (int) parent_.size() - 1; + } + + int find (int x) + { + while (parent_[x] != x) + { + parent_[x] = parent_[parent_[x]]; // path halving + x = parent_[x]; + } + return x; + } + + // Merge labels a and b, combining their metric as a SUM (GLSZM zone size) + int union_sum (int a, int b) { return union_impl (a, b, true); } + // Merge labels a and b, combining their metric as a MIN (GLDZM zone distance) + int union_min (int a, int b) { return union_impl (a, b, false); } + + // Add this voxel to an existing zone's running size (GLSZM) + void add_sum (int label, long long delta) { int r = find (label); metric_[r] += delta; } + // Fold this voxel's distance into an existing zone's running minimum (GLDZM) + void update_min (int label, long long val) { int r = find (label); metric_[r] = (std::min)(metric_[r], val); } + + long long metric (int label) { return metric_[find (label)]; } + PixIntens intensity_of (int label) const { return intensity_[label]; } + size_t num_labels() const { return parent_.size(); } + +private: + int union_impl (int a, int b, bool sumMode) + { + int ra = find (a), rb = find (b); + if (ra == rb) return ra; + if (rank_[ra] < rank_[rb]) std::swap (ra, rb); + parent_[rb] = ra; + if (rank_[ra] == rank_[rb]) rank_[ra]++; + metric_[ra] = sumMode ? (metric_[ra] + metric_[rb]) : (std::min)(metric_[ra], metric_[rb]); + return ra; + } + + std::vector parent_, rank_; + std::vector metric_; + std::vector intensity_; +}; diff --git a/src/nyx/features/voxel_cloud_nontriv.cpp b/src/nyx/features/voxel_cloud_nontriv.cpp new file mode 100644 index 00000000..76729578 --- /dev/null +++ b/src/nyx/features/voxel_cloud_nontriv.cpp @@ -0,0 +1,171 @@ +#include +#include +#include +#include +#include +#include +#include +#include "../environment.h" +#include "../helpers/fsystem.h" +#include "../helpers/helpers.h" +#include "voxel_cloud_nontriv.h" + +OutOfRamVoxelCloud::OutOfRamVoxelCloud() {} + +OutOfRamVoxelCloud::~OutOfRamVoxelCloud() +{ + if (pF) + { + fclose(pF); + pF = nullptr; + fs::remove(filepath); + } +} + +void OutOfRamVoxelCloud::check_io_ok() const +{ + if (!pF) + throw (std::runtime_error("ERROR in OutOfRamVoxelCloud - file might not be open with init()")); +} + +Pixel3 OutOfRamVoxelCloud::decode_record (const char* rec) const +{ + Pixel3 vx; + std::memcpy (&vx.x, rec, sizeof(vx.x)); rec += sizeof(vx.x); + std::memcpy (&vx.y, rec, sizeof(vx.y)); rec += sizeof(vx.y); + std::memcpy (&vx.z, rec, sizeof(vx.z)); rec += sizeof(vx.z); + std::memcpy (&vx.inten, rec, sizeof(vx.inten)); + return vx; +} + +void OutOfRamVoxelCloud::init (unsigned int _roi_label, const std::string& name) +{ + n_items = 0; + slab_first.clear(); + slab_count.clear(); + slab_open = false; + invalidate_cache(); + + auto tid = std::this_thread::get_id(); + + std::stringstream ssPath; + ssPath << Nyxus::get_temp_dir_path() << name << "_roi" << _roi_label << "_tid" << tid << ".vox.nyxus"; + filepath = ssPath.str(); + + errno = 0; + pF = fopen(filepath.c_str(), "w+b"); + if (!pF) + throw (std::runtime_error("Error creating file " + filepath + " Error code:" + std::to_string(errno))); + + if (std::setvbuf(pF, nullptr, _IOFBF, 32768) != 0) + std::cout << "setvbuf failed\n"; +} + +void OutOfRamVoxelCloud::close() +{ + if (pF) + { + fclose(pF); + pF = nullptr; + } +} + +void OutOfRamVoxelCloud::clear() +{ + if (pF) + { + close(); + fs::remove (filepath); + n_items = 0; + } + slab_first.clear(); + slab_count.clear(); + slab_open = false; + invalidate_cache(); +} + +void OutOfRamVoxelCloud::begin_slab (size_t z) +{ + // Grow the index so slab_first[z]/slab_count[z] are addressable + if (slab_first.size() <= z) + { + slab_first.resize (z + 1, n_items); + slab_count.resize (z + 1, 0); + } + slab_first[z] = n_items; + slab_count[z] = 0; + // Remember which plane is open so add_voxel() counts into it explicitly + cur_slab = z; + slab_open = true; +} + +void OutOfRamVoxelCloud::add_voxel (const Pixel3& v) +{ + check_io_ok(); + + fwrite ((const void*) &(v.x), sizeof(v.x), 1, pF); + fwrite ((const void*) &(v.y), sizeof(v.y), 1, pF); + fwrite ((const void*) &(v.z), sizeof(v.z), 1, pF); + fwrite ((const void*) &(v.inten), sizeof(v.inten), 1, pF); + + n_items++; + // Credit the voxel to the plane begin_slab() opened, not to the index's last entry -- the + // two coincide only while planes arrive in strictly ascending z + if (slab_open) + slab_count[cur_slab]++; + + // A write past the cached block would make it stale; the population phase never interleaves + // reads, so this just holds the cache empty until the first read. + invalidate_cache(); +} + +size_t OutOfRamVoxelCloud::size() const +{ + return n_items; +} + +Pixel3 OutOfRamVoxelCloud::get_at (size_t idx) const +{ + check_io_ok(); + + // Serve from the cached block; on a miss, bulk-read the block containing idx with one fread + // (sequential consumers then hit the cache BLK_RECORDS times before the next miss). + if (blk_first < 0 || idx < (size_t) blk_first || idx >= (size_t) blk_first + blk_count) + { + size_t start = (idx / BLK_RECORDS) * BLK_RECORDS; + size_t want = (std::min) (BLK_RECORDS, n_items - start); + blk_buf.resize (want * item_size); + fseek (pF, (long) (start * item_size), SEEK_SET); + size_t got = fread (blk_buf.data(), item_size, want, pF); + blk_first = (long long) start; + blk_count = got; + } + return decode_record (blk_buf.data() + (idx - (size_t) blk_first) * item_size); +} + +void OutOfRamVoxelCloud::read_range (size_t first, size_t count, std::vector& out) const +{ + check_io_ok(); + + out.clear(); + out.reserve (count); + if (count == 0) + return; + + // One bulk fread of the whole range, then decode in RAM (a slab is bounded by the plane bbox) + std::vector buf (count * item_size); + fseek (pF, (long) (first * item_size), SEEK_SET); + size_t got = fread (buf.data(), item_size, count, pF); + for (size_t i = 0; i < got; i++) + out.push_back (decode_record (buf.data() + i * item_size)); +} + +void OutOfRamVoxelCloud::read_slab (size_t z, std::vector& out) const +{ + if (z >= slab_first.size()) + { + out.clear(); + return; + } + read_range (slab_first[z], slab_count[z], out); +} diff --git a/src/nyx/features/voxel_cloud_nontriv.h b/src/nyx/features/voxel_cloud_nontriv.h new file mode 100644 index 00000000..a1c9ce9c --- /dev/null +++ b/src/nyx/features/voxel_cloud_nontriv.h @@ -0,0 +1,86 @@ +#pragma once + +#include +#include +#include "pixel.h" + +/// @brief Writeable out-of-memory 3D voxel cloud. +/// +/// The 3D analog of OutOfRamPixelCloud: it stores Pixel3 records (x,y,z,inten) on disk +/// so an oversized volumetric ROI can be featurized without holding the whole voxel cube +/// in RAM. Records are appended in ascending-Z order by the streaming population scan, so +/// they land on disk already Z-sorted; a small in-RAM per-plane index (first-record offset +/// and count per Z) lets a consumer seek to any Z-slab and read just that slab's voxels. +/// The whole-cloud linear accessors (size/get_at/iterator) serve the Z-agnostic intensity +/// and histogram consumers; the slab accessors serve the sliding-window consumers (surface, +/// and later the 3D texture features). +class OutOfRamVoxelCloud +{ +public: + OutOfRamVoxelCloud(); + OutOfRamVoxelCloud (const OutOfRamVoxelCloud&) = delete; + ~OutOfRamVoxelCloud(); + + // Lifecycle. init() creates the backing file under get_temp_dir_path(). + void init (unsigned int roi_label, const std::string& name); + void close(); + void clear(); + + // Population. begin_slab(z) opens plane z's run of records; add_voxel() appends one record + // to the open plane. No end_slab() is needed -- the count is maintained by add_voxel and the + // run is closed implicitly by the next begin_slab() (or by the end of the population scan). + void begin_slab (size_t z); + void add_voxel (const Pixel3& v); + + // Whole-cloud linear access (intensity, histogram) + size_t size() const; + Pixel3 get_at (size_t idx) const; + Pixel3 operator[] (size_t idx) const { return get_at(idx); } + + // Slab access (surface + the 3D texture features): one Z-plane's voxels + size_t depth() const { return slab_first.size(); } + void read_slab (size_t z, std::vector& out) const; + + struct iterator + { + public: + iterator (const OutOfRamVoxelCloud& obj, std::size_t idx) : m_object(obj), m_index(idx) {} + Pixel3 operator * () const { return m_object.get_at(m_index); } + bool operator == (iterator const& it) const { return (&m_object == &it.m_object) && (m_index == it.m_index); } + bool operator != (iterator const& it) const { return (&m_object != &it.m_object) || (m_index != it.m_index); } + iterator& operator ++ () { ++m_index; return *this; } + private: + const OutOfRamVoxelCloud& m_object; + std::size_t m_index; + }; + iterator begin() const { return iterator(*this, 0); } + iterator end() const { return iterator(*this, size()); } + +private: + size_t n_items = 0; + std::string filepath; + FILE* pF = nullptr; + // One on-disk record = x,y,z (Pixel3's Point3i fields) + intensity + size_t item_size = sizeof(Pixel3::x) + sizeof(Pixel3::y) + sizeof(Pixel3::z) + sizeof(Pixel3::inten); + // Per-Z index built during population: first record index and record count of each plane + std::vector slab_first, slab_count; + // Plane whose run of records is currently open, so add_voxel() credits the count to THAT + // plane rather than to whichever plane happens to be last in the index + size_t cur_slab = 0; + bool slab_open = false; + void check_io_ok() const; + void read_range (size_t first, size_t count, std::vector& out) const; + + // Block read cache so linear consumers (get_at/operator[]/iterator, histogram) do one bulk + // fread per block instead of an fseek + 4 freads per voxel -- the latter made the out-of-core + // intensity/histogram passes ~25x slower than the in-RAM path. One block resident (bounded). + // constexpr (implicitly inline in C++17+) so ODR-uses like (std::min)(BLK_RECORDS, ...), + // which bind it to a const&, don't need a separate out-of-line definition -- GCC errors + // on the missing definition at link time where MSVC silently tolerates it. + static constexpr size_t BLK_RECORDS = 65536; + mutable std::vector blk_buf; + mutable long long blk_first = -1; // index of first cached record; -1 = cache empty + mutable size_t blk_count = 0; + void invalidate_cache() const { blk_first = -1; blk_count = 0; } + Pixel3 decode_record (const char* rec) const; +}; diff --git a/src/nyx/globals.h b/src/nyx/globals.h index b72938db..3e4d9ef4 100644 --- a/src/nyx/globals.h +++ b/src/nyx/globals.h @@ -18,6 +18,8 @@ #include "results_cache.h" #include "roi_cache.h" #include "save_option.h" + +class FeatureMethod; // forward decl for is_3d_ooc_supported() #include "arrow_output_stream.h" #include "nested_feature_aggregation.h" // Nested ROI #include "cli_nested_roi_options.h" @@ -38,7 +40,12 @@ namespace Nyxus const char colname_intensity_image[] = "intensity_image", colname_mask_image[] = "mask_image", colname_roi_label[] = "ROI_label", - colname_t_index[] = "t_index"; + colname_t_index[] = "t_index", + colname_c_index[] = "c_index", // FIX (IO): channel index column, mirrors t_index + colname_phys_unit[] = "phys_unit", // FIX (IO): physical-size unit (leading string column) + colname_phys_x[] = "phys_x", // FIX (IO): physical voxel spacing columns (numeric) + colname_phys_y[] = "phys_y", + colname_phys_z[] = "phys_z"; // segmented 2D workflow int processDataset_2D_segmented( @@ -78,18 +85,39 @@ namespace Nyxus bool gatherRoisMetrics(int slide_idx, const std::string& intens_fpath, const std::string& label_fpath, Environment & env, ImageLoader & L); bool gather_wholeslide_metrics(const std::string& intens_fpath, ImageLoader& L, LR& roi); bool gatherRoisMetrics_25D (Environment& env, size_t sidx, const std::string& intens_fpath, const std::string& mask_fpath, const std::vector& z_indices); - bool gatherRoisMetrics_3D (Environment& env, size_t sidx, const std::string& intens_fpath, const std::string& mask_fpath, size_t t_index); + bool gatherRoisMetrics_3D (Environment& env, size_t sidx, const std::string& intens_fpath, const std::string& mask_fpath, size_t t_index, size_t channel); bool processTrivialRois (Environment& env, const std::vector& trivRoiLabels, const std::string& intens_fpath, const std::string& label_fpath, size_t memory_limit); bool processTrivialRois_25D (Environment & env, const std::vector& trivRoiLabels, const std::string& intens_fpath, const std::string& label_fpath, size_t memory_limit, const std::vector& z_indices); - bool processTrivialRois_3D (Environment & env, size_t sidx, size_t t_index, const std::vector& trivRoiLabels, const std::string& intens_fpath, const std::string& label_fpath, size_t memory_limit); + bool processTrivialRois_3D (Environment & env, size_t sidx, size_t t_index, size_t channel, const std::vector& trivRoiLabels, const std::string& intens_fpath, const std::string& label_fpath, size_t memory_limit); bool processNontrivialRois (Environment& env, const std::vector& nontrivRoiLabels, const std::string& intens_fpath, const std::string& label_fpath); + bool processNontrivialRois_3D (Environment& env, const std::vector& nontrivRoiLabels, const std::string& intens_fpath, const std::string& label_fpath, size_t channel, size_t timeframe); + // Allow-list of 3D feature classes whose osized_calculate() streams from the disk-backed voxel + // cloud; used by processNontrivialRois_3D()'s per-feature out-of-core guard, and independently + // unit-testable (see TEST_3D_OOC_GUARD_REJECTS_UNSUPPORTED_FEATURE). + bool is_3d_ooc_supported (FeatureMethod* f); + // Streams r.raw_voxels_NT from 'imlo' plane-by-plane; when wholevolume is true every voxel is + // kept (no mask), otherwise only voxels matching r.label. Returns false (cloud cleared) if the + // loader can't deliver the volume plane-by-plane. Shared by the segmented and whole-volume + // out-of-core paths (processNontrivialRois_3D, workflow_3d_whole.cpp's oversized branch). + bool populate_3d_voxel_cloud (ImageLoader& imlo, LR& r, size_t channel, size_t timeframe, bool wholevolume); + // Runs every requested feature's out-of-core path over an already-populated r.raw_voxels_NT, + // guarded by is_3d_ooc_supported(); writes into r.fvals via save_value(). Shared the same way. + void run_3d_ooc_features (Environment& env, LR& r, ImageLoader& imloader); bool scan_trivial_wholeslide (LR& vroi, const std::string& intens_fpath, ImageLoader& ldr); // reads pixels of whole slide 'intens_fpath' into virtual ROI 'vroi' bool scan_trivial_wholeslide_anisotropic (LR& vroi, const std::string& intens_fpath, ImageLoader& ldr, double aniso_x, double aniso_y); - bool scan_trivial_wholevolume (LR& vroi, const std::string& intens_fpath, ImageLoader& ldr); - bool scan_trivial_wholevolume_anisotropic (LR& vroi, const std::string& intens_fpath, ImageLoader& ldr, double aniso_x, double aniso_y, double aniso_z); + // FIX (IO): resolve the effective 3D voxel spacing for slide `sidx` and whether the + // anisotropic (resampling) scan path should run. Explicit --aniso* wins; else, when + // --use-physical-spacing is on, the slide's OME PhysicalSize* ratio-normalized (min=1). + // Returns false (and ax=ay=az=1) when the grid is effectively isotropic. + bool resolve_slide_anisotropy (const Environment& env, size_t sidx, double& ax, double& ay, double& az); + + // FIX (IO): channel/timeframe select which C/T plane the whole-volume read assembles (was pinned to c=0,t=0) + bool scan_trivial_wholevolume (LR& vroi, const std::string& intens_fpath, ImageLoader& ldr, size_t channel, size_t timeframe); + bool scan_trivial_wholevolume_anisotropic (LR& vroi, const std::string& intens_fpath, ImageLoader& ldr, double aniso_x, double aniso_y, double aniso_z, size_t channel, size_t timeframe); - bool scanTrivialRois_3D (Environment& env, const std::vector& batch_labels, const std::string& intens_fpath, const std::string& label_fpath, size_t t_index); + bool scanTrivialRois_3D (Environment& env, const std::vector& batch_labels, const std::string& intens_fpath, const std::string& label_fpath, size_t t_index, size_t channel); + bool scanTrivialRois_3D_anisotropic (Environment& env, const std::vector& batch_labels, const std::string& intens_fpath, const std::string& label_fpath, size_t t_index, size_t channel, double aniso_x, double aniso_y, double aniso_z); void dump_roi_metrics (const int dim, const std::string& output_dir, const size_t ram_limit, const std::string& seg_fpath, const Uniqueids& uniqueLabels, const Roidata& roiData); void dump_roi_pixels (const int dim, const std::string& output_dir, const std::vector& batch_labels, const std::string& seg_fpath, const Uniqueids& uniqueLabels, const Roidata& roiData); void dump_2d_image_with_halfcontour( @@ -132,30 +160,37 @@ namespace Nyxus // 2 scenarios of saving a result of feature calculation of a label-intensity file pair: saving to a CSV-file and saving to a matrix to be later consumed by a Python endpoint std::string get_feature_output_fname (Environment& env, const std::string& intFpath, const std::string& segFpath); extern const std::vector mandatory_output_columns; - bool save_features_2_csv (Environment & env, const std::string & intFpath, const std::string & segFpath, const std::string & outputDir, size_t t_index, bool need_aggregation); - bool save_features_2_csv_wholeslide (Environment & env, const LR & r, const std::string & ifpath, const std::string & mfpath, const std::string & outdir, size_t t_index); - bool save_features_2_buffer (ResultsCache &results_cache, Environment &env, size_t t_index); + // FIX (IO): thread c_index alongside t_index so the output carries the channel column + bool save_features_2_csv (Environment & env, const std::string & intFpath, const std::string & segFpath, const std::string & outputDir, size_t t_index, size_t c_index, bool need_aggregation); + bool save_features_2_csv_wholeslide (Environment & env, const LR & r, const std::string & ifpath, const std::string & mfpath, const std::string & outdir, size_t t_index, size_t c_index); + bool save_features_2_buffer (ResultsCache &results_cache, Environment &env, size_t t_index, size_t c_index); bool save_features_2_buffer_wholeslide( ResultsCache& rescache, - Environment& env, + Environment& env, const LR& r, const std::string& ifpath, - const std::string& mfpath); - std::tuple> save_features_2_apache_wholeslide (Environment & env, const LR & wsi_roi, const std::string & wsi_path); + const std::string& mfpath, + size_t t_index, + size_t c_index); + std::tuple> save_features_2_apache_wholeslide (Environment & env, const LR & wsi_roi, const std::string & wsi_path, size_t t_index, size_t c_index); std::vector get_feature_values ( Environment & env, const FeatureSet & user_selected_features, const Uniqueids & uniqueLabels, const Roidata & roiData, - const Dataset & dataset); + const Dataset & dataset, + size_t t_index, + size_t c_index); std::vector get_feature_values_roi ( Environment & env, const FeatureSet & fset, const LR& r, const std::string & ifpath, - const std::string & mfpath); + const std::string & mfpath, + size_t t_index, + size_t c_index); std::vector get_header (Environment & env); std::string get_arrow_filename(const std::string& output_path, const std::string& default_filename, const SaveOption& arrow_file_type); diff --git a/src/nyx/grayscale_tiff.h b/src/nyx/grayscale_tiff.h index 931c1159..6c6b0caf 100644 --- a/src/nyx/grayscale_tiff.h +++ b/src/nyx/grayscale_tiff.h @@ -14,7 +14,8 @@ #include #include // std::is_signed_v to guard signed->unsigned wraparound in loadTile #include -#include // for INT_MAX +#include // for INT_MAX +#include "ome/ome_tiff_meta.h" // parse_ome_xml -> OmeAxes (OME-TIFF plane->IFD mapping) constexpr size_t STRIP_TILE_HEIGHT = 1024; constexpr size_t STRIP_TILE_WIDTH = 1024; @@ -70,18 +71,38 @@ class NyxusGrayscaleTiffTileLoader : public AbstractTileLoader TIFFGetField(tiff_, TIFFTAG_SAMPLESPERPIXEL, &samplesPerPixel); TIFFGetField(tiff_, TIFFTAG_BITSPERSAMPLE, &(this->bitsPerSample_)); TIFFGetField(tiff_, TIFFTAG_SAMPLEFORMAT, &(this->sampleFormat_)); + // FIX (IO): SAMPLEFORMAT is optional; TIFF's default is 1 (unsigned int). tifffile + // omits it for uint images, leaving sampleFormat_=0 -> "format not supported". Mirror + // the strip loader's defaulting so tiled uint OME-TIFFs read. + if (sampleFormat_ < 1 || sampleFormat_ > 3) + sampleFormat_ = 1; // Test if the file is greyscale - if (samplesPerPixel != 1) + if (samplesPerPixel != 1) { std::stringstream message; message << "Tile Loader ERROR: The file is not greyscale: SamplesPerPixel = " << samplesPerPixel << "."; throw (std::runtime_error(message.str())); } + + // FIX (IO): a TILED multi-plane OME-TIFF carries its (z,c,t) layout in the IFD-0 + // OME-XML just like the strip variant. Parse it so loadTileFromFile addresses the + // right IFD per (z,c,t); a plain single-plane tiled TIFF stays 2D (fullDepth 1). + fullDepth_ = TIFFNumberOfDirectories(tiff_); + char* desc = nullptr; + if (TIFFGetField(tiff_, TIFFTAG_IMAGEDESCRIPTION, &desc) && desc != nullptr) + { + ome_ = Nyxus::parse_ome_xml(desc); + if (ome_.valid) + { + is_ome_ = true; + fullDepth_ = ome_.sizeZ; // depth is SizeZ, NOT the total IFD count (= Z*C*T) + } + } } - else - { - throw (std::runtime_error("Tile Loader ERROR: The file can not be opened.")); + else + { + throw (std::runtime_error("Tile Loader ERROR: The file can not be opened.")); } } @@ -104,11 +125,28 @@ class NyxusGrayscaleTiffTileLoader : public AbstractTileLoader void loadTileFromFile(std::shared_ptr> tile, size_t indexRowGlobalTile, size_t indexColGlobalTile, - size_t indexLayerGlobalTile, + size_t indexLayerGlobalTile, // Z plane + size_t indexChannel, // C plane (OME); 0 for plain 2D TIFF + size_t indexTimeframe, // T plane (OME); 0 for plain 2D TIFF size_t level) override { std::string err; + // FIX (IO): select the (z,c,t) plane's IFD before reading the tile. A tiled OME-TIFF + // stores one plane per IFD in DimensionOrder raster order (ifdForPlane); a plain tiled + // TIFF maps layer->IFD directly. Range-guard OME so an out-of-range plane throws instead + // of reading the wrong IFD or zeros. + if (is_ome_ && (indexLayerGlobalTile >= ome_.sizeZ + || indexChannel >= ome_.sizeC || indexTimeframe >= ome_.sizeT)) + { + throw std::runtime_error("NyxusGrayscaleTiffTileLoader: (z,c,t)=(" + std::to_string(indexLayerGlobalTile) + + "," + std::to_string(indexChannel) + "," + std::to_string(indexTimeframe) + ") out of range"); + } + size_t ifd = is_ome_ ? ome_.ifdForPlane(indexLayerGlobalTile, indexChannel, indexTimeframe) + : indexLayerGlobalTile; + if (TIFFSetDirectory(tiff_, (uint16_t)ifd) != 1) + throw std::runtime_error("NyxusGrayscaleTiffTileLoader: TIFFSetDirectory(ifd=" + std::to_string(ifd) + ") failed"); + // Get ahold of the logical (feature extraction facing) tile buffer from its smart pointer std::vector& tileDataVec = *tile; @@ -224,6 +262,16 @@ class NyxusGrayscaleTiffTileLoader : public AbstractTileLoader /// @return 1 [[nodiscard]] size_t numberPyramidLevels() const override { return 1; } + // FIX (IO): a tiled multi-plane OME-TIFF advertises its Z depth + C/T extents + physical + // spacing from the parsed OME-XML, just like the strip variant (1 / plain values otherwise). + [[nodiscard]] size_t fullDepth([[maybe_unused]] size_t level) const override { return fullDepth_; } + [[nodiscard]] size_t numberChannels() const override { return is_ome_ ? ome_.sizeC : 1; } + [[nodiscard]] size_t fullTimestamps([[maybe_unused]] size_t level) const override { return is_ome_ ? ome_.sizeT : 1; } + [[nodiscard]] double physicalSizeX() const override { return is_ome_ ? ome_.physX : 1.0; } + [[nodiscard]] double physicalSizeY() const override { return is_ome_ ? ome_.physY : 1.0; } + [[nodiscard]] double physicalSizeZ() const override { return is_ome_ ? ome_.physZ : 1.0; } + [[nodiscard]] std::string physicalSizeUnit() const override { return is_ome_ ? ome_.unitXY : std::string(); } + private: #if 0 // A faster implementation is available. Keeping this for records. @@ -359,9 +407,13 @@ class NyxusGrayscaleTiffTileLoader : public AbstractTileLoader size_t fullHeight_ = 0, ///< Full height in pixel fullWidth_ = 0, ///< Full width in pixel + fullDepth_ = 1, ///< Full depth (Z) in planes; >1 for multi-plane OME-TIFF tileHeight_ = 0, ///< Tile height tileWidth_ = 0; ///< Tile width + bool is_ome_ = false; ///< true when IFD-0 carries an OME-XML block + Nyxus::OmeAxes ome_; ///< parsed OME dimensions (drives the (z,c,t)->IFD map) + short sampleFormat_ = 0, ///< Sample format as defined by libtiff bitsPerSample_ = 0; ///< Bit Per Sample as defined by libtiff @@ -430,6 +482,23 @@ class NyxusGrayscaleTiffStripLoader : public AbstractTileLoader fullDepth_ = TIFFNumberOfDirectories(tiff_); + // OME-TIFF: the IFDs are a (z,c,t) rasterization, not a plain Z-stack. + // Parse the OME-XML (IFD-0 ImageDescription) so (z,c,t) map to the right + // IFD and fullDepth reflects SizeZ, not the total page count. + { + char* desc = nullptr; + if (TIFFGetField(tiff_, TIFFTAG_IMAGEDESCRIPTION, &desc) == 1 && desc + && std::string(desc).find(" void loadTileFromFile(std::shared_ptr> tile, size_t indexRowGlobalTile, size_t indexColGlobalTile, - size_t indexLayerGlobalTile, - [[maybe_unused]] size_t level) override + size_t indexLayerGlobalTile, // Z (plane page for non-OME multi-page TIFF) + size_t indexChannel, // C plane (OME-TIFF only) + size_t indexTimeframe, // T plane (OME-TIFF only) + [[maybe_unused]] size_t level) override { // Get ahold of the logical (feature extraction facing) tile buffer from its smart pointer std::vector& tileDataVec = *tile; @@ -484,6 +555,16 @@ class NyxusGrayscaleTiffStripLoader : public AbstractTileLoader buf = _TIFFmalloc(TIFFScanlineSize(tiff_)); + // OME plane indices are computed into an IFD (not iterated), so validate the + // range explicitly — an out-of-range Z would otherwise skip the layer loop + // below and silently return a zero tile. + if (is_ome_ && (indexLayerGlobalTile >= ome_.sizeZ + || indexChannel >= ome_.sizeC || indexTimeframe >= ome_.sizeT)) + { + _TIFFfree(buf); + throw std::runtime_error("NyxusGrayscaleTiffStripLoader: (z,c,t) plane out of range"); + } + size_t startLayer = indexLayerGlobalTile * tileDepth_, endLayer = std::min((indexLayerGlobalTile + 1) * tileDepth_, fullDepth_), @@ -492,9 +573,18 @@ class NyxusGrayscaleTiffStripLoader : public AbstractTileLoader startCol = indexColGlobalTile * tileWidth_, endCol = std::min((indexColGlobalTile + 1) * tileWidth_, fullWidth_); - for (layer = startLayer; layer < endLayer; ++layer) + for (layer = startLayer; layer < endLayer; ++layer) { - TIFFSetDirectory(tiff_, layer); + // OME-TIFF: page = (z,c,t) IFD per DimensionOrder; plain multi-page: page = Z. + size_t ifd = is_ome_ ? ome_.ifdForPlane(layer, indexChannel, indexTimeframe) : layer; + // Check the return so an out-of-range plane throws instead of silently + // leaving the previous directory current and reading the wrong page. + if (TIFFSetDirectory(tiff_, (uint16_t)ifd) != 1) + { + _TIFFfree(buf); + throw std::runtime_error("NyxusGrayscaleTiffStripLoader: TIFFSetDirectory(ifd=" + + std::to_string(ifd) + ") failed (out-of-range plane?)"); + } for (row = startRow; row < endRow; row++) { TIFFReadScanline(tiff_, buf, row); @@ -576,6 +666,18 @@ class NyxusGrayscaleTiffStripLoader : public AbstractTileLoader /// @return Full Depth [[nodiscard]] size_t fullDepth([[maybe_unused]] size_t level) const override { return fullDepth_; } + // FIX: advertise C/T extents from the parsed OME-XML so the volumetric pipeline + // iterates channels/timeframes; non-OME TIFF keeps the single-plane default of 1. + /// @brief Channel (C) extent from OME-XML (1 for plain TIFF) + [[nodiscard]] size_t numberChannels() const override { return is_ome_ ? ome_.sizeC : 1; } + /// @brief Time (T) extent from OME-XML (1 for plain TIFF) + [[nodiscard]] size_t fullTimestamps([[maybe_unused]] size_t level) const override { return is_ome_ ? ome_.sizeT : 1; } + // FIX (IO): physical voxel spacing from OME-XML PhysicalSize* (1.0 for plain TIFF) + [[nodiscard]] double physicalSizeX() const override { return is_ome_ ? ome_.physX : 1.0; } + [[nodiscard]] double physicalSizeY() const override { return is_ome_ ? ome_.physY : 1.0; } + [[nodiscard]] double physicalSizeZ() const override { return is_ome_ ? ome_.physZ : 1.0; } + [[nodiscard]] std::string physicalSizeUnit() const override { return is_ome_ ? ome_.unitXY : std::string(); } + /// @brief Tiff tile width /// @param level Tiff level [not used] /// @return Tile width @@ -692,4 +794,7 @@ class NyxusGrayscaleTiffStripLoader : public AbstractTileLoader sampleFormat_ = 0, ///< Sample format as defined by libtiff bitsPerSample_ = 0; ///< Bit Per Sample as defined by libtiff + bool is_ome_ = false; ///< true when IFD-0 carries an OME-XML block + Nyxus::OmeAxes ome_; ///< parsed OME dimensions (drives the plane->IFD map) + }; diff --git a/src/nyx/image_loader.cpp b/src/nyx/image_loader.cpp index 6e99b9e8..b44479ed 100644 --- a/src/nyx/image_loader.cpp +++ b/src/nyx/image_loader.cpp @@ -1,4 +1,4 @@ -#define NOMINMAX +#define NOMINMAX #include #include "nyxus_dicom_loader.h" #include "image_loader.h" @@ -8,6 +8,7 @@ #include "dirs_and_files.h" #include "helpers/fsystem.h" #include "raw_nifti.h" +#include "ome/format_detect.h" // container-family classification for loader dispatch ImageLoader::ImageLoader() {} @@ -22,9 +23,10 @@ bool ImageLoader::open (SlideProps & p, const FpImageOptions & fpopts) try { - std::string ext = Nyxus::get_big_extension (int_fpath); + // Classify by container family so loader dispatch is identical across all loaders. + Nyxus::ContainerKind fmt = Nyxus::detect_container_family (int_fpath); - if (ext == ".zarr" || ext == ".ome.zarr") + if (fmt == Nyxus::ContainerKind::OmeZarr) { #ifdef OMEZARR_SUPPORT intFL = new NyxusOmeZarrLoader(n_threads, int_fpath); @@ -36,13 +38,13 @@ bool ImageLoader::open (SlideProps & p, const FpImageOptions & fpopts) std::cerr << erm << "\n"; #endif } - else - if (ext == ".dcm" || ext == ".dicom") + else + if (fmt == Nyxus::ContainerKind::Dicom) { #ifdef DICOM_SUPPORT // HU offset base must be the scanned (HU-domain) slide min. In preserve_hu // mode the slope-1 map bypasses fp min/max/dr, so we must NOT take - // fpopts.min_intensity() (defaults to 0 when --fpimgmin is absent) — that + // fpopts.min_intensity() (defaults to 0 when --fpimgmin is absent)  that // would clamp every negative HU to 0. Only the non-HU float path uses the // fp override min. intFL = new NyxusGrayscaleDicomLoader(n_threads, int_fpath, @@ -57,7 +59,7 @@ bool ImageLoader::open (SlideProps & p, const FpImageOptions & fpopts) #endif } else - if (ext == ".nii" || ext == ".nii.gz") + if (fmt == Nyxus::ContainerKind::Nifti) { intFL = new NiftiLoader (int_fpath, (fpopts.preserve_hu() || fpopts.empty()) ? p.min_preroi_inten : (double)fpopts.min_intensity(), // HU offset base = scanned HU-domain slide min; ignore fp min in preserve_hu mode (else negative HU clamps to 0) @@ -71,7 +73,7 @@ bool ImageLoader::open (SlideProps & p, const FpImageOptions & fpopts) double fpmin = p.min_preroi_inten, fpmax = p.max_preroi_inten; // Only the non-HU float path honors the fp override min/max. In preserve_hu - // mode fpmin is the HU offset base and must stay the scanned slide min — + // mode fpmin is the HU offset base and must stay the scanned slide min  // taking fpopts.min_intensity() (0 by default) would clamp every negative // HU to 0. hu_offset() ignores fpmax entirely. if (! fpopts.empty() && ! fpopts.preserve_hu()) @@ -133,9 +135,11 @@ bool ImageLoader::open (SlideProps & p, const FpImageOptions & fpopts) try { - std::string ext = Nyxus::get_big_extension(seg_fpath); + // The mask is classified by the same container family as the intensity, so an + // .ome.zarr mask routes to the Zarr loader (not the TIFF fallback). + Nyxus::ContainerKind fmt = Nyxus::detect_container_family (seg_fpath); - if (ext == ".zarr") + if (fmt == Nyxus::ContainerKind::OmeZarr) { #ifdef OMEZARR_SUPPORT segFL = new NyxusOmeZarrLoader(n_threads, seg_fpath); @@ -143,17 +147,17 @@ bool ImageLoader::open (SlideProps & p, const FpImageOptions & fpopts) std::cout << "This version of Nyxus was not build with OmeZarr support." <(n_threads, seg_fpath); #else - std::cout << "This version of Nyxus was not build with DICOM support." < (seg_fpath); } @@ -236,12 +240,12 @@ bool ImageLoader::load_tile(size_t tile_idx) auto tRow = tile_idx / ntw; auto tCol = tile_idx % ntw; - intFL->loadTileFromFile (ptrI, tRow, tCol, lyr, lvl); + intFL->loadTileFromFile (ptrI, tRow, tCol, lyr, cur_channel, cur_timeframe, lvl); // segmentation loader is not available in wholeslide if (segFL) - segFL->loadTileFromFile (ptrL, tRow, tCol, lyr, lvl); - + segFL->loadTileFromFile (ptrL, tRow, tCol, lyr, cur_channel, cur_timeframe, lvl); + return true; } @@ -250,14 +254,169 @@ bool ImageLoader::load_tile (size_t tile_row, size_t tile_col) if (tile_row >= nth || tile_col >= ntw) return false; - intFL->loadTileFromFile (ptrI, tile_row, tile_col, lyr, lvl); + intFL->loadTileFromFile (ptrI, tile_row, tile_col, lyr, cur_channel, cur_timeframe, lvl); // segmentation loader is not available in wholeslide if (segFL) - segFL->loadTileFromFile (ptrL, tile_row, tile_col, lyr, lvl); + segFL->loadTileFromFile (ptrL, tile_row, tile_col, lyr, cur_channel, cur_timeframe, lvl); + + return true; +} + +void ImageLoader::assemble_volume (AbstractTileLoader* fl, + std::shared_ptr>& ptr, + std::vector& dst, size_t channel, size_t timeframe) +{ + const size_t sliceSize = (size_t)fw * fh; + + // Use THIS loader's own layout: per-plane loaders (OME-Zarr, multi-page TIFF) + // deliver one Z-plane per read (tileDepth==1, tileTimestamps==1) with the + // timeframe chosen by the loadTileFromFile arg; a whole-4D loader (NIfTI) + // delivers the entire x*y*z*t blob ([t][z][y][x]) in one read and ignores the + // timeframe arg, so the requested frame is slabbed out via frameBase. + const size_t ltd = fl->tileDepth (lvl); + const size_t ltt = fl->tileTimestamps (lvl); + const size_t lntd = fl->numberTileDepth (lvl); + const size_t frameStride = ltd * th * tw; + const size_t frameBase = (ltt > 1) ? timeframe * frameStride : 0; + + // FIX: walk the whole tile GRID of each plane, not just tile (0,0). A plane commonly + // spans several tiles/chunks (OME-Zarr chunks are typically 512x512, tiled OME-TIFF + // 256x256), and the old code read tile (0,0) once and then copied fh*fw out of that + // single tile's buffer -- wrong data past the first tile plus an out-of-bounds read. + // Edge tiles are partial, so each tile contributes only its valid [validH x validW]. + const size_t lnth = fl->numberTileHeight (lvl); + const size_t lntw = fl->numberTileWidth (lvl); + + for (size_t lz = 0; lz < lntd; ++lz) + { + for (size_t tr = 0; tr < lnth; ++tr) + for (size_t tc = 0; tc < lntw; ++tc) + { + fl->loadTileFromFile (ptr, tr, tc, lz, channel, timeframe, lvl); + + const size_t row0 = tr * th, col0 = tc * tw; + if (row0 >= fh || col0 >= fw) + continue; + const size_t validH = (std::min) (th, fh - row0), + validW = (std::min) (tw, fw - col0); + + for (size_t pz = 0; pz < ltd && (lz * ltd + pz) < fd; ++pz) + { + const size_t gz = lz * ltd + pz; + for (size_t row = 0; row < validH; ++row) + { + const size_t src = frameBase + (pz * th + row) * tw; + const size_t d = gz * sliceSize + (row0 + row) * fw + col0; + std::copy (ptr->begin() + src, ptr->begin() + src + validW, dst.begin() + d); + } + } + } + } +} + +bool ImageLoader::load_volume (size_t channel, size_t timeframe, size_t mask_timeframe) +{ + cur_channel = channel; + cur_timeframe = timeframe; + + const size_t volSize = (size_t)fw * fh * fd; // the whole X*Y*Z volume + + if (vol_int_.size() != volSize) + vol_int_.resize (volSize); + assemble_volume (intFL, ptrI, vol_int_, channel, timeframe); + + if (segFL != nullptr) + { + if (vol_seg_.size() != volSize) + vol_seg_.resize (volSize); + // FIX (IO): the mask is usually channel-agnostic (a single-channel segmentation that + // applies to every intensity channel), so index into the mask's OWN channels only when + // it actually has that many; otherwise fall back to channel 0. Without this, featurizing + // intensity channel c>0 against a 1-channel mask read the mask out of range and dropped + // the ROI. Mirrors the mask_timeframe separation for the 1-mask : N-intensity case. + size_t mask_channel = (channel < segFL->numberChannels()) ? channel : 0; + // FIX: clamp the mask TIMEFRAME too (callers already pass 0 for a single-timeframe mask, + // but make load_volume self-defending like it is for the channel -- an unclamped t>0 + // would read the TIFF mask past its last IFD and throw uncaught). Symmetric with above. + size_t mask_tf = (mask_timeframe < segFL->fullTimestamps (lvl)) ? mask_timeframe : 0; + assemble_volume (segFL, ptrL, vol_seg_, mask_channel, mask_tf); + } + return true; +} + +void ImageLoader::assemble_one_plane (AbstractTileLoader* fl, + std::shared_ptr>& ptr, + std::vector& dst_plane, size_t gz, size_t channel, size_t timeframe) +{ + // Per-plane loader (tileDepth==1): the global Z index IS the loader's layer, and the + // within-slab plane offset is 0. Mirrors assemble_volume's per-tile copy for one plane. + const size_t ltt = fl->tileTimestamps (lvl); + const size_t frameStride = th * tw; // tileDepth==1 + const size_t frameBase = (ltt > 1) ? timeframe * frameStride : 0; + const size_t lnth = fl->numberTileHeight (lvl); + const size_t lntw = fl->numberTileWidth (lvl); + + for (size_t tr = 0; tr < lnth; ++tr) + for (size_t tc = 0; tc < lntw; ++tc) + { + fl->loadTileFromFile (ptr, tr, tc, gz, channel, timeframe, lvl); + + const size_t row0 = tr * th, col0 = tc * tw; + if (row0 >= fh || col0 >= fw) + continue; + const size_t validH = (std::min) (th, fh - row0), + validW = (std::min) (tw, fw - col0); + + for (size_t row = 0; row < validH; ++row) + { + const size_t src = frameBase + row * tw; + const size_t d = (row0 + row) * fw + col0; + std::copy (ptr->begin() + src, ptr->begin() + src + validW, dst_plane.begin() + d); + } + } +} + +bool ImageLoader::stream_volume_planes (size_t channel, size_t timeframe, size_t mask_timeframe, + const std::function&, const std::vector&)>& sink) +{ + cur_channel = channel; + cur_timeframe = timeframe; + + // A whole-4D loader (NIfTI) hands over the entire cube in one read; there is nothing to + // stream plane-by-plane, so decline and let the caller fail loudly. + if (intFL->tileDepth (lvl) != 1) + return false; + if (segFL != nullptr && segFL->tileDepth (lvl) != 1) + return false; + // Mask is usually channel-agnostic and single-timeframe; clamp like load_volume does + size_t mask_channel = 0, mask_tf = 0; + if (segFL != nullptr) + { + mask_channel = (channel < segFL->numberChannels()) ? channel : 0; + mask_tf = (mask_timeframe < segFL->fullTimestamps (lvl)) ? mask_timeframe : 0; + } + + const size_t sliceSize = (size_t) fw * fh; + std::vector intPlane (sliceSize), segPlane (segFL != nullptr ? sliceSize : 0); + + for (size_t gz = 0; gz < fd; ++gz) + { + std::fill (intPlane.begin(), intPlane.end(), 0u); + assemble_one_plane (intFL, ptrI, intPlane, gz, channel, timeframe); + + if (segFL != nullptr) + { + std::fill (segPlane.begin(), segPlane.end(), 0u); + assemble_one_plane (segFL, ptrL, segPlane, gz, mask_channel, mask_tf); + } + + sink (gz, intPlane, segPlane); + } return true; } + const std::vector& ImageLoader::get_int_tile_buffer() { return *ptrI; diff --git a/src/nyx/image_loader.h b/src/nyx/image_loader.h index 6802fb3a..7beca029 100644 --- a/src/nyx/image_loader.h +++ b/src/nyx/image_loader.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -19,6 +20,28 @@ class ImageLoader void close(); bool load_tile (size_t tile_idx); bool load_tile (size_t tile_row, size_t tile_col); + + // Assemble the whole X*Y*Z volume for one (channel, timeframe) into an + // internal buffer by looping over Z-planes. This is what lets plane-by-plane + // loaders (OME-Zarr, multi-page/OME-TIFF) feed the volumetric pipeline, which + // otherwise assumes the whole volume arrives in one read (the NIfTI model). + // The mask may live on a different timeframe than the intensity (the 1-mask : + // N-intensity case), so `mask_timeframe` is separable; the 2-arg overload uses + // the same frame for both. + bool load_volume (size_t channel, size_t timeframe, size_t mask_timeframe); + bool load_volume (size_t channel, size_t timeframe) { return load_volume(channel, timeframe, timeframe); } + const std::vector& get_int_volume_buffer() const { return vol_int_; } + const std::vector& get_seg_volume_buffer() const { return vol_seg_; } + + // Stream the whole X*Y*Z volume for one (channel, timeframe) WITHOUT materializing it: + // one Z-plane is assembled into a reused buffer and handed to 'sink', so peak memory is + // two planes (intensity + mask) rather than the whole cube. This is what lets an oversized + // volumetric ROI be featurized out-of-core. Applies the same mask channel/timeframe clamp as + // load_volume (the mask is usually channel-agnostic). Returns false for a whole-4D loader + // (NIfTI, tileDepth>1) which delivers the entire cube in one read and cannot be streamed here. + bool stream_volume_planes (size_t channel, size_t timeframe, size_t mask_timeframe, + const std::function& int_plane, const std::vector& seg_plane)>& sink); + const std::vector& get_int_tile_buffer(); const std::vector& get_seg_tile_buffer(); const std::shared_ptr>& get_seg_tile_sptr(); @@ -64,5 +87,26 @@ class ImageLoader int lvl = 0, // Pyramid level lyr = 0; // Layer + + // Channel (C) / timeframe (T) plane that load_tile() reads. Set by load_volume() / + // stream_volume_planes(), which take the plane as an argument; 0/0 (the single-channel, + // single-timepoint case) until then. + size_t cur_channel = 0, + cur_timeframe = 0; + + // Whole-volume (X*Y*Z) assembly buffers filled by load_volume() + std::vector vol_int_, vol_seg_; + + // Assemble one loader's X*Y*Z volume (for the given channel/timeframe) into dst, + // honoring that loader's own tileDepth/tileTimestamps (per-plane vs whole-4D). + void assemble_volume (AbstractTileLoader* fl, + std::shared_ptr>& ptr, + std::vector& dst, size_t channel, size_t timeframe); + + // Assemble a single global Z-plane 'gz' (X*Y) into 'dst_plane' for a per-plane loader + // (tileDepth==1). The streaming counterpart of assemble_volume's inner plane fill. + void assemble_one_plane (AbstractTileLoader* fl, + std::shared_ptr>& ptr, + std::vector& dst_plane, size_t gz, size_t channel, size_t timeframe); }; diff --git a/src/nyx/image_loader1x.cpp b/src/nyx/image_loader1x.cpp index 147bf7f4..b6d5b023 100644 --- a/src/nyx/image_loader1x.cpp +++ b/src/nyx/image_loader1x.cpp @@ -1,4 +1,4 @@ -#include +#include #define NOMINMAX @@ -9,6 +9,7 @@ #include "nyxus_dicom_loader.h" #include "dirs_and_files.h" #include "helpers/fsystem.h" +#include "ome/format_detect.h" // container-family classification for loader dispatch ImageLoader1x::ImageLoader1x() {} @@ -18,7 +19,11 @@ bool ImageLoader1x::open (const std::string& fpath, const FpImageOptions & fpopt try { - if (fs::path(fpath).extension() == ".zarr") + // Classify by container family for parity with the other loaders. This loader carries no + // NIfTI backend, so a NIfTI path falls through to the TIFF branch. + Nyxus::ContainerKind fmt = Nyxus::detect_container_family (fpath); + + if (fmt == Nyxus::ContainerKind::OmeZarr) { #ifdef OMEZARR_SUPPORT FL = std::make_unique>(n_threads, fpath); @@ -26,7 +31,7 @@ bool ImageLoader1x::open (const std::string& fpath, const FpImageOptions & fpopt std::cout << "This version of Nyxus was not build with OmeZarr support." <>(n_threads, fpath); #else @@ -95,7 +100,7 @@ bool ImageLoader1x::load_tile(size_t tile_idx) auto row = tile_idx / ntw; auto col = tile_idx % ntw; - FL->loadTileFromFile(ptr, row, col, lyr, lvl); + FL->loadTileFromFile(ptr, row, col, lyr, 0/*channel*/, 0/*timeframe*/, lvl); return true; } @@ -105,7 +110,7 @@ bool ImageLoader1x::load_tile(size_t tile_row, size_t tile_col) if (tile_row >= nth || tile_col >= ntw) return false; - FL->loadTileFromFile(ptr, tile_row, tile_col, lyr, lvl); + FL->loadTileFromFile(ptr, tile_row, tile_col, lyr, 0/*channel*/, 0/*timeframe*/, lvl); return true; } diff --git a/src/nyx/nyxus_dicom_loader.h b/src/nyx/nyxus_dicom_loader.h index d4cb9d43..8c29953f 100644 --- a/src/nyx/nyxus_dicom_loader.h +++ b/src/nyx/nyxus_dicom_loader.h @@ -135,7 +135,9 @@ class NyxusGrayscaleDicomLoader : public AbstractTileLoader size_t indexRowGlobalTile, size_t indexColGlobalTile, size_t indexLayerGlobalTile, - [[maybe_unused]] size_t level) override + [[maybe_unused]] size_t indexChannel, // DICOM grayscale: single channel + [[maybe_unused]] size_t indexTimeframe, // DICOM: frames are WSI tiles, not time + [[maybe_unused]] size_t level) override { // Get ahold of the logical (feature extraction facing) tile buffer from its smart pointer std::vector& tileDataVec = *tile; diff --git a/src/nyx/ome/format_detect.cpp b/src/nyx/ome/format_detect.cpp new file mode 100644 index 00000000..45ca849d --- /dev/null +++ b/src/nyx/ome/format_detect.cpp @@ -0,0 +1,97 @@ +#define NOMINMAX + +#include "format_detect.h" + +#include +#include +#include + +#include + +#include "../helpers/fsystem.h" + +namespace Nyxus +{ + // Big-extension classification, with TIFF flavors left undifferentiated. This is all the + // loader dispatch needs, and it touches no file content. + static ContainerKind kind_of_extension (const std::string& path) + { + std::string ext = Nyxus::get_big_extension(path); + std::transform(ext.begin(), ext.end(), ext.begin(), + [](unsigned char c) { return (char)std::tolower(c); }); + + if (ext == ".zarr" || ext == ".ome.zarr") + return ContainerKind::OmeZarr; + if (ext == ".dcm" || ext == ".dicom") + return ContainerKind::Dicom; + if (ext == ".nii" || ext == ".nii.gz") + return ContainerKind::Nifti; + + // Everything else is a TIFF flavor (.tif/.tiff/.ome.tif/.ome.tiff/.tf2/.tf8/.btf). + return ContainerKind::TiffPlain; + } + + ContainerKind detect_container_family (const std::string& path) + { + return kind_of_extension (path); + } + + // Does this TIFF carry an OME-XML block in IFD-0's ImageDescription? + static bool tiff_is_ome(const std::string& path) + { + // silence libtiff's stderr chatter on non-OME / odd TIFFs during the sniff + TIFFErrorHandler oldE = TIFFSetErrorHandler(nullptr); + TIFFErrorHandler oldW = TIFFSetWarningHandler(nullptr); + bool ome = false; + TIFF* t = TIFFOpen(path.c_str(), "r"); + if (t) + { + char* desc = nullptr; // libtiff-owned, do not free + if (TIFFGetField(t, TIFFTAG_IMAGEDESCRIPTION, &desc) == 1 && desc) + { + std::string d(desc); + ome = d.find("(f)), std::istreambuf_iterator()); + if (content.find("multiscales") != std::string::npos) + return true; + } + return false; + } + + InputFormat detect_input_format(const std::string& path) + { + switch (kind_of_extension (path)) + { + case ContainerKind::OmeZarr: + return { ContainerKind::OmeZarr, zarr_has_multiscales(path) }; + case ContainerKind::Dicom: + return { ContainerKind::Dicom, false }; + case ContainerKind::Nifti: + return { ContainerKind::Nifti, false }; + default: + // TIFF: only the content sniff can tell an OME-TIFF from a plain one + return tiff_is_ome(path) ? InputFormat{ ContainerKind::OmeTiff, true } + : InputFormat{ ContainerKind::TiffPlain, false }; + } + } +} diff --git a/src/nyx/ome/format_detect.h b/src/nyx/ome/format_detect.h new file mode 100644 index 00000000..7d084038 --- /dev/null +++ b/src/nyx/ome/format_detect.h @@ -0,0 +1,31 @@ +#pragma once + +// Single source of truth for "what kind of file is this?" used by all three +// loader dispatch sites (image_loader, raw_image_loader, image_loader1x). Adds +// the OME content-sniff that is absent today and unifies the extension handling +// (image_loader1x currently uses single-extension matching and cannot see +// .ome.zarr / .nii.gz). STL + libtiff + only. + +#include + +namespace Nyxus +{ + enum class ContainerKind { TiffPlain, OmeTiff, OmeZarr, Dicom, Nifti, Unknown }; + + struct InputFormat + { + ContainerKind kind = ContainerKind::Unknown; + bool is_ome = false; // OME metadata actually present (OME-XML / NGFF multiscales) + }; + + // Which loader backend handles this path, from the big-extension alone -- TIFF flavors are + // reported as TiffPlain because a single TIFF loader serves both. This is what the loader + // dispatch sites use: it opens nothing, so it stays cheap even though ImageLoader::open() + // is called once per oversized ROI. Never throws. + ContainerKind detect_container_family (const std::string& path); + + // Classify by big-extension, then content-sniff: OME-XML in TIFF IFD-0 + // ImageDescription, or NGFF `multiscales` in a Zarr group. Distinguishes OmeTiff from + // TiffPlain and reports is_ome, at the cost of opening the file. Never throws. + InputFormat detect_input_format(const std::string& path); +} diff --git a/src/nyx/ome/ome_axes.h b/src/nyx/ome/ome_axes.h new file mode 100644 index 00000000..fbb0054d --- /dev/null +++ b/src/nyx/ome/ome_axes.h @@ -0,0 +1,227 @@ +#pragma once + +// ----------------------------------------------------------------------------- +// OmeAxes — one format-agnostic dimension descriptor for native 3D/4D/5D OME +// input. Both container families populate this from real metadata: +// - OME-TIFF : parse_ome_xml() (ome_tiff_meta.h) — OME-XML block +// - OME-Zarr : parse_ome_zarr() (ome_zarr_meta.h) — multiscales/axes/datasets +// Everything downstream (loaders, SlideProps, output schema) consumes OmeAxes +// instead of assuming C=T=1 and a hard-coded TCZYX axis order. +// +// This header is STL-only so it compiles in every build configuration. +// ----------------------------------------------------------------------------- + +#include +#include +#include +#include + +namespace Nyxus +{ + // Role of an axis, mirroring the OME-NGFF 'type' field (space|channel|time). + enum class AxisKind { Space, Channel, Time }; + + // Resolved pixel/element type (consolidates the ad-hoc dtype codes that + // omezarr.h / raw_omezarr.h and the TIFF sampleFormat switch use today). + enum class PixelType { UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64, Unknown }; + + inline unsigned bits_of(PixelType t) + { + switch (t) + { + case PixelType::UInt8: case PixelType::Int8: return 8; + case PixelType::UInt16: case PixelType::Int16: return 16; + case PixelType::UInt32: case PixelType::Int32: case PixelType::Float32: return 32; + case PixelType::UInt64: case PixelType::Int64: case PixelType::Float64: return 64; + default: return 0; + } + } + inline bool is_float(PixelType t) { return t == PixelType::Float32 || t == PixelType::Float64; } + inline bool is_signed(PixelType t) + { + return t == PixelType::Int8 || t == PixelType::Int16 || t == PixelType::Int32 || t == PixelType::Int64 || is_float(t); + } + + // OME 'Type' attribute (OME-XML) -> PixelType. + inline PixelType pixel_type_from_ome_type(const std::string& s) + { + if (s == "uint8") return PixelType::UInt8; + if (s == "uint16") return PixelType::UInt16; + if (s == "uint32") return PixelType::UInt32; + if (s == "uint64") return PixelType::UInt64; + if (s == "int8") return PixelType::Int8; + if (s == "int16") return PixelType::Int16; + if (s == "int32") return PixelType::Int32; + if (s == "int64") return PixelType::Int64; + if (s == "float") return PixelType::Float32; + if (s == "double") return PixelType::Float64; + return PixelType::Unknown; + } + + // Zarr dtype -> PixelType. Accepts v2 numpy strings ("i4", "|u1", ...) + // and v3 names ("uint16", "int32", "float32", ...). + inline PixelType pixel_type_from_zarr_dtype(const std::string& in) + { + std::string s = in; + // strip a leading byte-order char for the v2 form + if (!s.empty() && (s[0] == '<' || s[0] == '>' || s[0] == '|' || s[0] == '=')) + s = s.substr(1); + if (s == "u1" || s == "uint8") return PixelType::UInt8; + if (s == "u2" || s == "uint16") return PixelType::UInt16; + if (s == "u4" || s == "uint32") return PixelType::UInt32; + if (s == "u8" || s == "uint64") return PixelType::UInt64; + if (s == "i1" || s == "int8") return PixelType::Int8; + if (s == "i2" || s == "int16") return PixelType::Int16; + if (s == "i4" || s == "int32") return PixelType::Int32; + if (s == "i8" || s == "int64") return PixelType::Int64; + if (s == "f2") return PixelType::Float32; // half -> promote to float32 + if (s == "f4" || s == "float32" || s == "float") return PixelType::Float32; + if (s == "f8" || s == "float64" || s == "double") return PixelType::Float64; + return PixelType::Unknown; + } + + // One axis in on-disk (storage) order. + struct OmeAxis + { + AxisKind kind = AxisKind::Space; + char label = 'X'; // one of 'X','Y','Z','C','T' + std::size_t size = 1; // extent along this axis + double physical = 1.0; // voxel spacing / frame interval; 1.0 if unknown + std::string unit; // e.g. "micrometer","second"; empty if unknown + }; + + // One resolution level of a pyramid (level 0 == full-res). + struct OmePyramidLevel + { + std::size_t index = 0; + std::size_t sizeX = 0, sizeY = 0, sizeZ = 1; // 0 == extent not yet resolved for this sub-level + double scaleX = 1, scaleY = 1, scaleZ = 1; // coordinateTransformations 'scale' + std::string path; // Zarr dataset path ("0","1",...) / TIFF SubIFD ordinal + }; + + struct OmeAxes + { + // Sizes resolved by axis role (independent of on-disk order). + std::size_t sizeX = 1, sizeY = 1, sizeZ = 1, sizeC = 1, sizeT = 1; + + // On-disk axis order (numpy/slowest-first), e.g. "TCZYX" or "CTZYX" or "ZYX". + std::string storageOrder; + // OME DimensionOrder (fastest-first, always begins "XY"), e.g. "XYZCT". + std::string omeDimensionOrder; + std::vector storageAxes; // in storageOrder order + + // Physical calibration (convenience mirror of the space/time axes). + double physX = 1, physY = 1, physZ = 1, timeIncrement = 1; + std::string unitXY, unitZ, unitT; + + PixelType dtype = PixelType::UInt16; + unsigned bitsPerSample = 16; + + std::vector levels; // >=1; levels[0] full-res + std::vector channelNames; // optional (OME 'Channel'/omero) + + // Explicit plane-ordinal -> IFD map from the OME elements, indexed by the + // canonical DimensionOrder ordinal (see canonicalPlaneOrdinal). Empty when the OME-XML + // carries no TiffData (or none that maps same-file planes), in which case the plane + // ordinal IS the IFD (the OME default: planes stored contiguously from IFD 0 in + // DimensionOrder). A writer that offsets the first IFD or reorders planes populates this. + std::vector planeToIfd; + + // True when a names planes in another file (a child): companion / + // multi-file OME-TIFF, which this reader does not resolve. Those planes are left at the + // canonical fallback; the flag lets callers detect the unsupported layout. + bool multiFileTiff = false; + + bool valid = false; // set true by a successful parse + + // Position of a label in storageAxes; -1 if absent. + int storageIndexOf(char lbl) const + { + for (std::size_t i = 0; i < storageAxes.size(); ++i) + if (storageAxes[i].label == lbl) return static_cast(i); + return -1; + } + + // The plane's ordinal under the OME DimensionOrder (default rasterization: the axes + // after XY, fastest to slowest). E.g. XYZCT -> z + c*sizeZ + t*sizeZ*sizeC. This is the + // IFD only when planes are stored contiguously from IFD 0; TiffData can say otherwise. + std::size_t canonicalPlaneOrdinal(std::size_t z, std::size_t c, std::size_t t) const + { + std::size_t idx = 0, stride = 1; + for (char ax : omeDimensionOrder) + { + if (ax == 'X' || ax == 'Y') continue; + std::size_t coord = (ax == 'Z') ? z : (ax == 'C') ? c : t; + std::size_t sz = (ax == 'Z') ? sizeZ : (ax == 'C') ? sizeC : sizeT; + idx += coord * stride; + stride *= sz; + } + return idx; + } + + // Map a (z,c,t) plane to the physical IFD holding its pixels. Honors an explicit + // plane->IFD map when present (writers that start at a non-zero IFD or + // reorder planes); otherwise falls back to the canonical ordinal. + std::size_t ifdForPlane(std::size_t z, std::size_t c, std::size_t t) const + { + std::size_t ord = canonicalPlaneOrdinal(z, c, t); + if (!planeToIfd.empty() && ord < planeToIfd.size()) + return planeToIfd[ord]; + return ord; + } + + std::size_t numberPyramidLevels() const { return levels.empty() ? 1 : levels.size(); } + bool isVolumetric() const { return sizeZ > 1; } + bool isMultiChannel() const { return sizeC > 1; } + bool isTimeSeries() const { return sizeT > 1; } + }; + + // ---- small shared helpers ---------------------------------------------- + + // Length-unit -> micrometer scale factor (OME-XML/NGFF use the full UDUNITS-2 names; + // tolerate common short forms too, since not every writer follows the spec exactly). + // Returns 1.0 (no-op) for an empty/unrecognized unit -- an unrecognized unit is left + // as-is rather than silently misscaled, and the caller keeps the original unit string + // in that case so the mismatch stays visible instead of masquerading as "micrometer". + inline double unit_scale_to_micrometer(const std::string& unit) + { + if (unit == "meter" || unit == "metre" || unit == "m") return 1e6; + if (unit == "centimeter" || unit == "centimetre" || unit == "cm") return 1e4; + if (unit == "millimeter" || unit == "millimetre" || unit == "mm") return 1e3; + if (unit == "micrometer" || unit == "micrometre" || unit == "micron" || unit == "um" || unit == "\xC2\xB5m") return 1.0; + if (unit == "nanometer" || unit == "nanometre" || unit == "nm") return 1e-3; + if (unit == "angstrom" || unit == "\xC3\x85") return 1e-4; + if (unit == "picometer" || unit == "picometre" || unit == "pm") return 1e-6; + return 0.0; // unrecognized (includes "" == uncalibrated) + } + + // Canonicalize a physical-size value + its declared unit to micrometer in place. + // A recognized non-micrometer unit is converted and relabeled "micrometer"; an + // unrecognized/empty unit is left untouched (value AND unit string both unchanged). + inline void canonicalize_to_micrometer(double& physical, std::string& unit) + { + double scale = unit_scale_to_micrometer(unit); + if (scale == 0.0) + return; + physical *= scale; + unit = "micrometer"; + } + + inline AxisKind axis_kind_of(char label) + { + return (label == 'C') ? AxisKind::Channel : (label == 'T') ? AxisKind::Time : AxisKind::Space; + } + + // OME DimensionOrder (fastest->slowest) from an on-disk storage order + // (slowest-first). For a C-order array the last axis varies fastest, so the + // OME order is the reverse of storageOrder, padded with any absent axes kept + // in OME's canonical relative order so the result is always a legal value. + inline std::string ome_dimension_order_from_storage(const std::string& storageOrder) + { + std::string rev(storageOrder.rbegin(), storageOrder.rend()); + for (char ax : std::string("XYZCT")) + if (rev.find(ax) == std::string::npos) + rev.push_back(ax); + return rev; + } +} diff --git a/src/nyx/ome/ome_tiff_meta.cpp b/src/nyx/ome/ome_tiff_meta.cpp new file mode 100644 index 00000000..b35df94a --- /dev/null +++ b/src/nyx/ome/ome_tiff_meta.cpp @@ -0,0 +1,250 @@ +#include "ome_tiff_meta.h" + +#include +#include +#include +#include +#include +#include + +namespace Nyxus +{ + // Extract the start-tag text "" of the first element called `name`. + // Returns "" if not found. Matches a name boundary so "Pixels" does not match + // a hypothetical "PixelsFoo". + static std::string element_start_tag(const std::string& xml, const std::string& name) + { + const std::string open = "<" + name; + std::size_t p = 0; + while ((p = xml.find(open, p)) != std::string::npos) + { + char after = (p + open.size() < xml.size()) ? xml[p + open.size()] : '\0'; + if (after == ' ' || after == '\t' || after == '\n' || after == '\r' || after == '>' || after == '/') + { + std::size_t gt = xml.find('>', p); + if (gt == std::string::npos) return ""; + return xml.substr(p, gt - p + 1); + } + p += open.size(); + } + return ""; + } + + // Read attribute `name` from a single start-tag string. Boundary-checked so + // "SizeX" does not match inside "PhysicalSizeX". Handles ' and " quoting. + static bool get_attr(const std::string& tag, const std::string& name, std::string& out) + { + std::size_t pos = 0; + while ((pos = tag.find(name, pos)) != std::string::npos) + { + bool ok_before = (pos == 0) || std::isspace((unsigned char)tag[pos - 1]); + std::size_t q = pos + name.size(); + while (q < tag.size() && std::isspace((unsigned char)tag[q])) ++q; + if (ok_before && q < tag.size() && tag[q] == '=') + { + ++q; + while (q < tag.size() && std::isspace((unsigned char)tag[q])) ++q; + if (q < tag.size() && (tag[q] == '"' || tag[q] == '\'')) + { + char quote = tag[q]; + std::size_t start = q + 1, end = tag.find(quote, start); + if (end != std::string::npos) { out = tag.substr(start, end - start); return true; } + } + } + pos += name.size(); + } + return false; + } + + OmeAxes parse_ome_xml(const std::string& xml) + { + OmeAxes ax; + const std::string pix = element_start_tag(xml, "Pixels"); + if (pix.empty()) + return ax; // valid stays false + + auto s = [&](const char* n, const std::string& def) { std::string v; return get_attr(pix, n, v) ? v : def; }; + // Size reader (std::from_chars): absent / non-numeric / negative / overflow + // -> default; "0" -> 1. OME extents are >=1; from_chars rejects a leading + // '-' and overflow via errc, so no strtoull wrap or divide-by-zero leaks out. + auto i = [&](const char* n, std::size_t def) -> std::size_t { + std::string v; if (!get_attr(pix, n, v)) return def; + std::size_t r = 0; + auto [ptr, ec] = std::from_chars(v.data(), v.data() + v.size(), r); + if (ec != std::errc() || ptr == v.data()) return def; + return r < 1 ? std::size_t(1) : r; + }; + // Double reader: absent / non-numeric / non-finite -> default. libc++ provides + // std::from_chars for integral types only (its floating-point overloads are + // deleted or constrained to is_integral), so the double form does not compile + // there; a stream imbued with the classic locale reads the same value on every + // standard library and, unlike strtod, ignores the host's LC_NUMERIC, which a + // process embedding the library may have set to a comma-decimal locale. + // "nan"/"inf" prefixes (e.g. "NaNsense") still parse, so the isfinite check + // keeps those out of physical-calibration math. + auto d = [&](const char* n, double def) -> double { + std::string v; if (!get_attr(pix, n, v)) return def; + std::istringstream is(v); + is.imbue(std::locale::classic()); + double r = 0; + is >> r; + if (is.fail() || !std::isfinite(r)) return def; + return r; + }; + + ax.sizeX = i("SizeX", 1); + ax.sizeY = i("SizeY", 1); + ax.sizeZ = i("SizeZ", 1); + ax.sizeC = i("SizeC", 1); + ax.sizeT = i("SizeT", 1); + + // DimensionOrder must be a permutation of the 5 canonical axes; a missing + // or malformed value falls back to the OME default so storageOrder stays sane. + std::string dord = s("DimensionOrder", "XYZCT"); + bool perm = dord.size() == 5; + for (char c : std::string("XYZCT")) + if (std::count(dord.begin(), dord.end(), c) != 1) perm = false; + ax.omeDimensionOrder = perm ? dord : "XYZCT"; + + // Explicit plane->IFD mapping from (OME-XML). Absent it, planes are stored + // contiguously from IFD 0 in DimensionOrder (the ifdForPlane default), which is what + // tifffile's single "" also means -- so this only changes + // behavior for writers that start at a non-zero IFD or reorder planes (e.g. bioformats + // per-plane blocks, or a multi-image container). Each block maps PlaneCount consecutive + // DimensionOrder planes starting at (FirstZ,FirstC,FirstT) to consecutive IFDs from IFD. + { + const std::size_t totalPlanes = ax.sizeZ * ax.sizeC * ax.sizeT; + // integer attribute of a start-tag, with default (same guards as `i` above) + auto iat = [](const std::string& tag, const char* n, std::size_t def) -> std::size_t { + std::string v; if (!get_attr(tag, n, v)) return def; + std::size_t r = 0; + auto [ptr, ec] = std::from_chars(v.data(), v.data() + v.size(), r); + return (ec != std::errc() || ptr == v.data()) ? def : r; + }; + + bool anyTiffData = false, sawMultiFile = false; + std::vector map; // identity-initialized on first same-file block + std::size_t tp = 0; + while ((tp = xml.find("' || after == '/')) + { tp += 9; continue; } + std::size_t gt = xml.find('>', tp); + if (gt == std::string::npos) break; + const std::string tag = xml.substr(tp, gt - tp + 1); + anyTiffData = true; + + // A with a child names a plane in ANOTHER file (companion / + // multi-file OME-TIFF) -- unsupported here. Detect it (tag not self-closed AND a + // ) and skip mapping it, leaving those planes at + // the canonical fallback rather than pointing them at a wrong local IFD. + bool selfClosed = (gt > 0 && xml[gt - 1] == '/'); + if (!selfClosed) + { + std::size_t close = xml.find("", gt); + std::size_t uuid = xml.find(" 0) + { map.resize(totalPlanes); for (std::size_t k = 0; k < totalPlanes; ++k) map[k] = k; } + for (std::size_t k = 0; k < count && (startOrd + k) < totalPlanes; ++k) + map[startOrd + k] = ifd0 + k; + tp = gt + 1; + } + ax.multiFileTiff = sawMultiFile; + // Only keep a non-identity map (a plain identity means canonical -> leave empty so + // the common no-/canonical-TiffData path stays allocation-free and obviously canonical). + if (anyTiffData && !map.empty()) + { + bool identity = true; + for (std::size_t k = 0; k < map.size(); ++k) if (map[k] != k) { identity = false; break; } + if (!identity) ax.planeToIfd = std::move(map); + } + } + + ax.dtype = pixel_type_from_ome_type(s("Type", "uint16")); + if (ax.dtype == PixelType::Unknown) ax.dtype = PixelType::UInt16; + ax.bitsPerSample = bits_of(ax.dtype); + + ax.physX = d("PhysicalSizeX", 1.0); + ax.physY = d("PhysicalSizeY", 1.0); + ax.physZ = d("PhysicalSizeZ", 1.0); + ax.timeIncrement = d("TimeIncrement", 1.0); + ax.unitXY = s("PhysicalSizeXUnit", ""); + ax.unitZ = s("PhysicalSizeZUnit", ""); + ax.unitT = s("TimeIncrementUnit", ""); + + // Canonicalize each axis to micrometer using ITS OWN declared unit (X/Y share + // unitXY, Z has its own unitZ) -- a file that declares Z in a different unit than + // X/Y (or either in nm/mm/etc.) previously reported raw, uncomparable values under + // a unit label that only ever reflected X/Y. No-op for an already-micrometer or + // unrecognized/uncalibrated unit. + std::string unitY = ax.unitXY; + canonicalize_to_micrometer(ax.physX, ax.unitXY); + canonicalize_to_micrometer(ax.physY, unitY); + canonicalize_to_micrometer(ax.physZ, ax.unitZ); + + // On-disk order = reverse(DimensionOrder) with singleton axes dropped + // (planes are XY, so the result always ends in "YX"). + auto size_of = [&](char c) -> std::size_t { + switch (c) { case 'X': return ax.sizeX; case 'Y': return ax.sizeY; + case 'Z': return ax.sizeZ; case 'C': return ax.sizeC; case 'T': return ax.sizeT; } + return 1; + }; + std::string rev(ax.omeDimensionOrder.rbegin(), ax.omeDimensionOrder.rend()); + for (char c : rev) + if (c == 'X' || c == 'Y' || size_of(c) > 1) + ax.storageOrder.push_back(c); + + for (char c : ax.storageOrder) + { + OmeAxis a; + a.label = c; + a.kind = axis_kind_of(c); + a.size = size_of(c); + if (c == 'X') { a.physical = ax.physX; a.unit = ax.unitXY; } + else if (c == 'Y') { a.physical = ax.physY; a.unit = ax.unitXY; } + else if (c == 'Z') { a.physical = ax.physZ; a.unit = ax.unitZ; } + else if (c == 'T') { a.physical = ax.timeIncrement; a.unit = ax.unitT; } + else /* C */ { a.physical = 1.0; } + ax.storageAxes.push_back(a); + } + + // Single full-res level (SubIFDs pyramid parsing not implemented yet). + OmePyramidLevel lv; + lv.index = 0; + lv.sizeX = ax.sizeX; lv.sizeY = ax.sizeY; lv.sizeZ = ax.sizeZ; + lv.scaleX = ax.physX; lv.scaleY = ax.physY; lv.scaleZ = ax.physZ; + lv.path = "0"; + ax.levels.push_back(lv); + + // Optional channel names. + std::size_t cp = 0; + while ((cp = xml.find("') + { + std::size_t gt = xml.find('>', cp); + if (gt == std::string::npos) break; + std::string ctag = xml.substr(cp, gt - cp + 1); + std::string nm; + if (get_attr(ctag, "Name", nm)) ax.channelNames.push_back(nm); + cp = gt + 1; + } + else cp += 8; + } + + ax.valid = true; + return ax; + } +} diff --git a/src/nyx/ome/ome_tiff_meta.h b/src/nyx/ome/ome_tiff_meta.h new file mode 100644 index 00000000..d98cef45 --- /dev/null +++ b/src/nyx/ome/ome_tiff_meta.h @@ -0,0 +1,20 @@ +#pragma once + +// OME-TIFF metadata -> OmeAxes. Parses the dimensional attributes of the OME-XML +// element (SizeX/Y/Z/C/T, DimensionOrder, Type, PhysicalSize*, +// TimeIncrement) plus names. STL-only. +// +// Resolves dimensions + calibration + dtype + the plane->IFD map +// (same-file; a multi-file block sets OmeAxes::multiFileTiff and is left +// at the canonical fallback). SubIFDs pyramids are not parsed yet; levels here is +// a single full-res level. + +#include +#include "ome_axes.h" + +namespace Nyxus +{ + // Parse an OME-XML document (the string from a TIFF IFD-0 ImageDescription). + // Returns an OmeAxes with .valid==false if no element is found. + OmeAxes parse_ome_xml(const std::string& omeXml); +} diff --git a/src/nyx/ome/ome_zarr_layout.h b/src/nyx/ome/ome_zarr_layout.h new file mode 100644 index 00000000..b8514923 --- /dev/null +++ b/src/nyx/ome/ome_zarr_layout.h @@ -0,0 +1,172 @@ +#pragma once + +// Shared OME-Zarr array layout resolution for the two Zarr loaders (omezarr.h's +// NyxusOmeZarrLoader and raw_omezarr.h's RawOmezarrLoader). Both open the same containers and +// need the same answers -- which storage dimension carries which axis role, the X/Y/Z extents +// and chunking, the C/T extents, the physical voxel spacing and the pixel type -- so the +// resolution lives here once instead of being duplicated in both headers. +// +// Guarded by OMEZARR_SUPPORT because it depends on z5 (and, through ome_zarr_meta.h, on +// nlohmann::json vendored with z5). + +#ifdef OMEZARR_SUPPORT + +#include +#include +#include +#include + +#include "nlohmann/json.hpp" +#include "z5/types/types.hxx" + +#include "ome_axes.h" +#include "ome_zarr_meta.h" + +namespace Nyxus +{ + // z5 Datatype -> the " read as uint16 + } + } + + /// @brief Resolve the array layout from NGFF metadata + the level-0 array's shape/chunking. + /// Prefers the 'axes' block (so the on-disk order is honored rather than assumed to be + /// TCZYX); falls back to a rank-safe positional mapping when 'axes' is absent or unusable. + /// Throws if the metadata is self-inconsistent (axis count vs array rank) or if X/Y cannot + /// be resolved -- either would make the read index out of bounds. + inline ZarrLayout resolve_zarr_layout ( + const nlohmann::json& file_attributes, + const std::vector& level0Shape, + const std::vector& chunkShape, + z5::types::Datatype dt) + { + ZarrLayout L; + + const std::string dtype_str = zarr_dtype_string_of (dt); + L.dtype = pixel_type_from_zarr_dtype (dtype_str); + L.bits_per_sample = (short) bits_of (L.dtype); + L.data_format = zarr_read_format_of (L.dtype); + L.fp_pixels = is_float (L.dtype); + + OmeAxes axes = parse_ome_zarr (file_attributes, level0Shape, dtype_str); + if (axes.valid) + { + // Reject self-inconsistent metadata rather than guess: if the 'axes' count + // disagrees with the array rank, indexing the shape by axis role would read + // out of bounds. + if (axes.storageAxes.size() != level0Shape.size()) + throw std::runtime_error("OME-Zarr: 'axes' count " + std::to_string(axes.storageAxes.size()) + + " does not match array rank " + std::to_string(level0Shape.size())); + L.ndim = axes.storageAxes.size(); + L.ix = axes.storageIndexOf('X'); L.iy = axes.storageIndexOf('Y'); + L.iz = axes.storageIndexOf('Z'); L.ic = axes.storageIndexOf('C'); + L.it = axes.storageIndexOf('T'); + L.n_levels = axes.numberPyramidLevels(); + // Advertise the real C/T extents so the pipeline iterates channels and timeframes; + // without this the base class default of 1 keeps it on plane (c=0,t=0). + L.n_channels = axes.sizeC; + L.n_timeframes = axes.sizeT; + // Keep the parsed physical voxel spacing for opt-in calibration. + L.phys_x = axes.physX; L.phys_y = axes.physY; L.phys_z = axes.physZ; + L.phys_unit = axes.unitXY; + } + else + { + // No usable 'axes': map by position (X,Y last; Z,C,T before) -- rank-safe. + L.ndim = level0Shape.size(); + int n = (int) L.ndim; + L.ix = n - 1; L.iy = n - 2; + L.iz = (n >= 3) ? n - 3 : -1; + L.ic = (n >= 4) ? n - 4 : -1; + L.it = (n >= 5) ? n - 5 : -1; + L.n_levels = 1; + // Derive C/T extents from the positional axes so the fallback path also reports + // multi-channel / time-series counts (1 when the axis is absent). + L.n_channels = (L.ic >= 0) ? level0Shape[L.ic] : 1; + L.n_timeframes = (L.it >= 0) ? level0Shape[L.it] : 1; + } + + // X and Y must resolve to real dimensions, else the read would index OOB. + if (L.ix < 0 || L.iy < 0 || (std::size_t) L.ix >= level0Shape.size() || (std::size_t) L.iy >= level0Shape.size()) + throw std::runtime_error("OME-Zarr: cannot resolve X/Y axes from metadata"); + + L.full_width = level0Shape[L.ix]; + L.full_height = level0Shape[L.iy]; + L.full_depth = (L.iz >= 0) ? level0Shape[L.iz] : 1; + L.tile_width = chunkShape[L.ix]; + L.tile_height = chunkShape[L.iy]; + L.tile_depth = (L.iz >= 0) ? chunkShape[L.iz] : 1; + + return L; + } +} + +#endif // OMEZARR_SUPPORT diff --git a/src/nyx/ome/ome_zarr_meta.cpp b/src/nyx/ome/ome_zarr_meta.cpp new file mode 100644 index 00000000..bbd95eb9 --- /dev/null +++ b/src/nyx/ome/ome_zarr_meta.cpp @@ -0,0 +1,129 @@ +#ifdef OMEZARR_SUPPORT + +#include "ome_zarr_meta.h" +#include + +namespace Nyxus +{ + // Pull the 'scale' vector out of a dataset's coordinateTransformations (the + // entry whose type == "scale"); empty if the key/entry is absent or malformed. + // Safe on datasets that lack coordinateTransformations entirely (else a const + // operator[] on a missing key throws). + static std::vector scale_of(const nlohmann::json& dset) + { + std::vector s; + if (!dset.is_object() || !dset.contains("coordinateTransformations")) + return s; + const auto& cts = dset["coordinateTransformations"]; + if (cts.is_array()) + for (const auto& ct : cts) + if (ct.is_object() && ct.value("type", std::string()) == "scale" && + ct.contains("scale") && ct["scale"].is_array()) + { + for (const auto& v : ct["scale"]) s.push_back(v.is_number() ? v.get() : 1.0); + break; + } + return s; + } + + OmeAxes parse_ome_zarr(const nlohmann::json& groupAttrs, + const std::vector& level0Shape, + const std::string& dtypeStr) + { + OmeAxes ax; + + // 0.5 nests the model under "ome"; 0.4 puts it at the group root. + const nlohmann::json* root = nullptr; + if (groupAttrs.contains("ome") && groupAttrs["ome"].contains("multiscales")) + root = &groupAttrs["ome"]; + else if (groupAttrs.contains("multiscales")) + root = &groupAttrs; + if (!root) + return ax; + + const auto& multiscales = (*root)["multiscales"]; + if (!multiscales.is_array() || multiscales.empty()) + return ax; + const auto& ms = multiscales[0]; + if (!ms.contains("axes") || !ms.contains("datasets")) + return ax; + + const auto& axes = ms["axes"]; + const auto& datasets = ms["datasets"]; + if (!axes.is_array() || !datasets.is_array() || datasets.empty()) + return ax; + + // Level-0 scale (per-axis), from the first dataset's transformations. + std::vector scale0 = scale_of(datasets[0]); + if (scale0.size() != axes.size()) + scale0.assign(axes.size(), 1.0); + + ax.dtype = pixel_type_from_zarr_dtype(dtypeStr); + if (ax.dtype == PixelType::Unknown) ax.dtype = PixelType::UInt16; + ax.bitsPerSample = bits_of(ax.dtype); + + for (std::size_t k = 0; k < axes.size(); ++k) + { + const auto& a = axes[k]; + if (!a.is_object()) // a malformed axis entry -> skip (keeps role sizes at default) + continue; + std::string name = a.value("name", std::string()); + std::string type = a.value("type", std::string()); + std::string unit = a.value("unit", std::string()); + char label = name.empty() ? '?' : (char)std::toupper((unsigned char)name[0]); + + OmeAxis oa; + oa.label = label; + if (type == "channel") oa.kind = AxisKind::Channel; + else if (type == "time") oa.kind = AxisKind::Time; + else oa.kind = AxisKind::Space; + oa.size = (k < level0Shape.size()) ? level0Shape[k] : 1; + oa.physical = (k < scale0.size()) ? scale0[k] : 1.0; + // Canonicalize space axes (X/Y/Z) to micrometer using their OWN declared unit + // (a Z axis calibrated in a different unit than X/Y previously reported its raw, + // uncomparable value under whatever label X/Y happened to carry). Time/channel + // axes are left alone -- a time unit like "second" isn't a length to convert. + if (label == 'X' || label == 'Y' || label == 'Z') + canonicalize_to_micrometer(oa.physical, unit); + oa.unit = unit; + ax.storageAxes.push_back(oa); + ax.storageOrder.push_back(label); + + switch (label) + { + case 'X': ax.sizeX = oa.size; ax.physX = oa.physical; ax.unitXY = unit; break; + case 'Y': ax.sizeY = oa.size; ax.physY = oa.physical; if (ax.unitXY.empty()) ax.unitXY = unit; break; + case 'Z': ax.sizeZ = oa.size; ax.physZ = oa.physical; ax.unitZ = unit; break; + case 'C': ax.sizeC = oa.size; break; + case 'T': ax.sizeT = oa.size; ax.timeIncrement = oa.physical; ax.unitT = unit; break; + default: break; + } + } + + ax.omeDimensionOrder = ome_dimension_order_from_storage(ax.storageOrder); + + // Pyramid levels: one per dataset entry. + int ix = ax.storageIndexOf('X'), iy = ax.storageIndexOf('Y'), iz = ax.storageIndexOf('Z'); + for (std::size_t li = 0; li < datasets.size(); ++li) + { + const auto& dset = datasets[li]; + OmePyramidLevel lv; + lv.index = li; + lv.path = dset.is_object() ? dset.value("path", std::to_string(li)) : std::to_string(li); + std::vector sc = scale_of(dset); + if (ix >= 0 && (std::size_t)ix < sc.size()) lv.scaleX = sc[ix]; + if (iy >= 0 && (std::size_t)iy < sc.size()) lv.scaleY = sc[iy]; + if (iz >= 0 && (std::size_t)iz < sc.size()) lv.scaleZ = sc[iz]; + if (li == 0) // only level-0 extents are known from the supplied shape + { + lv.sizeX = ax.sizeX; lv.sizeY = ax.sizeY; lv.sizeZ = ax.sizeZ; + } + ax.levels.push_back(lv); + } + + ax.valid = true; + return ax; + } +} + +#endif // OMEZARR_SUPPORT diff --git a/src/nyx/ome/ome_zarr_meta.h b/src/nyx/ome/ome_zarr_meta.h new file mode 100644 index 00000000..08d08596 --- /dev/null +++ b/src/nyx/ome/ome_zarr_meta.h @@ -0,0 +1,34 @@ +#pragma once + +// OME-Zarr (NGFF) metadata -> OmeAxes. Parses the group `multiscales` +// block: `axes` (order + type + unit), `datasets` (pyramid paths + +// coordinateTransformations), for both 0.4 (metadata at the group root) and 0.5 +// (metadata under the "ome" key). The caller supplies the level-0 array shape +// and dtype (read from the dataset's .zarray/zarr.json via z5) so this parser +// stays decoupled from the storage backend. +// +// Guarded by OMEZARR_SUPPORT because it depends on nlohmann::json (vendored with +// z5). Only meaningful when Nyxus is built with OME-Zarr support. + +#ifdef OMEZARR_SUPPORT + +#include +#include +#include +#include "nlohmann/json.hpp" +#include "ome_axes.h" + +namespace Nyxus +{ + // Parse NGFF group attributes into an OmeAxes. + // groupAttrs : the group's attributes JSON (.zattrs contents for v2, or the + // "attributes" object of zarr.json for v3). + // level0Shape : shape of the level-0 dataset, in on-disk axis order. + // dtypeStr : the level-0 dtype string (v2 "& level0Shape, + const std::string& dtypeStr); +} + +#endif // OMEZARR_SUPPORT diff --git a/src/nyx/omezarr.h b/src/nyx/omezarr.h index 5222657c..33bf0071 100644 --- a/src/nyx/omezarr.h +++ b/src/nyx/omezarr.h @@ -3,8 +3,11 @@ #ifdef OMEZARR_SUPPORT #include +#include +#include #include "abs_tile_loader.h" #include "nlohmann/json.hpp" +#include "ome/ome_zarr_layout.h" // resolve_zarr_layout() -- shared with raw_omezarr.h // factory functions to create files, groups and datasets #include "z5/factory.hxx" @@ -23,7 +26,7 @@ /// @brief Tile Loader for OMEZarr /// @tparam DataType AbstractView's internal type template -class NyxusOmeZarrLoader : public AbstractTileLoader +class NyxusOmeZarrLoader : public AbstractTileLoader { public: @@ -37,40 +40,32 @@ class NyxusOmeZarrLoader : public AbstractTileLoader { // Open the file zarr_ptr_ = std::make_unique(filePath.c_str()); - nlohmann::json file_attributes, ds_attributes; + nlohmann::json file_attributes; z5::readAttributes(*zarr_ptr_, file_attributes); - // assume only one dataset is present - ds_name_ = file_attributes["multiscales"][0]["datasets"][0]["path"].get(); - const auto ds_handle = z5::filesystem::handle::Dataset(*zarr_ptr_, ds_name_); - fs::path metadata_path; - auto success = z5::filesystem::metadata_detail::getMetadataPath(ds_handle, metadata_path); - z5::filesystem::metadata_detail::readMetadata(metadata_path, ds_attributes); - - full_depth_ = ds_attributes["shape"][2].get(); - full_height_ = ds_attributes["shape"][3].get(); - full_width_ = ds_attributes["shape"][4].get(); - tile_depth_ = ds_attributes["chunks"][2].get(); - tile_height_ = ds_attributes["chunks"][3].get(); - tile_width_ = ds_attributes["chunks"][4].get(); - std::string dtype_str = ds_attributes["dtype"].get(); - if (dtype_str == "(); + + // FIX (IO): open the dataset via z5, which auto-detects Zarr v2 (.zarray) vs v3 + // (zarr.json) and handles the v3 chunk-key encoding, codecs (zstd/blosc/...) and + // sharding. Query shape/chunking/dtype from the Dataset object (format-agnostic) + // instead of parsing the v2-only .zarray by hand -- this is what lets the loader read + // BOTH OME-Zarr 0.4 and 0.5. The dataset handle is cached (metadata is immutable). ds_ = z5::openDataset(*zarr_ptr_, ds_name_); + std::vector level0Shape(ds_->shape().begin(), ds_->shape().end()); + std::vector chunkShape(ds_->defaultChunkShape().begin(), ds_->defaultChunkShape().end()); + + // Axis roles, extents, chunking, calibration and pixel type, resolved from the NGFF + // 'axes' metadata (shared with RawOmezarrLoader -- see ome/ome_zarr_layout.h) + Nyxus::ZarrLayout L = Nyxus::resolve_zarr_layout (file_attributes, level0Shape, chunkShape, ds_->getDtype()); + ndim_ = L.ndim; + ix_ = L.ix; iy_ = L.iy; iz_ = L.iz; ic_ = L.ic; it_ = L.it; + full_width_ = L.full_width; full_height_ = L.full_height; full_depth_ = L.full_depth; + tile_width_ = L.tile_width; tile_height_ = L.tile_height; tile_depth_ = L.tile_depth; + n_levels_ = L.n_levels; n_channels_ = L.n_channels; n_timeframes_ = L.n_timeframes; + phys_x_ = L.phys_x; phys_y_ = L.phys_y; phys_z_ = L.phys_z; phys_unit_ = L.phys_unit; + bits_per_sample_ = L.bits_per_sample; + data_format_ = L.data_format; } /// @brief NyxusOmeZarrLoader destructor @@ -90,54 +85,57 @@ class NyxusOmeZarrLoader : public AbstractTileLoader size_t indexRowGlobalTile, size_t indexColGlobalTile, size_t indexLayerGlobalTile, - [[maybe_unused]] size_t level) override + size_t indexChannel, // C plane to read (offset into the channel axis) + size_t indexTimeframe, // T plane to read (offset into the time axis) + [[maybe_unused]] size_t level) override { size_t pixel_row_index = indexRowGlobalTile*tile_height_; size_t pixel_col_index = indexColGlobalTile*tile_width_; size_t pixel_layer_index = indexLayerGlobalTile*tile_depth_; - + switch (data_format_) { case 1: - loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; case 2: - loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; case 3: - loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; case 4: - loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; case 5: - loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; case 6: - loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; case 7: - loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; case 8: - loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; case 9: - loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; case 10: - loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; default: - loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(tile, pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; } } - + template - void loadTile(std::shared_ptr> &dest, size_t pixel_row_index, - size_t pixel_col_index, size_t pixel_layer_index) { + void loadTile(std::shared_ptr> &dest, size_t pixel_row_index, + size_t pixel_col_index, size_t pixel_layer_index, + size_t pixel_channel_index, size_t pixel_timeframe_index) { size_t data_height = tile_height_, data_width = tile_width_; if (pixel_row_index + data_height > full_height_) { data_height = full_height_ - pixel_row_index; @@ -145,23 +143,40 @@ class NyxusOmeZarrLoader : public AbstractTileLoader if (pixel_col_index + data_width > full_width_) { data_width = full_width_ - pixel_col_index; } + // A chunk may span several Z-planes (tile_depth_ > 1); read its whole Z-extent in one + // subarray call, clamped at the last (possibly partial) chunk. + size_t data_depth = 1; + if (iz_ >= 0) { + data_depth = tile_depth_; + if (pixel_layer_index + data_depth > full_depth_) + data_depth = full_depth_ - pixel_layer_index; + } // Create a buffer to hold the read data - std::vector buffer(data_height * data_width); - - // Create an ArrayView into the buffer (z5 3.0.1 uses ArrayView instead of xtensor) - z5::types::ShapeType shape = {1, 1, 1, data_height, data_width}; + std::vector buffer(data_height * data_width * data_depth); + + // Build the read window by axis ROLE (honoring the resolved 'axes' order), + // reading a Z*Y*X block (data_depth Z-planes) at the requested C/T. z5 3.0.1 uses ArrayView. + z5::types::ShapeType shape(ndim_, 1), offset(ndim_, 0); + shape[iy_] = data_height; offset[iy_] = pixel_row_index; + shape[ix_] = data_width; offset[ix_] = pixel_col_index; + if (iz_ >= 0) { shape[iz_] = data_depth; offset[iz_] = pixel_layer_index; } + if (ic_ >= 0) offset[ic_] = pixel_channel_index; + if (it_ >= 0) offset[it_] = pixel_timeframe_index; auto view = z5::multiarray::makeView(buffer.data(), shape); - z5::types::ShapeType offset = {0, 0, pixel_layer_index, pixel_row_index, pixel_col_index}; - + // Read subarray from the cached z5 dataset z5::multiarray::readSubarray(*ds_, view, offset.begin()); - - // Copy from buffer to destination tile, handling partial tiles - for (size_t k = 0; k < data_height; ++k) { - std::copy(buffer.begin() + k * data_width, - buffer.begin() + (k + 1) * data_width, - dest->begin() + k * tile_width_); + + // Copy from buffer to destination tile, handling partial tiles and multiple Z-planes + // (dest is laid out plane-major: plane p, row k at offset (p*tile_height_ + k)*tile_width_, + // matching ImageLoader::assemble_volume's/RawImageLoader::for_each_voxel's expected stride). + for (size_t p = 0; p < data_depth; ++p) { + for (size_t k = 0; k < data_height; ++k) { + std::copy(buffer.begin() + (p * data_height + k) * data_width, + buffer.begin() + (p * data_height + k) * data_width + data_width, + dest->begin() + (p * tile_height_ + k) * tile_width_); + } } } @@ -191,12 +206,19 @@ class NyxusOmeZarrLoader : public AbstractTileLoader /// @return Tile depth [[nodiscard]] size_t tileDepth([[maybe_unused]] size_t level) const override { return tile_depth_; } - /// @brief Tiff bits per sample - /// @return Size of a sample in bits - [[nodiscard]] short bitsPerSample() const override { return 1; } - /// @brief Level accessor - /// @return 1 - [[nodiscard]] size_t numberPyramidLevels() const override { return 1; } + /// @brief Bits per sample (resolved from the dataset dtype) + [[nodiscard]] short bitsPerSample() const override { return bits_per_sample_; } + /// @brief Number of resolution (pyramid) levels declared in multiscales + [[nodiscard]] size_t numberPyramidLevels() const override { return n_levels_; } + /// @brief Channel (C) extent resolved from the NGFF axes (1 if no channel axis) + [[nodiscard]] size_t numberChannels() const override { return n_channels_; } + /// @brief Time (T) extent resolved from the NGFF axes (1 if no time axis) + [[nodiscard]] size_t fullTimestamps([[maybe_unused]] size_t level) const override { return n_timeframes_; } + /// @brief Physical voxel spacing from the NGFF coordinateTransformations (1.0 if uncalibrated) + [[nodiscard]] double physicalSizeX() const override { return phys_x_; } + [[nodiscard]] double physicalSizeY() const override { return phys_y_; } + [[nodiscard]] double physicalSizeZ() const override { return phys_z_; } + [[nodiscard]] std::string physicalSizeUnit() const override { return phys_unit_; } private: @@ -208,6 +230,16 @@ class NyxusOmeZarrLoader : public AbstractTileLoader tile_height_ = 0, ///< Tile height tile_depth_ = 0; ///< Tile depth + // Storage-dimension index of each axis role (-1 if the axis is absent). + int ix_ = 4, iy_ = 3, iz_ = 2, ic_ = 1, it_ = 0; + size_t ndim_ = 5; ///< Number of on-disk dimensions (2..5) + short bits_per_sample_ = 16; ///< Real bit depth + size_t n_levels_ = 1; ///< Pyramid level count + size_t n_channels_ = 1; ///< Channel (C) extent + size_t n_timeframes_ = 1; ///< Time (T) extent + double phys_x_ = 1.0, phys_y_ = 1.0, phys_z_ = 1.0; ///< Physical voxel spacing + std::string phys_unit_; ///< Physical-size unit (e.g. "micrometer") + short data_format_ = 0; std::unique_ptr zarr_ptr_; std::string ds_name_; diff --git a/src/nyx/output_2_apache.cpp b/src/nyx/output_2_apache.cpp index 6a263ba6..09bc8765 100644 --- a/src/nyx/output_2_apache.cpp +++ b/src/nyx/output_2_apache.cpp @@ -24,9 +24,9 @@ namespace Nyxus { static std::mutex mx1; - std::tuple> save_features_2_apache_wholeslide (Environment & env, const LR & wsi_roi, const std::string & wsi_path) + std::tuple> save_features_2_apache_wholeslide (Environment & env, const LR & wsi_roi, const std::string & wsi_path, size_t t_index, size_t c_index) { std::lock_guard lg (mx1); - return env.arrow_stream.write_arrow_file (Nyxus::get_feature_values_roi (env, env.theFeatureSet, wsi_roi, wsi_path, "")); + return env.arrow_stream.write_arrow_file (Nyxus::get_feature_values_roi (env, env.theFeatureSet, wsi_roi, wsi_path, "", t_index, c_index)); } } \ No newline at end of file diff --git a/src/nyx/output_2_buffer.cpp b/src/nyx/output_2_buffer.cpp index 619aa6bf..abece24f 100644 --- a/src/nyx/output_2_buffer.cpp +++ b/src/nyx/output_2_buffer.cpp @@ -25,12 +25,14 @@ namespace Nyxus bool save_features_2_buffer_wholeslide ( // out - ResultsCache & rescache, + ResultsCache & rescache, // in Environment & env, const LR & r, const std::string& ifpath, - const std::string& mfpath) + const std::string& mfpath, + size_t t_index, + size_t c_index) { std::lock_guard lg (mx1); @@ -44,7 +46,11 @@ namespace Nyxus if (fill_header) { - rescache.add_to_header({ Nyxus::colname_intensity_image, Nyxus::colname_mask_image, Nyxus::colname_roi_label, Nyxus::colname_t_index }); + // FIX (IO): phys_unit is a leading STRING column (kept with intensity/mask so the + // Python "string columns first" split still works); phys_x/y/z are numeric. + rescache.add_to_header({ Nyxus::colname_intensity_image, Nyxus::colname_mask_image, Nyxus::colname_phys_unit, + Nyxus::colname_roi_label, Nyxus::colname_t_index, Nyxus::colname_c_index, + Nyxus::colname_phys_x, Nyxus::colname_phys_y, Nyxus::colname_phys_z }); for (auto& enabdF : F) { @@ -178,11 +184,19 @@ namespace Nyxus rescache.inc_num_rows(); - // - slide info + // - slide info + phys_unit (FIX (IO): 3rd leading string column) + const SlideProps& sp = env.dataset.dataset_props[r.slide_idx]; rescache.add_string (ifpath); rescache.add_string (mfpath); + rescache.add_string (sp.phys_unit); rescache.add_numeric (r.label); - rescache.add_numeric (DEFAULT_T_INDEX); + // FIX (IO): emit the real time/channel indices (was hard-coded DEFAULT_T_INDEX with no channel) + rescache.add_numeric ((double)t_index); + rescache.add_numeric ((double)c_index); + // FIX (IO): physical voxel spacing (numeric columns) + rescache.add_numeric (sp.phys_x); + rescache.add_numeric (sp.phys_y); + rescache.add_numeric (sp.phys_z); // - features for (auto& enabdF : F) @@ -300,7 +314,7 @@ namespace Nyxus } /// @brief Copies ROIs' feature values into a ResultsCache structure that will then shape them as a table - bool save_features_2_buffer (ResultsCache& rescache, Environment & env, size_t t_index) + bool save_features_2_buffer (ResultsCache& rescache, Environment & env, size_t t_index, size_t c_index) { std::vector L{ env.uniqueLabels.begin(), env.uniqueLabels.end() }; std::sort(L.begin(), L.end()); @@ -313,7 +327,11 @@ namespace Nyxus // -- Header if (fill_header) { - rescache.add_to_header({ Nyxus::colname_intensity_image, Nyxus::colname_mask_image, Nyxus::colname_roi_label, Nyxus::colname_t_index }); + // FIX (IO): phys_unit is a leading STRING column (kept with intensity/mask so the + // Python "string columns first" split still works); phys_x/y/z are numeric. + rescache.add_to_header({ Nyxus::colname_intensity_image, Nyxus::colname_mask_image, Nyxus::colname_phys_unit, + Nyxus::colname_roi_label, Nyxus::colname_t_index, Nyxus::colname_c_index, + Nyxus::colname_phys_x, Nyxus::colname_phys_y, Nyxus::colname_phys_z }); for (auto& enabdF : F) { @@ -463,8 +481,13 @@ namespace Nyxus rescache.add_string (intfname); rescache.add_string (segfname); + rescache.add_string (slide.phys_unit); // FIX (IO): 3rd leading string column rescache.add_numeric (l); - rescache.add_numeric (t_index); + rescache.add_numeric ((double)t_index); + rescache.add_numeric ((double)c_index); // FIX (IO): channel index column, mirrors t_index + rescache.add_numeric (slide.phys_x); // FIX (IO): physical voxel spacing (numeric) + rescache.add_numeric (slide.phys_y); + rescache.add_numeric (slide.phys_z); for (auto& enabdF : F) { diff --git a/src/nyx/output_2_csv.cpp b/src/nyx/output_2_csv.cpp index e282c432..5ad5af9f 100644 --- a/src/nyx/output_2_csv.cpp +++ b/src/nyx/output_2_csv.cpp @@ -63,6 +63,10 @@ namespace Nyxus head.push_back (Nyxus::colname_intensity_image); head.push_back (Nyxus::colname_mask_image); + // Physical-size unit (FIX (IO): a leading STRING column so the Python buffer's + // "string columns first" reconstruction still holds; the numeric phys_x/y/z go later) + head.push_back (Nyxus::colname_phys_unit); + // Annotation columns if (env.resultOptions.need_annotation()) { @@ -80,6 +84,14 @@ namespace Nyxus // time head.push_back (Nyxus::colname_t_index); + // channel (FIX (IO): mirrors t_index; one column per channel plane) + head.push_back (Nyxus::colname_c_index); + + // physical voxel spacing (FIX (IO): numeric columns) + head.push_back (Nyxus::colname_phys_x); + head.push_back (Nyxus::colname_phys_y); + head.push_back (Nyxus::colname_phys_z); + // Optional columns for (auto& enabdF : F) { @@ -211,11 +223,12 @@ namespace Nyxus bool save_features_2_csv_wholeslide ( Environment & env, - const LR & r, - const std::string & ifpath, - const std::string & mfpath, + const LR & r, + const std::string & ifpath, + const std::string & mfpath, const std::string & outdir, - size_t t_index) + size_t t_index, + size_t c_index) { std::lock_guard lg (mutex1); // Lock the mutex @@ -224,16 +237,18 @@ namespace Nyxus char rvbuf[VAL_BUF_LEN]; // real value buffer large enough to fit a float64 value in range (2.2250738585072014E-308 to 1.79769313486231570e+308) const char rvfmt[] = "%g"; // instead of "%20.12f" which produces a too massive output - static bool mustRenderHeader = true; // In 'singlecsv' scenario this flag flips from 'T' to 'F' when necessary (after the header is rendered) - // Make the file name and write mode std::string fullPath = get_feature_output_fname (env, ifpath, mfpath); VERBOSLVL2 (env.get_verbosity_level(), std::cout << "\t--> " << fullPath << "\n"); - // Single CSV: create or continue? - const char* mode = "w"; - if (!env.separateCsv) - mode = mustRenderHeader ? "w" : "a"; + // FIX: create or continue, per PATH. This sink is called once per (channel, timeframe) + // plane, but separatecsv derives one path per slide and used to open it "w" every time, + // so each plane truncated the previous and the file kept only the last one. Truncate and + // render the header on the first write to a path, append after that -- which is what the + // t_index/c_index columns are for, and matches the Arrow and buffer sinks. Replaces a + // function-static 'mustRenderHeader' that could only express this for singlecsv. + const bool mustRenderHeader = env.csv_claim_first_write (fullPath); + const char* mode = mustRenderHeader ? "w" : "a"; // Open it FILE* fp = nullptr; @@ -272,10 +287,7 @@ namespace Nyxus auto head_string = ssHead.str(); fprintf(fp, "%s\n", head_string.c_str()); - - // Prevent rendering the header again for another image's portion of labels - if (env.separateCsv == false) - mustRenderHeader = false; + // (the path is already recorded, so later writes to it append without a header) } // ********** Values @@ -288,10 +300,16 @@ namespace Nyxus ssVals << std::fixed; // slide info - ssVals << ifpath << "," << mfpath; - - // ROI and time - ssVals << "," << r.label << "," << t_index; + // FIX (IO): physical calibration of this slide (phys_unit is a leading string column, + // phys_x/y/z are numeric; 1.0/"" when uncalibrated). r.slide_idx set for the vROI. + const SlideProps& sp = env.dataset.dataset_props[r.slide_idx]; + ssVals << ifpath << "," << mfpath << "," << sp.phys_unit; + + // ROI, time and channel + ssVals << "," << r.label << "," << t_index << "," << c_index; + + // physical voxel spacing (numeric) + ssVals << "," << sp.phys_x << "," << sp.phys_y << "," << sp.phys_z; // features for (auto& enabdF : F) @@ -418,10 +436,11 @@ namespace Nyxus // Saves the result of image scanning and feature calculation. Must be called after the reduction phase. bool save_features_2_csv ( Environment & env, - const std::string& intFpath, - const std::string& segFpath, + const std::string& intFpath, + const std::string& segFpath, const std::string& outputDir, size_t t_index, + size_t c_index, bool need_aggregation) { // Non-exotic formatting for compatibility with the buffer output (Python API, Apache) @@ -433,16 +452,15 @@ namespace Nyxus std::vector L{ env.uniqueLabels.begin(), env.uniqueLabels.end() }; std::sort(L.begin(), L.end()); - static bool mustRenderHeader = true; // In 'singlecsv' scenario this flag flips from 'T' to 'F' when necessary (after the header is rendered) - // Make the file name and write mode std::string fullPath = get_feature_output_fname (env, intFpath, segFpath); VERBOSLVL2 (env.get_verbosity_level(), std::cout << "\t--> " << fullPath << "\n"); - // Single CSV: create or continue? - const char* mode = "w"; - if (!env.separateCsv) - mode = mustRenderHeader ? "w" : "a"; + // FIX: create or continue, per PATH -- see the whole-slide sink above. The 3D segmented + // workflow also calls this once per (channel, timeframe), and separatecsv truncated the + // file on every call, keeping only the last plane's ROI rows. + const bool mustRenderHeader = env.csv_claim_first_write (fullPath); + const char* mode = mustRenderHeader ? "w" : "a"; // Open it FILE* fp = nullptr; @@ -482,20 +500,17 @@ namespace Nyxus auto head_string = ssHead.str(); fprintf(fp, "%s\n", head_string.c_str()); - - // Prevent rendering the header again for another image's portion of labels - if (env.separateCsv == false) - mustRenderHeader = false; + // (the path is already recorded, so later writes to it append without a header) } if (need_aggregation) { - auto allres = Nyxus::get_feature_values (env, env.theFeatureSet, env.uniqueLabels, env.roiData, env.dataset); // shape: td::vector, int, std::vector>> + auto allres = Nyxus::get_feature_values (env, env.theFeatureSet, env.uniqueLabels, env.roiData, env.dataset, t_index, c_index); // shape: vector, label, t_index, c_index, vector>> if (allres.size()) { // aggregate const auto& tup0 = allres[0]; - const auto& v0 = std::get<3>(tup0); + const auto& v0 = std::get(tup0); // FIX (IO): FBEGIN shifted 3->4 after c_index; was hard-coded std::get<3> auto n_feats = v0.size(); std::vector a (n_feats); @@ -507,7 +522,7 @@ namespace Nyxus // we ned to add Skipping blacklisted ROI // ... - const auto& v = std::get<3>(tup); + const auto& v = std::get(tup); // FIX (IO): FBEGIN shifted 3->4 after c_index for (size_t i = 0; i < n_feats; i++) { double x = v[i], @@ -521,11 +536,11 @@ namespace Nyxus } } - // send the aggregate as ROI #-1 to output + // send the aggregate as ROI #-1 to output const std::vector & fnames = std::get<0>(tup0); std::stringstream ssVals; - // ... slide/mask file info - ssVals << fnames[0] << "," << fnames[1]; + // ... slide/mask file info + phys_unit (FIX (IO): fnames[2] is the leading unit string) + ssVals << fnames[0] << "," << fnames[1] << "," << fnames[2]; // ... annotation info auto lab0 = std::get<1>(tup0); // annotation info is per slide, so OK to grab it from the 1st ROI LR& r0 = env.roiData [lab0]; @@ -534,6 +549,11 @@ namespace Nyxus ssVals << "," << a; // ... ROI id ssVals << "," << -1; + // ... time and channel (FIX (IO): the aggregate row previously omitted these, + // misaligning it against the header; emit both so columns line up) + ssVals << "," << t_index << "," << c_index; + // ... physical voxel spacing (FIX (IO): numeric columns, from the 1st ROI's tuple) + ssVals << "," << std::get(tup0) << "," << std::get(tup0) << "," << std::get(tup0); // ... aggregated feature values for (size_t i = 0; i < n_feats; i++) ssVals << "," << a[i]; @@ -560,7 +580,8 @@ namespace Nyxus const SlideProps& sli = env.dataset.dataset_props [r.slide_idx]; fs::path pseg (sli.fname_seg), pint (sli.fname_int); - ssVals << pint.filename() << "," << pseg.filename(); + // FIX (IO): phys_unit is a leading string column (after the file names) + ssVals << pint.filename() << "," << pseg.filename() << "," << sli.phys_unit; // annotation if (env.resultOptions.need_annotation()) @@ -569,8 +590,11 @@ namespace Nyxus ssVals << "," << a; } - // ROI label and time - ssVals << "," << l << "," << t_index; + // ROI label, time and channel + ssVals << "," << l << "," << t_index << "," << c_index; + + // physical voxel spacing (FIX (IO): numeric columns) + ssVals << "," << sli.phys_x << "," << sli.phys_y << "," << sli.phys_z; for (auto& enabdF : F) { @@ -761,7 +785,9 @@ namespace Nyxus const FeatureSet & fset, const LR & r, const std::string & ifpath, - const std::string & mfpath) + const std::string & mfpath, + size_t t_index, + size_t c_index) { std::vector features; @@ -878,13 +904,21 @@ namespace Nyxus fvals.push_back(vv[0]); } - // other columns + // physical calibration of this slide (FIX (IO)) + const SlideProps& sp = env.dataset.dataset_props[r.slide_idx]; + + // other columns (textcols[2] = phys_unit, a leading string column) std::vector textcols; textcols.push_back ((fs::path(ifpath)).filename().string()); textcols.push_back (""); + textcols.push_back (sp.phys_unit); int roilabl = r.label; // whole-slide roi # - features.push_back (std::make_tuple(textcols, roilabl, -999.88, fvals)); + // FIX (IO): emit the real time/channel indices (was hard-coded -999.88 for time + // and had no channel field) + physical voxel spacing, so the Apache/buffer whole-slide + // row is addressable by (t,c) and carries calibration. + features.push_back (std::make_tuple(textcols, roilabl, (double)t_index, (double)c_index, + sp.phys_x, sp.phys_y, sp.phys_z, fvals)); return features; } @@ -894,7 +928,9 @@ namespace Nyxus const FeatureSet & fset, const Uniqueids & uniqueLabels, const Roidata & roiData, - const Dataset & dataset) + const Dataset & dataset, + size_t t_index, + size_t c_index) { std::vector features; @@ -923,6 +959,7 @@ namespace Nyxus std::vector filenames; filenames.push_back(pint.filename().string()); filenames.push_back(pseg.filename().string()); + filenames.push_back(sli.phys_unit); // FIX (IO): phys_unit leading string column for (auto& enabdF : F) { @@ -1031,7 +1068,10 @@ namespace Nyxus feature_values.push_back(vv[0]); } - features.push_back (std::make_tuple(filenames, l, DEFAULT_T_INDEX, feature_values)); + // FIX (IO): thread the real time/channel indices instead of the hard-coded + // DEFAULT_T_INDEX (which pinned every Apache/Parquet row to t=0,c=0). + features.push_back (std::make_tuple(filenames, l, (double)t_index, (double)c_index, + sli.phys_x, sli.phys_y, sli.phys_z, feature_values)); } return features; diff --git a/src/nyx/output_writers.cpp b/src/nyx/output_writers.cpp index 39c5c650..c4ace62a 100644 --- a/src/nyx/output_writers.cpp +++ b/src/nyx/output_writers.cpp @@ -18,11 +18,14 @@ arrow::Status ParquetWriter::setup(const std::vector& header) { std::vector> fields; + // FIX (IO): schema is [intensity, mask, phys_unit] (utf8) + ROI_label (int32) + the rest + // (t_index, c_index, phys_x/y/z, features) as float64 - matches get_header's column order. fields.push_back(arrow::field(header[0], arrow::utf8())); fields.push_back(arrow::field(header[1], arrow::utf8())); - fields.push_back(arrow::field(header[2], arrow::int32())); + fields.push_back(arrow::field(header[2], arrow::utf8())); + fields.push_back(arrow::field(header[3], arrow::int32())); - for (int i = 3; i < header.size(); ++i) + for (int i = 4; i < header.size(); ++i) { fields.push_back(arrow::field(header[i], arrow::float64())); } @@ -114,6 +117,19 @@ arrow::Status ParquetWriter::write (const std::vector& features) } A.push_back (A_s); + + //cccccccc phys_unit (FIX (IO): 3rd leading string column) + b.Reset(); + std::shared_ptr A_pu; + for (int i = 0; i < num_rows; ++i) { + append_status = b.Append (std::get(features[i])[2]); + if (!append_status.ok()) + return append_status; + } + append_status = b.Finish (&A_pu); + if (!append_status.ok()) + return append_status; + A.push_back (A_pu); } //cccccccc ROI label @@ -160,9 +176,51 @@ arrow::Status ParquetWriter::write (const std::vector& features) A.push_back (A_t); } + //cccccccc channel (FIX (IO): mirrors the time column so multi-channel input is addressable) + { + arrow::DoubleBuilder b; + std::shared_ptr A_c; + for (int i = 0; i < num_rows; ++i) + { + append_status = b.Append (std::get(features[i])); + if (!append_status.ok()) { + // Handle read error + return append_status; + } + } + + append_status = b.Finish (&A_c); + if (!append_status.ok()) + { + // Handle read error + return append_status; + } + A.push_back (A_c); + } + + //cccccccc physical voxel spacing (FIX (IO): phys_x/y/z numeric columns) + for (int axis = FTABLE_PXPOS; axis <= FTABLE_PZPOS; ++axis) + { + arrow::DoubleBuilder b; + std::shared_ptr A_ps; + for (int i = 0; i < num_rows; ++i) + { + double v = (axis == FTABLE_PXPOS) ? std::get(features[i]) + : (axis == FTABLE_PYPOS) ? std::get(features[i]) + : std::get(features[i]); + append_status = b.Append (v); + if (!append_status.ok()) + return append_status; + } + append_status = b.Finish (&A_ps); + if (!append_status.ok()) + return append_status; + A.push_back (A_ps); + } + //cccccccc features { - // construct columns for each feature + // construct columns for each feature for (int j = 0; j < std::get(features[0]).size(); ++j) { arrow::DoubleBuilder b; @@ -225,11 +283,13 @@ arrow::Status ArrowIPCWriter::setup(const std::vector& header) { std::vector> fields; - fields.push_back(arrow::field("intensity_image", arrow::utf8())); - fields.push_back(arrow::field("mask_image", arrow::utf8())); - fields.push_back(arrow::field("ROI_label", arrow::int32())); + // FIX (IO): [intensity, mask, phys_unit] (utf8) + ROI_label (int32) + float64 rest + fields.push_back(arrow::field(header[0], arrow::utf8())); + fields.push_back(arrow::field(header[1], arrow::utf8())); + fields.push_back(arrow::field(header[2], arrow::utf8())); + fields.push_back(arrow::field(header[3], arrow::int32())); - for (int i = 3; i < header.size(); ++i) + for (int i = 4; i < header.size(); ++i) { fields.push_back(arrow::field(header[i], arrow::float64())); } @@ -306,6 +366,19 @@ arrow::Status ArrowIPCWriter::write (const std::vector& features) } A.push_back (A_s); + + //cccccccc phys_unit (FIX (IO): 3rd leading string column) + b.Reset(); + std::shared_ptr A_pu; + for (int i = 0; i < num_rows; ++i) { + append_status = b.Append (std::get(features[i])[2]); + if (!append_status.ok()) + return append_status; + } + append_status = b.Finish (&A_pu); + if (!append_status.ok()) + return append_status; + A.push_back (A_pu); } //cccccccc ROI label @@ -351,9 +424,49 @@ arrow::Status ArrowIPCWriter::write (const std::vector& features) A.push_back (A_t); } + //cccccccc channel (FIX (IO): mirrors the time column so multi-channel input is addressable) + { + arrow::DoubleBuilder b; + std::shared_ptr A_c; + for (int i = 0; i < num_rows; ++i) { + append_status = b.Append (std::get(features[i])); + if (!append_status.ok()) { + // Handle read error + return append_status; + } + } + + append_status = b.Finish (&A_c); + if (!append_status.ok()) { + // Handle read error + return append_status; + } + A.push_back (A_c); + } + + //cccccccc physical voxel spacing (FIX (IO): phys_x/y/z numeric columns) + for (int axis = FTABLE_PXPOS; axis <= FTABLE_PZPOS; ++axis) + { + arrow::DoubleBuilder b; + std::shared_ptr A_ps; + for (int i = 0; i < num_rows; ++i) + { + double v = (axis == FTABLE_PXPOS) ? std::get(features[i]) + : (axis == FTABLE_PYPOS) ? std::get(features[i]) + : std::get(features[i]); + append_status = b.Append (v); + if (!append_status.ok()) + return append_status; + } + append_status = b.Finish (&A_ps); + if (!append_status.ok()) + return append_status; + A.push_back (A_ps); + } + //cccccccc feature values { - // construct columns for each feature + // construct columns for each feature for (int j = 0; j < std::get(features[0]).size(); ++j) { arrow::DoubleBuilder b; diff --git a/src/nyx/output_writers.h b/src/nyx/output_writers.h index 53555ce7..fc4d6765 100644 --- a/src/nyx/output_writers.h +++ b/src/nyx/output_writers.h @@ -1,10 +1,21 @@ #pragma once -#define FTABLE_RECORD std::tuple,int,double,std::vector> +// FIX (IO): the record carries the index/calibration columns before the feature block: +// [0] text columns = {intensity_image, mask_image, phys_unit} (all leading strings, +// so the Python buffer's "strings first" reconstruction still holds) +// [1] ROI label (int) +// [2] t_index, [3] c_index, [4..6] phys_x/phys_y/phys_z (physical voxel spacing) +// [7..] feature values +// c_index was added for per-channel OME output; phys_* for physical calibration. +#define FTABLE_RECORD std::tuple,int,double,double,double,double,double,std::vector> #define FTABLE_INTSEG 0 #define FTABLE_ROILBL 1 #define FTABLE_TIMEPOS 2 -#define FTABLE_FBEGIN 3 +#define FTABLE_CPOS 3 +#define FTABLE_PXPOS 4 +#define FTABLE_PYPOS 5 +#define FTABLE_PZPOS 6 +#define FTABLE_FBEGIN 7 #define FTABLE_SAFENAN -0.0 #ifdef USE_ARROW diff --git a/src/nyx/phase1.cpp b/src/nyx/phase1.cpp index 51f48724..724c2aec 100644 --- a/src/nyx/phase1.cpp +++ b/src/nyx/phase1.cpp @@ -245,7 +245,7 @@ namespace Nyxus // segmented 3D case (true volumetric images e.g. .nii, .nii.gz, .dcm, etc) // prerequisite: 'env.theImLoader' needs to be pre-opened ! // - bool gatherRoisMetrics_3D (Environment& env, size_t sidx, const std::string& intens_fpath, const std::string& mask_fpath, size_t t_index) + bool gatherRoisMetrics_3D (Environment& env, size_t sidx, const std::string& intens_fpath, const std::string& mask_fpath, size_t t_index, size_t channel) { SlideProps & sprp = env.dataset.dataset_props [sidx]; if (! env.theImLoader.open(sprp, env.fpimageOptions)) @@ -279,47 +279,37 @@ namespace Nyxus int cnt = 1; - // Fetch a tile - bool ok = env.theImLoader.load_tile (0/*row*/ , 0/*col*/); + // FIX (IO): assemble the whole X*Y*Z volume for this timeframe (load_volume + // loops Z), so plane-by-plane loaders (OME-Zarr, multi-page/OME-TIFF) yield a + // real 3D volume here too — Phase-1 ROI metrics must see the same voxels the + // Phase-2 scan does. Mask on frame t_index (1:1) or frame 0 (1 mask : N int). + const size_t maskFrame = (nVoxI == nVoxM) ? t_index : 0; + bool ok = env.theImLoader.load_volume (channel, t_index, maskFrame); if (!ok) { std::string erm = "Error fetching tile (0,0) from I:" + intens_fpath + " M:" + mask_fpath; #ifdef WITH_PYTHON_H throw erm; - #endif + #endif std::cerr << erm << "\n"; return false; } - // Get ahold of tile's pixel buffer - auto dataI = env.theImLoader.get_int_tile_buffer(), - dataL = env.theImLoader.get_seg_tile_buffer(); + // Both buffers hold exactly this timeframe's X*Y*Z volume (frame already selected). + const std::vector& dataI = env.theImLoader.get_int_volume_buffer(); + const std::vector& dataL = env.theImLoader.get_seg_volume_buffer(); - size_t baseI, baseM; - if (nVoxI == nVoxM) - { - baseM = baseI = t_index * timeFrameSize; - } - else // nVoxI > nVoxM - { - baseM = 0; - baseI = t_index * timeFrameSize; - } - - // Iterate voxels + // Iterate the single-frame volume's voxels for (size_t i=0; i= w || y >= h || z >= d) @@ -330,7 +320,7 @@ namespace Nyxus label = 1; // Update pixel's ROI metrics - feed_pixel_2_metrics_3D (env.uniqueLabels, env.roiData, x, y, z, dataI[j], label, sidx); + feed_pixel_2_metrics_3D (env.uniqueLabels, env.roiData, x, y, z, dataI[i], label, sidx); } #ifdef WITH_PYTHON_H diff --git a/src/nyx/phase2_3d.cpp b/src/nyx/phase2_3d.cpp index f081ebd0..4e3d59c1 100644 --- a/src/nyx/phase2_3d.cpp +++ b/src/nyx/phase2_3d.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -13,15 +14,51 @@ #include "environment.h" #include "globals.h" +#include "helpers/helpers.h" // FIX (IO): Nyxus::near_eq for the physical-spacing resolver #include "helpers/fsystem.h" #include "helpers/timing.h" namespace Nyxus { + // FIX (IO): resolve the effective 3D voxel spacing for slide `sidx`. Explicit --aniso* + // (anisoOptions.customized()) always wins. Otherwise, when --use-physical-spacing is on, + // use the slide's OME PhysicalSize* ratio-normalized so the smallest axis == 1 (Nyxus's + // anisotropic path resamples by the multiplier, so ratios - not absolute units - are what + // correct for non-cubic voxels). Returns false + (1,1,1) when the grid is isotropic. + bool resolve_slide_anisotropy (const Environment& env, size_t sidx, double& ax, double& ay, double& az) + { + ax = ay = az = 1.0; + + if (env.anisoOptions.customized()) + { + ax = env.anisoOptions.get_aniso_x(); + ay = env.anisoOptions.get_aniso_y(); + az = env.anisoOptions.get_aniso_z(); + return true; + } + + if (env.use_physical_spacing() && sidx < env.dataset.dataset_props.size()) + { + const SlideProps& p = env.dataset.dataset_props[sidx]; + double sx = p.phys_x, sy = p.phys_y, sz = p.phys_z; + double mn = std::min(sx, std::min(sy, sz)); + if (mn > 0.0) + { + ax = sx / mn; ay = sy / mn; az = sz / mn; + // only take the anisotropic path if the voxels are actually non-cubic + if (! (Nyxus::near_eq(ax, 1.0) && Nyxus::near_eq(ay, 1.0) && Nyxus::near_eq(az, 1.0))) + return true; + } + ax = ay = az = 1.0; + } + + return false; + } + // // Loads ROI voxels into voxel clouds // - bool scanTrivialRois_3D (Environment & env, const std::vector& batch_labels, const std::string& intens_fpath, const std::string& label_fpath, size_t t_index) + bool scanTrivialRois_3D (Environment & env, const std::vector& batch_labels, const std::string& intens_fpath, const std::string& label_fpath, size_t t_index, size_t channel) { // Sort the batch's labels to enable binary searching in it std::vector whiteList = batch_labels; @@ -83,41 +120,33 @@ namespace Nyxus int cnt = 1; - // fetch 3D data - bool ok = env.theImLoader.load_tile (0/*row*/, 0/*col*/); + // FIX (IO): assemble the whole X*Y*Z volume for this timeframe via load_volume, + // which loops Z internally. The previous load_tile(0,0) delivered only plane + // z=0 for plane-by-plane loaders (OME-Zarr, multi-page/OME-TIFF), so those never + // read a real 3D volume. The mask sits on frame t_index (1:1) or frame 0 (the + // 1-mask : N-intensity case); the intensity sits on frame t_index. + const size_t maskFrame = (timeI == timeM) ? t_index : 0; + bool ok = env.theImLoader.load_volume (channel, t_index, maskFrame); if (!ok) { std::string erm = "Error fetching segmented data from " + intens_fpath + "(I) " + label_fpath + "(M)"; #ifdef WITH_PYTHON_H throw erm; -#endif +#endif std::cerr << erm << "\n"; return false; } - // Get ahold of tile's pixel buffer - auto dataI = env.theImLoader.get_int_tile_buffer(), - dataL = env.theImLoader.get_seg_tile_buffer(); + // Both buffers now hold exactly this timeframe's X*Y*Z volume (the frame was + // already selected by load_volume), so no base offset is needed. + const std::vector& dataI = env.theImLoader.get_int_volume_buffer(); + const std::vector& dataL = env.theImLoader.get_seg_volume_buffer(); - size_t baseI, baseM; - if (timeI == timeM) - { - baseM = baseI = t_index * timeFrameSize; - } - else // nVoxI > nVoxM + // Iterate the single-frame volume's voxels + for (size_t i = 0; i < timeFrameSize; i++) { - baseM = 0; - baseI = t_index * timeFrameSize; - } - - // Iterate voxels - for (size_t i=0; i= w || y >= h || z >= d) @@ -139,9 +166,9 @@ namespace Nyxus if (env.singleROI) label = 1; - // Cache this pixel + // Cache this pixel LR& r = env.roiData[label]; - feed_pixel_2_cache_3D_LR (x, y, z, dataI[j], r); + feed_pixel_2_cache_3D_LR (x, y, z, dataI[i], r); } #ifdef WITH_PYTHON_H @@ -290,6 +317,7 @@ namespace Nyxus const std::string& intens_fpath, const std::string& label_fpath, size_t t_index, + size_t channel, double aniso_x, double aniso_y, double aniso_z) @@ -343,35 +371,23 @@ namespace Nyxus int cnt = 1; - // fetch 3D data - if (!env.theImLoader.load_tile (0/*row*/, 0/*col*/)) + // FIX (IO): assemble the whole X*Y*Z volume for this timeframe (load_volume + // loops Z), so plane-by-plane loaders (OME-Zarr, multi-page/OME-TIFF) read a + // real 3D volume here too. Mask on frame t_index (1:1) or frame 0 (1 mask : N). + const size_t maskFrame = (timeI == timeM) ? t_index : 0; + if (!env.theImLoader.load_volume (channel, t_index, maskFrame)) { std::string erm = "Error fetching data from file pair " + intens_fpath + "(I) " + label_fpath + "(M)"; #ifdef WITH_PYTHON_H throw erm; - #endif + #endif std::cerr << erm << "\n"; return false; } - // get ahold of voxel buffers - auto dataI = env.theImLoader.get_int_tile_buffer(), - dataL = env.theImLoader.get_seg_tile_buffer(); - - // align time frame's mask and intensity volumes - size_t baseI, baseM; - if (timeI == timeM) - { - // trivial N mask : N intensity - baseM = - baseI = t_index * timeFrameSize; - } - else - { - // nontrivial 1 mask : N intensity - baseM = 0; - baseI = t_index * timeFrameSize; - } + // Both buffers hold exactly this timeframe's X*Y*Z volume (frame already selected). + const std::vector& dataI = env.theImLoader.get_int_volume_buffer(); + const std::vector& dataL = env.theImLoader.get_seg_volume_buffer(); // virtual dimensions size_t virt_h = h * aniso_y, @@ -408,7 +424,7 @@ namespace Nyxus // // skip non-mask pixels - auto lbl = dataL[baseM + i]; + auto lbl = dataL[i]; if (!lbl) continue; @@ -432,8 +448,8 @@ namespace Nyxus assert(vX < virt_w); #endif - // cache this voxel - auto inten = dataI[baseI + i]; + // cache this voxel + auto inten = dataI[i]; LR& r = env.roiData[lbl]; feed_pixel_2_cache_3D_LR (vX, vY, vZ, inten, r); } @@ -455,11 +471,10 @@ namespace Nyxus bool scan_trivial_wholevolume ( LR& vroi, const std::string& intens_fpath, - ImageLoader& ilo) + ImageLoader& ilo, + size_t channel, + size_t timeframe) { - int lvl = 0, // Pyramid level - lyr = 0; // Layer - // Read the tiffs size_t fullwidth = ilo.get_full_width(), @@ -468,18 +483,20 @@ namespace Nyxus sliceSize = fullwidth * fullheight, nVox = sliceSize * fullD; - // in the 3D case tiling is a formality, so fetch the only tile in the file - if (!ilo.load_tile(0, 0)) + // FIX (IO): assemble the whole X*Y*Z volume (load_volume loops Z) for the requested + // (channel, timeframe) so plane-by-plane loaders (OME-Zarr, multi-page/OME-TIFF) read + // the full volume of THIS C/T plane, not just plane z=0 of (c=0,t=0). No mask in WSI. + if (!ilo.load_volume(channel, timeframe, timeframe)) { #ifdef WITH_PYTHON_H throw "Error fetching tile"; -#endif +#endif std::cerr << "Error fetching tile\n"; return false; } - // Get ahold of tile's pixel buffer - const std::vector& dataI = ilo.get_int_tile_buffer(); + // Get ahold of the assembled volume buffer + const std::vector& dataI = ilo.get_int_volume_buffer(); // iterate abstract tiles (in a tiled slide /e.g. tiled tiff/ they correspond to physical tiles, in a nontiled slide /e.g. scanline tiff or strip tiff/ they correspond to ) int cnt = 1; @@ -515,11 +532,10 @@ namespace Nyxus ImageLoader& ilo, double aniso_x, double aniso_y, - double aniso_z) + double aniso_z, + size_t channel, + size_t timeframe) { - int lvl = 0, // Pyramid level - lyr = 0; // Layer - // Read the tiffs size_t fullW = ilo.get_full_width(), @@ -531,21 +547,22 @@ namespace Nyxus vw = (size_t) (double(fullW) * aniso_x), vd = (size_t) (double(fullD) * aniso_z); - // in the 3D case tiling is a formality, so fetch the only tile in the file - if (! ilo.load_tile(0, 0)) + // FIX (IO): assemble the whole X*Y*Z volume (load_volume loops Z) for the requested + // (channel, timeframe) so plane-by-plane loaders read the full volume of this C/T plane. + if (! ilo.load_volume(channel, timeframe, timeframe)) { #ifdef WITH_PYTHON_H throw "Error loading volume data"; -#endif +#endif std::cerr << "Error loading volume data\n"; return false; } - // Get ahold of tile's pixel buffer - const std::vector& dataI = ilo.get_int_tile_buffer(); + // Get ahold of the assembled volume buffer + const std::vector& dataI = ilo.get_int_volume_buffer(); // iterate virtual voxels - size_t vSliceSize = vh * vw, + size_t vSliceSize = vh * vw, nVox = vh * vw * vd; for (size_t i = 0; i < nVox; i++) { @@ -554,14 +571,18 @@ namespace Nyxus y = (i - z * vSliceSize) / vw, x = (i - z * vSliceSize) % vw; - // physical voxel position - size_t ph_x = (size_t) (double(x) / aniso_x), - ph_y = (size_t) (double(y) / aniso_y), - ph_z = (size_t) (double(z) / aniso_z); - i = ph_z * sliceSize + ph_y * fullH + ph_x; + // physical voxel position, clamped against float-rounding at the ratio boundary + // (own local variable -- must NOT reuse the loop counter 'i': doing so corrupted + // every subsequent iteration's z/y/x derivation above, since the loop's own i++ + // then advanced by the physical stride instead of by one virtual voxel, which + // could run far longer than nVox iterations before (if ever) satisfying i < nVox). + size_t ph_x = (std::min) ((size_t) (double(x) / aniso_x), fullW - 1), + ph_y = (std::min) ((size_t) (double(y) / aniso_y), fullH - 1), + ph_z = (std::min) ((size_t) (double(z) / aniso_z), fullD - 1); + size_t phys_i = ph_z * sliceSize + ph_y * fullW + ph_x; - // Cache this pixel - feed_pixel_2_cache_3D_LR (x, y, z, dataI[i], vroi); + // Cache this pixel + feed_pixel_2_cache_3D_LR (x, y, z, dataI[phys_i], vroi); } @@ -649,7 +670,7 @@ namespace Nyxus } - bool processTrivialRois_3D (Environment & env, size_t sidx, size_t t_index, const std::vector& trivRoiLabels, const std::string& intens_fpath, const std::string& label_fpath, size_t memory_limit) + bool processTrivialRois_3D (Environment & env, size_t sidx, size_t t_index, size_t channel, const std::vector& trivRoiLabels, const std::string& intens_fpath, const std::string& label_fpath, size_t memory_limit) { std::vector Pending; size_t batchDemand = 0; @@ -680,22 +701,26 @@ namespace Nyxus std::cout << ">>> (ROI labels " << Pending[0] << " ... " << Pending[Pending.size() - 1] << ")\n"; ); - if (env.anisoOptions.customized() == false) + // FIX (IO): --aniso* (explicit) or opt-in OME physical spacing selects the anisotropic path + double ax, ay, az; + if (! resolve_slide_anisotropy (env, sidx, ax, ay, az)) { - scanTrivialRois_3D (env, Pending, intens_fpath, label_fpath, t_index); + scanTrivialRois_3D (env, Pending, intens_fpath, label_fpath, t_index, channel); } else { - double ax = env.anisoOptions.get_aniso_x(), - ay = env.anisoOptions.get_aniso_y(), - az = env.anisoOptions.get_aniso_z(); - scanTrivialRois_3D_anisotropic (env, Pending, intens_fpath, label_fpath, t_index, ax, ay, az); + scanTrivialRois_3D_anisotropic (env, Pending, intens_fpath, label_fpath, t_index, channel, ax, ay, az); - // rescan and update ROI's AABB + // rescan and update ROI's AABB and voxel count -- gatherRoisMetrics_3D set + // aux_area from the PHYSICAL (pre-resample) voxel count; the anisotropic + // scan just populated raw_pixels_3D with the resampled (virtual) cloud, + // which is a different size. Features that divide by aux_area (e.g. MEAN) + // were silently off by the resampling factor without this. for (auto lbl : Pending) { LR& r = env.roiData[lbl]; r.aabb.update_from_voxelcloud (r.raw_pixels_3D); + r.aux_area = (unsigned int) r.raw_pixels_3D.size(); } } @@ -746,22 +771,23 @@ namespace Nyxus std::cout << ">>> (labels " << Pending[0] << " ... " << Pending[Pending.size() - 1] << ")\n"; ); - if (env.anisoOptions.customized() == false) + // FIX (IO): --aniso* (explicit) or opt-in OME physical spacing selects the anisotropic path + double ax, ay, az; + if (! resolve_slide_anisotropy (env, sidx, ax, ay, az)) { - scanTrivialRois_3D (env, Pending, intens_fpath, label_fpath, t_index); + scanTrivialRois_3D (env, Pending, intens_fpath, label_fpath, t_index, channel); } else { - double ax = env.anisoOptions.get_aniso_x(), - ay = env.anisoOptions.get_aniso_y(), - az = env.anisoOptions.get_aniso_z(); - scanTrivialRois_3D_anisotropic (env, Pending, intens_fpath, label_fpath, t_index, ax, ay, az); + scanTrivialRois_3D_anisotropic (env, Pending, intens_fpath, label_fpath, t_index, channel, ax, ay, az); - // rescan and update ROI's AABB + // rescan and update ROI's AABB and voxel count -- see the identical fix (and + // its rationale) in the main batch loop above. for (auto lbl : Pending) { LR& r = env.roiData[lbl]; r.aabb.update_from_voxelcloud(r.raw_pixels_3D); + r.aux_area = (unsigned int) r.raw_pixels_3D.size(); } } diff --git a/src/nyx/phase3.cpp b/src/nyx/phase3.cpp index 8650a187..86201fd5 100644 --- a/src/nyx/phase3.cpp +++ b/src/nyx/phase3.cpp @@ -99,7 +99,9 @@ namespace Nyxus try { const Fsettings& s = env.get_feature_settings (typeid(f)); - f->osized_scan_whole_image (r, s, env.theImLoader); + // Pass the Dataset so intensity/histogram osized features reach their + // Dataset-aware osized_calculate; the Dataset-less overload is a guard that throws. + f->osized_scan_whole_image (r, s, env.dataset, env.theImLoader); } catch (std::exception const& e) { diff --git a/src/nyx/phase3_3d.cpp b/src/nyx/phase3_3d.cpp new file mode 100644 index 00000000..89fcb855 --- /dev/null +++ b/src/nyx/phase3_3d.cpp @@ -0,0 +1,189 @@ +#include +#include +#include +#include +#include + +#ifdef WITH_PYTHON_H + #include +#endif + +#include "environment.h" +#include "feature_mgr.h" +#include "globals.h" +#include "features/3d_intensity.h" +#include "features/3d_surface.h" +#include "features/3d_glcm.h" +#include "features/3d_gldm.h" +#include "features/3d_ngldm.h" +#include "features/3d_ngtdm.h" +#include "features/3d_glrlm.h" +#include "features/3d_glszm.h" +#include "features/3d_gldzm.h" +#include "features/pixel.h" +#include "helpers/timing.h" + +namespace Nyxus +{ + /// @brief Returns true if 'f' is one of the 3D feature classes whose osized_calculate() streams + /// from the disk-backed voxel cloud (r.raw_voxels_NT) instead of reading the in-memory cube. + /// Every other 3D FeatureMethod (e.g. a future feature added without a streaming path) would + /// silently read an empty cube if allowed through, so processNontrivialRois_3D()'s per-feature + /// guard calls this and fails loudly on false. Kept as a standalone function (rather than inline + /// in the guard) so the allow-list itself is unit-testable independent of the streaming plumbing. + bool is_3d_ooc_supported (FeatureMethod* f) + { + return dynamic_cast(f) != nullptr + || dynamic_cast(f) != nullptr + || dynamic_cast(f) != nullptr + || dynamic_cast(f) != nullptr + || dynamic_cast(f) != nullptr + || dynamic_cast(f) != nullptr + || dynamic_cast(f) != nullptr + || dynamic_cast(f) != nullptr + || dynamic_cast(f) != nullptr; + } + + /// @brief Populates r.raw_voxels_NT by streaming the volume plane-by-plane. When 'wholevolume' + /// is false (the segmented-ROI case), only voxels whose mask value equals r.label are kept -- + /// segPlane must be non-empty (a real mask loader). When 'wholevolume' is true, every voxel is + /// kept regardless of any mask (there is none: workflow_3d_whole.cpp opens the loader with an + /// empty label path, so ImageLoader::stream_volume_planes's segPlane is empty in that case). + /// Shared by the segmented (processNontrivialRois_3D) and whole-volume out-of-core paths so + /// both stream through the exact same primitive. Returns false (cloud cleared) if the loader + /// can't deliver the volume plane-by-plane (e.g. NIfTI, a whole-4D-in-one-read format). + bool populate_3d_voxel_cloud (ImageLoader& imlo, LR& r, size_t channel, size_t timeframe, bool wholevolume) + { + r.raw_voxels_NT.init (r.label, "raw_voxels_NT"); + + const size_t W = imlo.get_full_width(), + H = imlo.get_full_height(); + const uint32_t want = (uint32_t) r.label; + + bool streamed = imlo.stream_volume_planes (channel, timeframe, timeframe, + [&](size_t z, const std::vector& intPlane, const std::vector& segPlane) + { + r.raw_voxels_NT.begin_slab (z); + for (size_t y = 0; y < H; y++) + for (size_t x = 0; x < W; x++) + { + size_t i = y * W + x; + if (wholevolume || segPlane[i] == want) + r.raw_voxels_NT.add_voxel (Pixel3(x, y, z, intPlane[i])); + } + }); + + if (! streamed) + r.raw_voxels_NT.clear(); + return streamed; + } + + /// @brief Runs every requested feature's out-of-core path over an already-populated + /// r.raw_voxels_NT, guarded by is_3d_ooc_supported(); writes results into r.fvals via + /// save_value(). Shared by the segmented and whole-volume out-of-core paths. 'imloader' is + /// passed through to match FeatureMethod's signature but is not read by any of the streaming + /// 3D features (they read raw_voxels_NT exclusively), so either path's own loader instance works. + void run_3d_ooc_features (Environment& env, LR& r, ImageLoader& imloader) + { + int nrf = env.theFeatureMgr.get_num_requested_features(); + for (int i = 0; i < nrf; i++) + { + auto f = env.theFeatureMgr.get_feature_method (i); + + try + { + // Every 3D feature that streams from raw_voxels_NT is covered by + // is_3d_ooc_supported(); anything else would silently read an empty cube, so fail + // loudly per-feature instead of emitting wrong values. Under the CLI this logs and + // moves on (the supported features still compute); under Python it raises. + if (! is_3d_ooc_supported (f)) + throw std::runtime_error("feature '" + f->feature_info + + "' is not yet supported out-of-core for oversized 3D ROIs; " + + "segment into smaller ROIs, raise --ramLimit, or add RAM"); + + const Fsettings& s = env.get_feature_settings (typeid(f)); + f->osized_scan_whole_image (r, s, env.dataset, imloader); + } + catch (std::exception const& e) + { + std::string erm = "Error while computing feature " + f->feature_info + " over oversized 3D ROI " + std::to_string(r.label) + " : " + e.what(); +#ifdef WITH_PYTHON_H + throw std::runtime_error(erm); +#endif + std::cerr << erm << "\n"; + } + + f->cleanup_instance(); + } + } + + /// @brief Processes oversized volumetric (3D) ROIs out-of-core: streams the voxel cloud to + /// disk one Z-plane at a time (bounded memory) instead of holding the whole cube. + /// @param nontrivRoiLabels Labels of ROIs whose in-memory footprint exceeds the RAM limit + /// @param intens_fpath Intensity image path + /// @param label_fpath Mask image path + /// @param channel Intensity channel being featurized + /// @param timeframe Timeframe being featurized (used for both intensity and mask) + /// @return Success status + /// + bool processNontrivialRois_3D (Environment& env, const std::vector& nontrivRoiLabels, const std::string& intens_fpath, const std::string& label_fpath, size_t channel, size_t timeframe) + { + // Sort labels for reproducibility with the trivial counterpart + auto L = nontrivRoiLabels; + std::sort (L.begin(), L.end()); + + for (auto lab : L) + { + LR& r = env.roiData[lab]; + + VERBOSLVL1 (env.get_verbosity_level(), std::cout << "processing oversized 3D ROI " << lab << "\n"); + + // Scan one label-intensity pair + SlideProps p (intens_fpath, label_fpath); + // Preserve the scanned HU-domain slide min/max + flag so load-time HU offset matches the + // prescan (bare SlideProps defaults min to -1). Guarded on preserve_hu so the default path + // is unchanged. Mirrors the 2D processNontrivialRois. + if (env.fpimageOptions.preserve_hu() && r.slide_idx >= 0) + { + const SlideProps& scanned = env.dataset.dataset_props [r.slide_idx]; + p.min_preroi_inten = scanned.min_preroi_inten; + p.max_preroi_inten = scanned.max_preroi_inten; + p.preserve_hu = scanned.preserve_hu; + } + if (! env.theImLoader.open(p, env.fpimageOptions)) + { + std::cout << "Terminating\n"; + return false; + } + + // Populate the ROI's disk-backed voxel cloud by streaming the volume plane-by-plane. + // Only this ROI's label voxels are written; z is preserved. Peak memory is two X*Y + // planes (intensity + mask) plus this plane's ROI voxels, never the whole cube. + if (! populate_3d_voxel_cloud (env.theImLoader, r, channel, timeframe, /*wholevolume=*/ false)) + { + std::string erm = "Error: out-of-core featurization of oversized 3D ROI " + std::to_string(r.label) + + " is not supported for this input format (the volume is not delivered plane-by-plane). " + + "Segment into smaller ROIs, raise --ramLimit, or add RAM."; +#ifdef WITH_PYTHON_H + throw std::runtime_error(erm); +#endif + std::cerr << erm << "\n"; + continue; + } + + //=== Reduce features over the streamed voxel cloud + run_3d_ooc_features (env, r, env.theImLoader); + + //=== Clean the ROI's cache + r.raw_voxels_NT.clear(); + + #ifdef WITH_PYTHON_H + // Allow keyboard interrupt + if (PyErr_CheckSignals() != 0) + throw pybind11::error_already_set(); + #endif + } + + return true; + } +} diff --git a/src/nyx/python/new_bindings_py.cpp b/src/nyx/python/new_bindings_py.cpp index c23c29ce..e5582243 100644 --- a/src/nyx/python/new_bindings_py.cpp +++ b/src/nyx/python/new_bindings_py.cpp @@ -81,7 +81,8 @@ void initialize_environment( float aniso_y, float aniso_z, bool merge_labels = false, - bool preserve_hu = false) // CT/HU: preserve absolute Hounsfield intensities + bool preserve_hu = false, // CT/HU: preserve absolute Hounsfield intensities + bool use_physical_spacing = false) // FIX (IO): opt-in OME PhysicalSize* voxel spacing (3D) { Environment & theEnvironment = Nyxus::findenv (instid); @@ -103,6 +104,7 @@ void initialize_environment( theEnvironment.fpimageOptions.set_min_intensity(min_intensity); theEnvironment.fpimageOptions.set_max_intensity(max_intensity); theEnvironment.fpimageOptions.set_preserve_hu(preserve_hu); + theEnvironment.use_physical_spacing_ = use_physical_spacing; // FIX (IO): opt-in physical spacing // Throws exception if invalid feature is passed theEnvironment.expand_featuregroups(); diff --git a/src/nyx/python/nyxus/nyxus.py b/src/nyx/python/nyxus/nyxus.py index 05a87874..d940b607 100644 --- a/src/nyx/python/nyxus/nyxus.py +++ b/src/nyx/python/nyxus/nyxus.py @@ -154,7 +154,7 @@ def __init__( 'gabor_thold', 'gabor_thetas', 'gabor_freqs', 'channel_signature', 'parent_channel', 'child_channel', 'aggregate', 'dynamic_range', 'min_intensity', 'max_intensity', 'ram_limit', 'verbose', - 'anisotropy_x', 'anisotropy_y', 'mergerois', 'preserve_hu' + 'anisotropy_x', 'anisotropy_y', 'mergerois', 'preserve_hu', 'use_physical_spacing' } # Check for unexpected keyword arguments @@ -181,6 +181,7 @@ def __init__( min_intensity = kwargs.get('min_intensity', 0.0) max_intensity = kwargs.get('max_intensity', 1.0) preserve_hu = kwargs.get('preserve_hu', False) # CT/HU: preserve absolute Hounsfield values + use_physical_spacing = kwargs.get('use_physical_spacing', False) # opt-in OME PhysicalSize* voxel spacing (3D) ram_limit = kwargs.get('ram_limit', -1) verb_lvl = kwargs.get('verbose', 0) aniso_x = kwargs.get('anisotropy_x', 1.0) @@ -233,7 +234,8 @@ def __init__( aniso_y, aniso_z, mergerois, - preserve_hu) + preserve_hu, + use_physical_spacing) self.set_gabor_feature_params( kersize = gabor_kersize, @@ -1001,7 +1003,7 @@ def __init__( 'verbose', 'anisotropy_x', 'anisotropy_y', - 'anisotropy_z', 'preserve_hu' + 'anisotropy_z', 'preserve_hu', 'use_physical_spacing' } # Check for unexpected keyword arguments @@ -1021,6 +1023,7 @@ def __init__( min_intensity = kwargs.get('min_intensity', 0.0) max_intensity = kwargs.get('max_intensity', 1.0) preserve_hu = kwargs.get('preserve_hu', False) # CT/HU: preserve absolute Hounsfield values + use_physical_spacing = kwargs.get('use_physical_spacing', False) # opt-in OME PhysicalSize* voxel spacing (3D) verb_lvl = kwargs.get ('verbose', 0) aniso_x = kwargs.get('anisotropy_x', 1.0) aniso_y = kwargs.get('anisotropy_y', 1.0) @@ -1080,7 +1083,8 @@ def __init__( aniso_y, aniso_z, False, # merge_labels: 2D-segmented only - preserve_hu) + preserve_hu, + use_physical_spacing) # list of valid outputs that are used throughout featurize functions self._valid_output_types = ['pandas', 'arrowipc', 'parquet'] @@ -1570,6 +1574,7 @@ def __init__( min_intensity = kwargs.get('min_intensity', 0.0) max_intensity = kwargs.get('max_intensity', 1.0) preserve_hu = kwargs.get('preserve_hu', False) # CT/HU: preserve absolute Hounsfield values + use_physical_spacing = kwargs.get('use_physical_spacing', False) # opt-in OME PhysicalSize* voxel spacing (3D) ram_limit = kwargs.get('ram_limit', -1) verb_lvl = kwargs.get ('verbose', 0) aniso_x = kwargs.get('anisotropy_x', 1.0) @@ -1626,7 +1631,8 @@ def __init__( aniso_y, aniso_z, False, # merge_labels: 2D-segmented only - preserve_hu) + preserve_hu, + use_physical_spacing) # list of valid outputs that are used throughout featurize functions self._valid_output_types = ['pandas', 'arrowipc', 'parquet'] diff --git a/src/nyx/raw_dicom.h b/src/nyx/raw_dicom.h index ad4d602c..e0e8e054 100644 --- a/src/nyx/raw_dicom.h +++ b/src/nyx/raw_dicom.h @@ -133,6 +133,8 @@ class RawDicomLoader : public RawFormatLoader size_t indexRowGlobalTile, size_t indexColGlobalTile, size_t indexLayerGlobalTile, + [[maybe_unused]] size_t indexChannel, // DICOM grayscale: single channel + [[maybe_unused]] size_t indexTimeframe, // DICOM: frames are WSI tiles, not time [[maybe_unused]] size_t level) override { tile.resize (tileWidth_ * tileHeight_); diff --git a/src/nyx/raw_format.h b/src/nyx/raw_format.h index 7dc9de27..76ad8cb3 100644 --- a/src/nyx/raw_format.h +++ b/src/nyx/raw_format.h @@ -19,7 +19,9 @@ class RawFormatLoader virtual void loadTileFromFile ( size_t indexRowGlobalTile, size_t indexColGlobalTile, - size_t indexLayerGlobalTile, + size_t indexLayerGlobalTile, // Z + size_t indexChannel, // C; 0 for single-channel data + size_t indexTimeframe, // T; 0 for non-time-series data size_t level) = 0; virtual void free_tile() = 0; // must follow loadTileFromFile() @@ -44,6 +46,13 @@ class RawFormatLoader return 1; } + // Physical voxel spacing along X/Y/Z (default 1.0 == uncalibrated) + unit string. + // OME loaders return the parsed PhysicalSize*; everything else stays 1.0 / "". + [[nodiscard]] virtual double physicalSizeX() const { return 1.0; } + [[nodiscard]] virtual double physicalSizeY() const { return 1.0; } + [[nodiscard]] virtual double physicalSizeZ() const { return 1.0; } + [[nodiscard]] virtual std::string physicalSizeUnit() const { return std::string(); } + [[nodiscard]] virtual size_t tileWidth(size_t level) const = 0; [[nodiscard]] virtual size_t tileHeight(size_t level) const = 0; @@ -52,6 +61,10 @@ class RawFormatLoader return 1; } + [[nodiscard]] virtual size_t tileTimestamps([[maybe_unused]] size_t level) const { + return 1; + } + [[nodiscard]] virtual short bitsPerSample() const = 0; [[nodiscard]] virtual size_t numberPyramidLevels() const = 0; diff --git a/src/nyx/raw_image_loader.cpp b/src/nyx/raw_image_loader.cpp index 350d1ba6..04180498 100644 --- a/src/nyx/raw_image_loader.cpp +++ b/src/nyx/raw_image_loader.cpp @@ -1,4 +1,4 @@ -#include +#include #define NOMINMAX @@ -9,6 +9,7 @@ #include "raw_nifti.h" #include "raw_omezarr.h" #include "raw_tiff.h" +#include "ome/format_detect.h" // container-family classification for loader dispatch RawImageLoader::RawImageLoader() {} @@ -16,9 +17,11 @@ bool RawImageLoader::open (const std::string& int_fpath, const std::string& seg_ { try { - std::string ext = Nyxus::get_big_extension(int_fpath); + // Classify by container family so .ome.zarr routes to Zarr and the DICOM test is a + // clean equality comparison. + Nyxus::ContainerKind fmt = Nyxus::detect_container_family (int_fpath); - if (ext == ".zarr") + if (fmt == Nyxus::ContainerKind::OmeZarr) { #ifdef OMEZARR_SUPPORT intFL = new RawOmezarrLoader (int_fpath); @@ -26,8 +29,8 @@ bool RawImageLoader::open (const std::string& int_fpath, const std::string& seg_ std::cout << "This version of Nyxus was not build with OmeZarr support." << std::endl; #endif } - else - if (ext == ".dcm" | ext == ".dicom") { + else + if (fmt == Nyxus::ContainerKind::Dicom) { #ifdef DICOM_SUPPORT intFL = new RawDicomLoader (int_fpath, preserve_hu); // CT/HU: scan in Hounsfield domain #else @@ -35,9 +38,9 @@ bool RawImageLoader::open (const std::string& int_fpath, const std::string& seg_ #endif } else - if (ext == ".nii" || ext == ".nii.gz") + if (fmt == Nyxus::ContainerKind::Nifti) { - intFL = new RawNiftiLoader (int_fpath, preserve_hu); // FIX: CT/HU: scan in Hounsfield domain (matches DICOM) + intFL = new RawNiftiLoader (int_fpath, preserve_hu); // CT/HU: scan in Hounsfield domain (matches DICOM) } else { @@ -82,8 +85,9 @@ bool RawImageLoader::open (const std::string& int_fpath, const std::string& seg_ // segmented slide try { - std::string ext = Nyxus::get_big_extension(seg_fpath); - if (ext == ".zarr") + // Classify the mask by the same container family as the intensity path. + Nyxus::ContainerKind fmt = Nyxus::detect_container_family (seg_fpath); + if (fmt == Nyxus::ContainerKind::OmeZarr) { #ifdef OMEZARR_SUPPORT segFL = new RawOmezarrLoader (seg_fpath); @@ -91,8 +95,8 @@ bool RawImageLoader::open (const std::string& int_fpath, const std::string& seg_ std::cout << "This version of Nyxus was not build with OmeZarr support." << std::endl; #endif } - else - if (ext == ".dcm" | ext == ".dicom") + else + if (fmt == Nyxus::ContainerKind::Dicom) { #ifdef DICOM_SUPPORT segFL = new RawDicomLoader (seg_fpath); @@ -101,7 +105,7 @@ bool RawImageLoader::open (const std::string& int_fpath, const std::string& seg_ #endif } else - if (ext == ".nii" || ext == ".nii.gz") + if (fmt == Nyxus::ContainerKind::Nifti) { segFL = new RawNiftiLoader (seg_fpath); } @@ -183,11 +187,11 @@ bool RawImageLoader::load_tile(size_t tile_idx) auto row = tile_idx / ntw; auto col = tile_idx % ntw; - intFL->loadTileFromFile (row, col, lyr, lvl); + intFL->loadTileFromFile (row, col, lyr, cur_channel, cur_timeframe, lvl); // segmentation loader is not available in wholeslide if (segFL) - segFL->loadTileFromFile (row, col, lyr, lvl); + segFL->loadTileFromFile (row, col, lyr, cur_channel, cur_timeframe, lvl); return true; } @@ -197,12 +201,12 @@ bool RawImageLoader::load_tile(size_t tile_row, size_t tile_col) if (tile_row >= nth || tile_col >= ntw) return false; - intFL->loadTileFromFile (tile_row, tile_col, lyr, lvl); - + intFL->loadTileFromFile (tile_row, tile_col, lyr, cur_channel, cur_timeframe, lvl); + // segmentation loader is not available in wholeslide if (segFL) - segFL->loadTileFromFile (tile_row, tile_col, lyr, lvl); - + segFL->loadTileFromFile (tile_row, tile_col, lyr, cur_channel, cur_timeframe, lvl); + return true; } @@ -289,6 +293,31 @@ size_t RawImageLoader::get_mask_time() return 0; // whole-slide mode } +size_t RawImageLoader::get_inten_channels() +{ + return intFL->numberChannels(); // FIX (IO): OME loaders report the real C; others default to 1 +} + +double RawImageLoader::get_physical_size_x() +{ + return intFL->physicalSizeX(); // FIX (IO): OME PhysicalSizeX; 1.0 if uncalibrated +} + +double RawImageLoader::get_physical_size_y() +{ + return intFL->physicalSizeY(); +} + +double RawImageLoader::get_physical_size_z() +{ + return intFL->physicalSizeZ(); +} + +std::string RawImageLoader::get_physical_size_unit() +{ + return intFL->physicalSizeUnit(); +} + std::string RawImageLoader::get_slide_descr() { std::string s = get_fp_phys_pixvoxels() ? "R-" : "N-"; diff --git a/src/nyx/raw_image_loader.h b/src/nyx/raw_image_loader.h index 62d50645..7840e716 100644 --- a/src/nyx/raw_image_loader.h +++ b/src/nyx/raw_image_loader.h @@ -21,6 +21,83 @@ class RawImageLoader uint32_t get_cur_tile_seg_pixel(size_t pixel_idx); double get_cur_tile_dpequiv_pixel(size_t idx); + // FIX: visit every voxel of one (channel, timeframe) volume WITHOUT materializing it. + // The 3D prescan used to do ONE load_tile(0,0) and then index W*H*D voxels off the tile + // buffer -- but a per-plane loader (OME-TIFF / OME-Zarr) fills only ONE Z-plane, so + // everything past the first slice read out of bounds and corrupted the slide min/max. + // (It also assumed the tile stride equals the full width.) Streaming rather than filling + // a W*H*D staging buffer matters because the prescan repeats this for every (c,t) and + // only needs running min/max plus mask-driven ROI geometry: a double buffer would cost + // 4x the raw uint16 volume in RAM per pass and buy nothing. + // + // Walks the tile GRID of each plane, so planes spanning several tiles/chunks are handled; + // NIfTI-style whole-4D loaders (tileDepth == full depth) are slabbed out via frameBase. + // fn is invoked for EVERY voxel, as fn(x, y, z, intensity, msk) -- msk may be 0 for an + // off-ROI voxel in segmented mode (the caller decides what, if anything, to do with it); + // in whole-slide mode every voxel is in-mask with msk == 1. + template + bool for_each_voxel (size_t channel, size_t timeframe, F&& fn) + { + const bool haveSeg = (segFL != nullptr); + + // Use each loader's OWN layout: per-plane loaders (OME-Zarr, multi-page/OME-TIFF) + // deliver one Z-plane per read (tileDepth == 1); a whole-4D loader delivers the entire + // x*y*z*t blob in one read and ignores the layer arg. Mirrors ImageLoader::assemble_volume. + const size_t ltd = intFL->tileDepth (lvl), + ltt = intFL->tileTimestamps (lvl), + lntd = intFL->numberTileDepth (lvl), + lnth = intFL->numberTileHeight (lvl), + lntw = intFL->numberTileWidth (lvl), + frameStride = ltd * th * tw, + frameBase = (ltt > 1) ? timeframe * frameStride : 0; + + // The mask is channel-/timeframe-agnostic unless it genuinely has that many. FIX: a + // single-timeframe mask (the common 1-mask : N-timeframe-intensity case) was read at + // the intensity's `timeframe`, so for T>1 the TIFF mask loader computed an IFD past the + // end (ifdForPlane with t>0) and TIFFSetDirectory threw -- uncaught here, crashing the + // process (0xC0000409). Clamp the mask timeframe like the channel. (Zarr masks have no + // T axis to over-index, so only TIFF crashed.) + const size_t maskChannel = (haveSeg && channel < segFL->numberChannels()) ? channel : 0; + const size_t maskTimeframe = (haveSeg && timeframe < segFL->fullTimestamps (lvl)) ? timeframe : 0; + + for (size_t lz = 0; lz < lntd; lz++) + { + for (size_t tr = 0; tr < lnth; tr++) + for (size_t tc = 0; tc < lntw; tc++) + { + intFL->loadTileFromFile (tr, tc, lz, channel, timeframe, lvl); + if (haveSeg) + segFL->loadTileFromFile (tr, tc, lz, maskChannel, maskTimeframe, lvl); + + const size_t row0 = tr * th, col0 = tc * tw; + if (row0 < fh && col0 < fw) + { + const size_t validH = (std::min) (th, fh - row0), + validW = (std::min) (tw, fw - col0); + + for (size_t pz = 0; pz < ltd && (lz * ltd + pz) < fd; pz++) + { + const size_t gz = lz * ltd + pz; + for (size_t row = 0; row < validH; row++) + for (size_t col = 0; col < validW; col++) + { + const size_t src = frameBase + (pz * th + row) * tw + col; + const uint32_t msk = haveSeg ? segFL->get_uint32_pixel (src) : (uint32_t)1; + fn (col0 + col, row0 + row, gz, intFL->get_dpequiv_pixel (src), msk); + } + } + } + + // the raw TIFF loaders malloc their tile buffer on each read (no-op elsewhere) + intFL->free_tile(); + if (haveSeg) + segFL->free_tile(); + } + } + + return true; + } + size_t get_tile_size(); size_t get_num_tiles_vert(); size_t get_num_tiles_hor(); @@ -34,6 +111,11 @@ class RawImageLoader size_t get_full_depth(); size_t get_inten_time(); size_t get_mask_time(); + size_t get_inten_channels(); // FIX (IO): number of intensity channels (>=1) + double get_physical_size_x(); // FIX (IO): physical voxel spacing (1.0 if uncalibrated) + double get_physical_size_y(); + double get_physical_size_z(); + std::string get_physical_size_unit(); std::string get_slide_descr(); bool get_fp_phys_pixvoxels(); @@ -63,5 +145,10 @@ class RawImageLoader int lvl = 0, // Pyramid level lyr = 0; // Layer + // Channel (C) / timeframe (T) plane that load_tile() reads. for_each_voxel() takes the + // plane as an argument instead; 0/0 (single-channel, single-timepoint) otherwise. + size_t cur_channel = 0, + cur_timeframe = 0; + }; diff --git a/src/nyx/raw_nifti.h b/src/nyx/raw_nifti.h index ad85db9e..e70026a1 100644 --- a/src/nyx/raw_nifti.h +++ b/src/nyx/raw_nifti.h @@ -101,6 +101,8 @@ class RawNiftiLoader : public RawFormatLoader size_t indexRowGlobalTile, size_t indexColGlobalTile, size_t indexLayerGlobalTile, + [[maybe_unused]] size_t indexChannel, // NIfTI: single channel + [[maybe_unused]] size_t indexTimeframe, // whole 4D volume held in memory; T sliced by consumer [[maybe_unused]] size_t level) override {} // NIFTI is not tiled @@ -223,6 +225,8 @@ class NiftiLoader : public AbstractTileLoader size_t indexRowGlobalTile, size_t indexColGlobalTile, size_t indexLayerGlobalTile, + [[maybe_unused]] size_t indexChannel, // NIfTI: single channel + [[maybe_unused]] size_t indexTimeframe, // whole 4D volume delivered at once; T sliced by consumer [[maybe_unused]] size_t level) override { tile->resize (tile_width_ * tile_height_ * tile_depth_ * numTimeFrames_); diff --git a/src/nyx/raw_omezarr.h b/src/nyx/raw_omezarr.h index 687814f9..0d4018c3 100644 --- a/src/nyx/raw_omezarr.h +++ b/src/nyx/raw_omezarr.h @@ -3,7 +3,10 @@ #ifdef OMEZARR_SUPPORT #include +#include +#include #include "nlohmann/json.hpp" +#include "ome/ome_zarr_layout.h" // resolve_zarr_layout() -- shared with omezarr.h // factory functions to create files, groups and datasets #include "z5/factory.hxx" @@ -29,43 +32,35 @@ class RawOmezarrLoader: public RawFormatLoader { // Open the file zarr_ptr_ = std::make_unique(filePath.c_str()); - nlohmann::json file_attributes, ds_attributes; + nlohmann::json file_attributes; z5::readAttributes(*zarr_ptr_, file_attributes); - // assume only one dataset is present - ds_name_ = file_attributes["multiscales"][0]["datasets"][0]["path"].get(); - const auto ds_handle = z5::filesystem::handle::Dataset(*zarr_ptr_, ds_name_); - fs::path metadata_path; - auto success = z5::filesystem::metadata_detail::getMetadataPath(ds_handle, metadata_path); - z5::filesystem::metadata_detail::readMetadata(metadata_path, ds_attributes); - - full_depth_ = ds_attributes["shape"][2].get(); - full_height_ = ds_attributes["shape"][3].get(); - full_width_ = ds_attributes["shape"][4].get(); - tile_depth_ = ds_attributes["chunks"][2].get(); - tile_height_ = ds_attributes["chunks"][3].get(); - tile_width_ = ds_attributes["chunks"][4].get(); - std::string dtype_str = ds_attributes["dtype"].get(); - if (dtype_str == " (tile_height_ * tile_width_); - - // Open the dataset once and cache the handle. The dataset metadata is - // immutable for the lifetime of this loader, so there is no need to - // re-open (and re-parse the .zarray metadata) on every tile read. + // Resolve the level-0 dataset path + ds_name_ = Nyxus::zarr_multiscales_root(file_attributes)["multiscales"][0]["datasets"][0]["path"].get(); + + // FIX (IO): open via z5 (auto-detects Zarr v2 .zarray vs v3 zarr.json, handles v3 + // chunk-key encoding, codecs and sharding) and query shape/chunking/dtype from the + // Dataset object -- format-agnostic, so this reads BOTH OME-Zarr 0.4 and 0.5. ds_ = z5::openDataset(*zarr_ptr_, ds_name_); + std::vector level0Shape(ds_->shape().begin(), ds_->shape().end()); + std::vector chunkShape(ds_->defaultChunkShape().begin(), ds_->defaultChunkShape().end()); + + // Axis roles, extents, chunking, calibration and pixel type, resolved from the NGFF + // 'axes' metadata (shared with NyxusOmeZarrLoader -- see ome/ome_zarr_layout.h) + Nyxus::ZarrLayout L = Nyxus::resolve_zarr_layout (file_attributes, level0Shape, chunkShape, ds_->getDtype()); + ndim_ = L.ndim; + ix_ = L.ix; iy_ = L.iy; iz_ = L.iz; ic_ = L.ic; it_ = L.it; + full_width_ = L.full_width; full_height_ = L.full_height; full_depth_ = L.full_depth; + tile_width_ = L.tile_width; tile_height_ = L.tile_height; tile_depth_ = L.tile_depth; + n_levels_ = L.n_levels; n_channels_ = L.n_channels; n_timeframes_ = L.n_timeframes; + phys_x_ = L.phys_x; phys_y_ = L.phys_y; phys_z_ = L.phys_z; phys_unit_ = L.phys_unit; + bits_per_sample_ = L.bits_per_sample; + data_format_ = L.data_format; + fp_pixels_ = L.fp_pixels; + + // The buffer holds the full chunk depth (tile_depth_ planes), so callers iterating + // pz in [0, tileDepth()) index valid data. + dest = std::vector (tile_height_ * tile_width_ * tile_depth_); } ~RawOmezarrLoader() override @@ -78,6 +73,8 @@ class RawOmezarrLoader: public RawFormatLoader size_t indexRowGlobalTile, size_t indexColGlobalTile, size_t indexLayerGlobalTile, + size_t indexChannel, // C plane to read (offset into the channel axis) + size_t indexTimeframe, // T plane to read (offset into the time axis) [[maybe_unused]] size_t level) override { size_t pixel_row_index = indexRowGlobalTile * tile_height_; @@ -87,37 +84,37 @@ class RawOmezarrLoader: public RawFormatLoader switch (data_format_) { case 1: - loadTile(pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; case 2: - loadTile(pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; case 3: - loadTile(pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; case 4: - loadTile(pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; case 5: - loadTile(pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; case 6: - loadTile(pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; case 7: - loadTile(pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; case 8: - loadTile(pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; case 9: - loadTile(pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; case 10: - loadTile(pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; default: - loadTile(pixel_row_index, pixel_col_index, pixel_layer_index); + loadTile(pixel_row_index, pixel_col_index, pixel_layer_index, indexChannel, indexTimeframe); break; } } @@ -139,7 +136,8 @@ class RawOmezarrLoader: public RawFormatLoader } template - void loadTile(size_t pixel_row_index, size_t pixel_col_index, size_t pixel_layer_index) + void loadTile(size_t pixel_row_index, size_t pixel_col_index, size_t pixel_layer_index, + size_t pixel_channel_index, size_t pixel_timeframe_index) { size_t data_height = tile_height_, data_width = tile_width_; if (pixel_row_index + data_height > full_height_) { @@ -148,25 +146,42 @@ class RawOmezarrLoader: public RawFormatLoader if (pixel_col_index + data_width > full_width_) { data_width = full_width_ - pixel_col_index; } + // A chunk may span several Z-planes (tile_depth_ > 1); read its whole Z-extent in one + // subarray call, clamped at the last (possibly partial) chunk. + size_t data_depth = 1; + if (iz_ >= 0) { + data_depth = tile_depth_; + if (pixel_layer_index + data_depth > full_depth_) + data_depth = full_depth_ - pixel_layer_index; + } // Create a buffer to hold the read data - std::vector buffer(data_height * data_width); - - // Create an ArrayView into the buffer (z5 3.0.1 uses ArrayView instead of xtensor) - z5::types::ShapeType shape = {1, 1, 1, data_height, data_width}; + std::vector buffer(data_height * data_width * data_depth); + + // Build the read window by axis ROLE (honoring the resolved 'axes' order), + // reading a Z*Y*X block (data_depth Z-planes) at the requested C/T. z5 3.0.1 uses ArrayView. + z5::types::ShapeType shape(ndim_, 1), offset(ndim_, 0); + shape[iy_] = data_height; offset[iy_] = pixel_row_index; + shape[ix_] = data_width; offset[ix_] = pixel_col_index; + if (iz_ >= 0) { shape[iz_] = data_depth; offset[iz_] = pixel_layer_index; } + if (ic_ >= 0) offset[ic_] = pixel_channel_index; + if (it_ >= 0) offset[it_] = pixel_timeframe_index; auto view = z5::multiarray::makeView(buffer.data(), shape); - z5::types::ShapeType offset = {0, 0, pixel_layer_index, pixel_row_index, pixel_col_index}; - + // Read subarray from the cached z5 dataset z5::multiarray::readSubarray(*ds_, view, offset.begin()); - + // zero-fill the buffer foreseeing its partial filling at incomplete (tail) tiles std::fill(dest.begin(), dest.end(), 0); - - // Copy from buffer to destination tile, handling partial tiles and type conversion - for (size_t k = 0; k < data_height; ++k) { - for (size_t j = 0; j < data_width; ++j) { - dest[k * tile_width_ + j] = static_cast(buffer[k * data_width + j]); + + // Copy from buffer to destination tile, handling partial tiles, multiple Z-planes and + // type conversion (dest is plane-major: plane p, row k at (p*tile_height_+k)*tile_width_). + for (size_t p = 0; p < data_depth; ++p) { + for (size_t k = 0; k < data_height; ++k) { + for (size_t j = 0; j < data_width; ++j) { + dest[(p * tile_height_ + k) * tile_width_ + j] = + static_cast(buffer[(p * data_height + k) * data_width + j]); + } } } } @@ -197,12 +212,19 @@ class RawOmezarrLoader: public RawFormatLoader /// @return Tile depth [[nodiscard]] size_t tileDepth([[maybe_unused]] size_t level) const override { return tile_depth_; } - /// @brief Tiff bits per sample - /// @return Size of a sample in bits - [[nodiscard]] short bitsPerSample() const override { return 1; } - /// @brief Level accessor - /// @return 1 - [[nodiscard]] size_t numberPyramidLevels() const override { return 1; } + /// @brief Bits per sample (resolved from the dataset dtype) + [[nodiscard]] short bitsPerSample() const override { return bits_per_sample_; } + /// @brief Number of resolution (pyramid) levels declared in multiscales + [[nodiscard]] size_t numberPyramidLevels() const override { return n_levels_; } + /// @brief Channel (C) extent resolved from the NGFF axes (1 if no channel axis) + [[nodiscard]] size_t numberChannels() const override { return n_channels_; } + /// @brief Time (T) extent resolved from the NGFF axes (1 if no time axis) + [[nodiscard]] size_t fullTimestamps([[maybe_unused]] size_t level) const override { return n_timeframes_; } + /// @brief Physical voxel spacing from the NGFF coordinateTransformations (1.0 if uncalibrated) + [[nodiscard]] double physicalSizeX() const override { return phys_x_; } + [[nodiscard]] double physicalSizeY() const override { return phys_y_; } + [[nodiscard]] double physicalSizeZ() const override { return phys_z_; } + [[nodiscard]] std::string physicalSizeUnit() const override { return phys_unit_; } private: @@ -214,6 +236,16 @@ class RawOmezarrLoader: public RawFormatLoader tile_height_ = 0, ///< Tile height tile_depth_ = 0; ///< Tile depth + // Storage-dimension index of each axis role (-1 if the axis is absent). + int ix_ = 4, iy_ = 3, iz_ = 2, ic_ = 1, it_ = 0; + size_t ndim_ = 5; ///< Number of on-disk dimensions (2..5) + short bits_per_sample_ = 16; ///< Real bit depth + size_t n_levels_ = 1; ///< Pyramid level count + size_t n_channels_ = 1; ///< Channel (C) extent + size_t n_timeframes_ = 1; ///< Time (T) extent + double phys_x_ = 1.0, phys_y_ = 1.0, phys_z_ = 1.0; ///< Physical voxel spacing + std::string phys_unit_; ///< Physical-size unit (e.g. "micrometer") + short data_format_ = 0; std::unique_ptr zarr_ptr_; std::string ds_name_; diff --git a/src/nyx/raw_tiff.h b/src/nyx/raw_tiff.h index cbef867f..88070c69 100644 --- a/src/nyx/raw_tiff.h +++ b/src/nyx/raw_tiff.h @@ -6,12 +6,12 @@ #include #undef uint64 #undef int64 -#else - #include -#endif -#include -#include -#include +#else + #include +#endif +#include +#include +#include #pragma once #include @@ -21,6 +21,7 @@ #include #include "raw_format.h" +#include "ome/ome_tiff_meta.h" // parse_ome_xml -> OmeAxes (OME-TIFF plane->IFD mapping) class RawTiffTileLoader : public RawFormatLoader { @@ -56,15 +57,33 @@ class RawTiffTileLoader : public RawFormatLoader TIFFGetField(tiff_, TIFFTAG_SAMPLESPERPIXEL, &samplesPerPixel); TIFFGetField(tiff_, TIFFTAG_BITSPERSAMPLE, &(this->bitsPerSample_)); TIFFGetField(tiff_, TIFFTAG_SAMPLEFORMAT, &(this->sampleFormat_)); + // FIX (IO): default absent SAMPLEFORMAT to 1 (unsigned int), like the strip loader, + // so tifffile-written tiled uint OME-TIFFs (which omit the tag) read correctly. + if (sampleFormat_ < 1 || sampleFormat_ > 3) + sampleFormat_ = 1; // Test if the file is greyscale - if (samplesPerPixel != 1) + if (samplesPerPixel != 1) { std::string erm = "RawTiffTileLoader error: file " + filePath + " is not greyscale, SamplesPerPixel = " + std::to_string(samplesPerPixel); std::cerr << erm << "\n"; throw (std::runtime_error(erm)); } + // FIX (IO): parse a tiled OME-TIFF's IFD-0 OME-XML so (z,c,t) map to the right IFD; + // a plain tiled TIFF stays 2D (fullDepth = directory count, usually 1). + fullDepth_ = TIFFNumberOfDirectories(tiff_); + char* desc = nullptr; + if (TIFFGetField(tiff_, TIFFTAG_IMAGEDESCRIPTION, &desc) && desc != nullptr) + { + ome_ = Nyxus::parse_ome_xml(desc); + if (ome_.valid) + { + is_ome_ = true; + fullDepth_ = ome_.sizeZ; // depth is SizeZ, not the total IFD count (= Z*C*T) + } + } + // Prepare the right typed getter function std::string message; switch (sampleFormat_) @@ -167,9 +186,23 @@ class RawTiffTileLoader : public RawFormatLoader void loadTileFromFile ( size_t indexRowGlobalTile, size_t indexColGlobalTile, - size_t indexLayerGlobalTile, + size_t indexLayerGlobalTile, // Z plane + size_t indexChannel, // C plane (OME); 0 for plain 2D TIFF + size_t indexTimeframe, // T plane (OME); 0 for plain 2D TIFF size_t level) override { + // FIX (IO): select the (z,c,t) plane's IFD before reading the tile (see grayscale_tiff.h). + if (is_ome_ && (indexLayerGlobalTile >= ome_.sizeZ + || indexChannel >= ome_.sizeC || indexTimeframe >= ome_.sizeT)) + { + throw std::runtime_error("RawTiffTileLoader: (z,c,t)=(" + std::to_string(indexLayerGlobalTile) + + "," + std::to_string(indexChannel) + "," + std::to_string(indexTimeframe) + ") out of range"); + } + size_t ifd = is_ome_ ? ome_.ifdForPlane(indexLayerGlobalTile, indexChannel, indexTimeframe) + : indexLayerGlobalTile; + if (TIFFSetDirectory(tiff_, (uint16_t)ifd) != 1) + throw std::runtime_error("RawTiffTileLoader: TIFFSetDirectory(ifd=" + std::to_string(ifd) + ") failed"); + // Low level read TIFF bytes auto t_szb = TIFFTileSize(tiff_); tiffTile = _TIFFmalloc(t_szb); @@ -215,10 +248,18 @@ class RawTiffTileLoader : public RawFormatLoader [[nodiscard]] size_t fullHeight([[maybe_unused]] size_t level) const { return fullHeight_; } [[nodiscard]] size_t fullWidth([[maybe_unused]] size_t level) const { return fullWidth_; } + [[nodiscard]] size_t fullDepth([[maybe_unused]] size_t level) const override { return fullDepth_; } [[nodiscard]] size_t tileWidth([[maybe_unused]] size_t level) const { return tileWidth_; } [[nodiscard]] size_t tileHeight([[maybe_unused]] size_t level) const { return tileHeight_; } [[nodiscard]] short bitsPerSample() const { return bitsPerSample_; } [[nodiscard]] size_t numberPyramidLevels() const { return 1; } + // FIX (IO): advertise C/T + physical spacing from the parsed OME-XML (1 / plain otherwise) + [[nodiscard]] size_t numberChannels() const override { return is_ome_ ? ome_.sizeC : 1; } + [[nodiscard]] size_t fullTimestamps([[maybe_unused]] size_t level) const override { return is_ome_ ? ome_.sizeT : 1; } + [[nodiscard]] double physicalSizeX() const override { return is_ome_ ? ome_.physX : 1.0; } + [[nodiscard]] double physicalSizeY() const override { return is_ome_ ? ome_.physY : 1.0; } + [[nodiscard]] double physicalSizeZ() const override { return is_ome_ ? ome_.physZ : 1.0; } + [[nodiscard]] std::string physicalSizeUnit() const override { return is_ome_ ? ome_.unitXY : std::string(); } private: @@ -282,18 +323,22 @@ class RawTiffTileLoader : public RawFormatLoader size_t fullHeight_ = 0, ///< Full height in pixel fullWidth_ = 0, ///< Full width in pixel + fullDepth_ = 1, ///< Full depth (Z); >1 for multi-plane OME-TIFF tileHeight_ = 0, ///< Tile height tileWidth_ = 0; ///< Tile width + bool is_ome_ = false; ///< true when IFD-0 carries an OME-XML block + Nyxus::OmeAxes ome_; ///< parsed OME dimensions (drives the (z,c,t)->IFD map) + short sampleFormat_ = 0, ///< Sample format as defined by libtiff bitsPerSample_ = 0; ///< Bit Per Sample as defined by libtiff - double minval, maxval; + double minval, maxval; // low level buffer tdata_t tiffTile = nullptr; - size_t t_szb = 0; + size_t t_szb = 0; }; class RawTiffStripLoader : public RawFormatLoader @@ -320,6 +365,23 @@ class RawTiffStripLoader : public RawFormatLoader fullDepth_ = TIFFNumberOfDirectories(tiff_); + // OME-TIFF: the IFDs are a (z,c,t) rasterization, not a plain Z-stack. + // Parse the OME-XML (IFD-0 ImageDescription) so (z,c,t) map to the right + // IFD and fullDepth reflects SizeZ, not the total page count. + { + char* desc = nullptr; + if (TIFFGetField(tiff_, TIFFTAG_IMAGEDESCRIPTION, &desc) == 1 && desc + && std::string(desc).find("(buf); + auto* fub = static_cast(buf); for (size_t r = 0; r < tileHeight_; r++) { size_t offs = r * scanline_szb; @@ -475,15 +552,33 @@ class RawTiffStripLoader : public RawFormatLoader [[nodiscard]] size_t fullHeight([[maybe_unused]] size_t level) const override { return fullHeight_; } [[nodiscard]] size_t fullWidth([[maybe_unused]] size_t level) const override { return fullWidth_; } [[nodiscard]] size_t fullDepth([[maybe_unused]] size_t level) const override { return fullDepth_; } + // FIX: advertise C/T extents from the parsed OME-XML so the volumetric pipeline + // iterates channels/timeframes; non-OME TIFF keeps the single-plane default of 1. + [[nodiscard]] size_t numberChannels() const override { return is_ome_ ? ome_.sizeC : 1; } + [[nodiscard]] size_t fullTimestamps([[maybe_unused]] size_t level) const override { return is_ome_ ? ome_.sizeT : 1; } + // FIX (IO): physical voxel spacing from OME-XML PhysicalSize* (1.0 for plain TIFF) + [[nodiscard]] double physicalSizeX() const override { return is_ome_ ? ome_.physX : 1.0; } + [[nodiscard]] double physicalSizeY() const override { return is_ome_ ? ome_.physY : 1.0; } + [[nodiscard]] double physicalSizeZ() const override { return is_ome_ ? ome_.physZ : 1.0; } + [[nodiscard]] std::string physicalSizeUnit() const override { return is_ome_ ? ome_.unitXY : std::string(); } [[nodiscard]] size_t tileWidth([[maybe_unused]] size_t level) const override { return tileWidth_; } [[nodiscard]] size_t tileHeight([[maybe_unused]] size_t level) const override { return tileHeight_; } [[nodiscard]] size_t tileDepth([[maybe_unused]] size_t level) const override { return tileDepth_; } [[nodiscard]] short bitsPerSample() const override { return bitsPerSample_; } [[nodiscard]] size_t numberPyramidLevels() const override { return 1; } + // FIX: honor the RawFormatLoader contract that free_tile() follows EACH + // loadTileFromFile(). This buffer used to be allocated once in the ctor and never + // re-allocated, so a second load after a free wrote into freed memory (hit when + // looping Z to assemble a volume, and latent for multi-tile 2D scans which free per + // tile). Freeing is now idempotent and loadTileFromFile lazily re-allocates. void free_tile() override { - _TIFFfree(buf); + if (buf) + { + _TIFFfree(buf); + buf = nullptr; + } } uint32_t get_uint32_pixel(size_t idx) const @@ -561,6 +656,9 @@ class RawTiffStripLoader : public RawFormatLoader sampleFormat_ = 0, ///< Sample format as defined by libtiff bitsPerSample_ = 0; ///< Bit Per Sample as defined by libtiff + bool is_ome_ = false; ///< true when IFD-0 carries an OME-XML block + Nyxus::OmeAxes ome_; ///< parsed OME dimensions (drives the plane->IFD map) + double minval, maxval; // low level buffer diff --git a/src/nyx/reduce_trivial_rois.cpp b/src/nyx/reduce_trivial_rois.cpp index 03245098..a9066381 100644 --- a/src/nyx/reduce_trivial_rois.cpp +++ b/src/nyx/reduce_trivial_rois.cpp @@ -660,7 +660,7 @@ namespace Nyxus if (D3_VoxelIntensityFeatures::required(env.theFeatureSet)) { Fsettings& s = env.fsett_D3_VoxelIntensity; - D3_VoxelIntensityFeatures::extract (r, s); + D3_VoxelIntensityFeatures::extract (r, s, env.dataset); // FIX: pass the Dataset (see 3d_intensity.h) } //==== shape if (D3_SurfaceFeature::required(env.theFeatureSet)) diff --git a/src/nyx/roi_cache.cpp b/src/nyx/roi_cache.cpp index ccf98132..0e4df92e 100644 --- a/src/nyx/roi_cache.cpp +++ b/src/nyx/roi_cache.cpp @@ -17,21 +17,28 @@ bool LR::nontrivial_roi (size_t n_rois, size_t limit) size_t LR::get_ram_footprint_estimate (size_t n_slide_rois) const { + // n_slide_rois==0 means zero OTHER rois to ever hold as neighbors, so that term is 0 bytes -- + // not (size_t)(0-1)*sizeof(int), which underflows to SIZE_MAX and then overflows the multiply + // to another huge wrapped value. A caller passing an in-progress batch count (which starts at + // 0) hits this on every batch's first item. + size_t neighborsBytes = (n_slide_rois > 0) ? (n_slide_rois - 1) * sizeof(int) : 0; size_t sz = int(Nyxus::FeatureIMQ::_COUNT_) * 10 * sizeof(double) + // feature values (approximately 10 each) aabb.get_width() * aabb.get_height() * sizeof(Pixel2) + // image matrix aux_area * sizeof(Pixel2) + // raw pixels - (n_slide_rois - 1) * sizeof(int); // neighbors + neighborsBytes; return sz; } size_t LR::get_ram_footprint_estimate_3D (size_t n_volume_rois) const { + // see get_ram_footprint_estimate() above for why n==0 is guarded rather than left to underflow + size_t neighborsBytes = (n_volume_rois > 0) ? (n_volume_rois - 1) * sizeof(int) : 0; size_t sz = int(Nyxus::FeatureIMQ::_COUNT_) * 10 * sizeof(double) + // feature values (approximately 10 each) aabb.get_width() * aabb.get_height() * aabb.get_z_depth() * sizeof(Pixel2) + // image matrix aux_area * sizeof(Pixel2) + // raw pixels - (n_volume_rois - 1) * sizeof(int); // neighbors + neighborsBytes; return sz; } diff --git a/src/nyx/roi_cache.h b/src/nyx/roi_cache.h index e01b6ac4..6116fada 100644 --- a/src/nyx/roi_cache.h +++ b/src/nyx/roi_cache.h @@ -7,6 +7,7 @@ #include "features/aabb.h" #include "features/image_matrix.h" #include "features/image_matrix_nontriv.h" +#include "features/voxel_cloud_nontriv.h" #include "features/image_cube.h" #include "features/pixel.h" #include "featureset.h" @@ -50,6 +51,9 @@ class LR: public BasicLR std::unordered_map> zplanes; OutOfRamPixelCloud raw_pixels_NT; + // Disk-backed 3D voxel source for oversized volumetric ROIs (populated by the 3D + // out-of-core scan). Keeps z, unlike raw_pixels_NT which is the 2D (z-less) cloud. + OutOfRamVoxelCloud raw_voxels_NT; unsigned int aux_area = 0; PixIntens aux_min, aux_max; diff --git a/src/nyx/slideprops.cpp b/src/nyx/slideprops.cpp index f96de6bb..c795362e 100644 --- a/src/nyx/slideprops.cpp +++ b/src/nyx/slideprops.cpp @@ -95,6 +95,11 @@ namespace Nyxus // time series p.inten_time = ilo.get_inten_time(); p.mask_time = ilo.get_mask_time(); + p.inten_channels = ilo.get_inten_channels(); // FIX (IO): number of channels (>=1) + p.phys_x = ilo.get_physical_size_x(); // FIX (IO): physical voxel spacing (1.0 if uncalibrated) + p.phys_y = ilo.get_physical_size_y(); + p.phys_z = ilo.get_physical_size_z(); + p.phys_unit = ilo.get_physical_size_unit(); // scan intensity slide's data @@ -140,15 +145,6 @@ namespace Nyxus // Iterate pixels for (size_t i = 0; i < tileSize; i++) { - // Mask - uint32_t msk = 1; // wholeslide by default - if (!wholeslide) - msk = ilo.get_cur_tile_seg_pixel(i); - - // Skip non-mask pixels - if (!msk) - continue; - int y = row * th + i / tw, x = col * tw + i % tw; @@ -156,11 +152,24 @@ namespace Nyxus if (x >= fullwidth || y >= fullheight) continue; - // dynamic range within- and off-ROI + // dynamic range within- and off-ROI: this baseline is the "pre-ROI" + // (whole-slide) range COVERED_IMAGE_INTENSITY_RANGE divides by, so it must + // include off-mask voxels too -- gating it on msk would degenerate it to the + // ROI's own range for any segmented (non-whole-slide) image. double dxequiv_I = ilo.get_cur_tile_dpequiv_pixel(i); slide_I_max = (std::max)(slide_I_max, dxequiv_I); slide_I_min = (std::min)(slide_I_min, dxequiv_I); + // Mask + uint32_t msk = 1; // wholeslide by default + if (!wholeslide) + msk = ilo.get_cur_tile_seg_pixel(i); + + // Skip non-mask pixels for ROI geometry (their intensity is already + // counted in the slide-wide range above) + if (!msk) + continue; + // Update pixel's ROI metrics // - the following block mocks feed_pixel_2_metrics (x, y, dataI[i], msk, tidx) if (U.find(msk) == U.end()) @@ -270,95 +279,97 @@ namespace Nyxus // time series p.inten_time = ilo.get_inten_time(); p.mask_time = ilo.get_mask_time(); + p.inten_channels = ilo.get_inten_channels(); // FIX (IO): number of channels (>=1) + p.phys_x = ilo.get_physical_size_x(); // FIX (IO): physical voxel spacing (1.0 if uncalibrated) + p.phys_y = ilo.get_physical_size_y(); + p.phys_z = ilo.get_physical_size_z(); + p.phys_unit = ilo.get_physical_size_unit(); // scan intensity slide's data - bool wholeslide = p.fname_seg.empty(); - double slide_I_max = (std::numeric_limits::lowest)(), slide_I_min = (std::numeric_limits::max)(); std::unordered_set U; // unique ROI mask labels std::unordered_map R; // ROI data - // Read the volume. The image loader is in the open state by previously called processDataset_XX_YY () + // The image loader is in the open state by previously called processDataset_XX_YY (). + // FIX: no whole-volume staging buffer and no flat voxel index any more -- the scan + // streams tiles and receives (x,y,z) directly, so sliceSize/nVox/wholeslide are gone. size_t fullW = ilo.get_full_width(), fullH = ilo.get_full_height(), - fullD = ilo.get_full_depth(), - sliceSize = fullW * fullH, - nVox = sliceSize * fullD; - - // in the 3D case tiling is a formality, so fetch the only tile in the file - if (!ilo.load_tile(0, 0)) + fullD = ilo.get_full_depth(); + + // FIX: scan EVERY (channel, timeframe) volume, not just (0,0). + // (a) for_each_voxel() streams the whole X*Y*Z volume, walking each plane's tile grid. + // The old single load_tile(0,0) filled only ONE tile of ONE Z-plane for per-plane + // loaders (OME-TIFF/OME-Zarr), so the W*H*D voxel scan read past the tile buffer + // -- garbage that corrupted the slide min/max. + // (b) the pipeline featurizes every (c,t) plane, so the slide intensity range must + // cover all of them. Scanning only (c0,t0) yields a range that is too small for + // c>0 / t>0, and intensity-indexed buffers sized from it overflow. + // ROI geometry is taken from the first pass only (the mask is channel-agnostic). + const size_t n_chan = (std::max)((size_t)1, p.inten_channels), + n_time = (std::max)((size_t)1, p.inten_time); + bool first_pass = true; + + for (size_t scan_c = 0; scan_c < n_chan; scan_c++) + for (size_t scan_t = 0; scan_t < n_time; scan_t++) { -#ifdef WITH_PYTHON_H - throw "Error fetching tile"; -#endif - std::cerr << "Error fetching tile\n"; - return false; - } - - // iterate abstract tiles (in a tiled slide /e.g. tiled tiff/ they correspond to physical tiles, in a nontiled slide /e.g. scanline tiff or strip tiff/ they correspond to ) - int cnt = 1; - - // iterate voxels - for (size_t i = 0; i < nVox; i++) + // Streams tile by tile -- (a) above needs the whole volume scanned, but nothing here + // needs it resident, and staging W*H*D doubles once per (c,t) is 4x the raw uint16 + // volume in RAM for no benefit. for_each_voxel calls back for every voxel. + bool ok = ilo.for_each_voxel (scan_c, scan_t, + [&](size_t x, size_t y, size_t z, double dxequiv_I, uint32_t msk) { - // Mask - uint32_t msk = 1; // wholeslide by default - if (!wholeslide) - msk = ilo.get_cur_tile_seg_pixel(i); - - // Skip non-mask pixels - if (!msk) - continue; - - int z = i / sliceSize, - y = (i - z * sliceSize) / fullW, - x = (i - z * sliceSize) % fullW; - - // Skip tile buffer pixels beyond the image's bounds - if (x >= fullW || y >= fullH || z >= fullD) - continue; - - // dynamic range within- and off-ROI - double dxequiv_I = ilo.get_cur_tile_dpequiv_pixel(i); + // dynamic range within- and off-ROI: this baseline is the "pre-ROI" (whole-slide) + // range COVERED_IMAGE_INTENSITY_RANGE divides by, so it must include off-mask voxels + // too -- gating it on msk would degenerate it to the ROI's own range for any + // segmented (non-whole-slide) image. slide_I_max = (std::max)(slide_I_max, dxequiv_I); slide_I_min = (std::min)(slide_I_min, dxequiv_I); - // Update pixel's ROI metrics - // - the following block mocks feed_pixel_2_metrics (x, y, dataI[i], msk, tidx) - if (U.find(msk) == U.end()) + if (!msk) + return; // off-ROI voxel: only contributes to the slide-wide range above + + // Update pixel's ROI metrics. FIX: geometry (labels, area, AABB) is derived from + // the FIRST (c,t) pass only -- the mask is the same for every channel/timeframe, + // so re-accumulating it per pass would multiply aux_area by n_chan*n_time. + if (first_pass) { - // Remember this label - U.insert(msk); + // - the following block mocks feed_pixel_2_metrics (x, y, dataI[i], msk, tidx) + if (U.find(msk) == U.end()) + { + // Remember this label + U.insert(msk); - // Initialize the ROI label record - LR r(msk); + // Initialize the ROI label record + LR r(msk); - // - mocking init_label_record_3 (newData, theSegFname, theIntFname, x, y, label, intensity, tile_index) - // Initialize basic counters - r.aux_area = 1; - r.aux_min = r.aux_max = 0; //we don't have uint-cast intensities at this moment - r.init_aabb_3D(x, y, z); + // - mocking init_label_record_3 (newData, theSegFname, theIntFname, x, y, label, intensity, tile_index) + // Initialize basic counters + r.aux_area = 1; + r.aux_min = r.aux_max = 0; //we don't have uint-cast intensities at this moment + r.init_aabb_3D((int)x, (int)y, (int)z); - // - not storing file names (r.segFname = segFile, r.intFname = intFile) but will do so in the future + // - not storing file names (r.segFname = segFile, r.intFname = intFile) but will do so in the future - // Attach - R[msk] = r; - } - else - { - // Update basic ROI info (info that doesn't require costly calculations) - LR& r = R[msk]; + // Attach + R[msk] = r; + } + else + { + // Update basic ROI info (info that doesn't require costly calculations) + LR& r = R[msk]; - // - mocking update_label_record_2 (r, x, y, label, intensity, tile_index) + // - mocking update_label_record_2 (r, x, y, label, intensity, tile_index) - // Per-ROI - r.aux_area++; + // Per-ROI + r.aux_area++; - // save - r.update_aabb_3D(x, y, z); + // save + r.update_aabb_3D((int)x, (int)y, (int)z); + } } #ifdef WITH_PYTHON_H @@ -367,7 +378,19 @@ namespace Nyxus throw pybind11::error_already_set(); #endif - } //- all voxels + }); //- all voxels + + if (!ok) + { +#ifdef WITH_PYTHON_H + throw "Error fetching volume"; +#endif + std::cerr << "Error fetching volume\n"; + return false; + } + + first_pass = false; + } //- all (channel, timeframe) volumes //****** fix ROIs' AABBs with respect to anisotropy diff --git a/src/nyx/slideprops.h b/src/nyx/slideprops.h index 5bbfcf0d..5a64c9ee 100644 --- a/src/nyx/slideprops.h +++ b/src/nyx/slideprops.h @@ -28,6 +28,9 @@ class SlideProps max_roi_h = 0; max_roi_d = 0; inten_time = mask_time = 0; + inten_channels = 1; // FIX (IO): >=1 so the channel loop runs at least once; set from the loader in scan_slide_props + phys_x = phys_y = phys_z = 1.0; // FIX (IO): physical voxel spacing (1.0 == uncalibrated); set in scan_slide_props + phys_unit = ""; } // pre-ROI intensity range in DP @@ -86,6 +89,14 @@ class SlideProps // time series size_t inten_time, mask_time; // number of time frames + + // channels (FIX (IO): number of intensity channels; drives the per-channel output rows) + size_t inten_channels; + + // physical voxel spacing (FIX (IO): OME PhysicalSize*; 1.0 == uncalibrated). Used only + // when --use-physical-spacing is set; also emitted as phys_x/y/z + phys_unit output columns. + double phys_x, phys_y, phys_z; + std::string phys_unit; }; namespace Nyxus diff --git a/src/nyx/workflow_2d_segmented.cpp b/src/nyx/workflow_2d_segmented.cpp index ebcc051c..c416957b 100644 --- a/src/nyx/workflow_2d_segmented.cpp +++ b/src/nyx/workflow_2d_segmented.cpp @@ -166,11 +166,12 @@ namespace Nyxus const SaveOption saveOption, const std::string& outputPath) { + env.reset_csv_output_state(); // FIX: this run's CSV files start fresh (see Environment::csv_paths_written) #ifdef CHECKTIMING if (Stopwatch::inclusive()) Stopwatch::reset(); -#endif +#endif //********************** prescan *********************** @@ -322,7 +323,7 @@ namespace Nyxus if (write_apache) { - auto [status, msg] = env.arrow_stream.write_arrow_file (Nyxus::get_feature_values(env, env.theFeatureSet, env.uniqueLabels, env.roiData, env.dataset)); + auto [status, msg] = env.arrow_stream.write_arrow_file (Nyxus::get_feature_values(env, env.theFeatureSet, env.uniqueLabels, env.roiData, env.dataset, DEFAULT_T_INDEX, DEFAULT_C_INDEX)); if (!status) { std::cerr << "Error writing Arrow file: " << msg.value() << std::endl; @@ -332,7 +333,7 @@ namespace Nyxus else if (saveOption == SaveOption::saveCSV) { - ok = save_features_2_csv (env, ifp, lfp, outputPath, 0/*pass 0 as t_index is not used in 2D scenario*/ , env.resultOptions.need_aggregation()); + ok = save_features_2_csv (env, ifp, lfp, outputPath, 0/*t_index unused in 2D*/, DEFAULT_C_INDEX/*channel unused in 2D*/, env.resultOptions.need_aggregation()); if (ok == false) { @@ -342,7 +343,7 @@ namespace Nyxus } else { - ok = save_features_2_buffer (env.theResultsCache, env, DEFAULT_T_INDEX); + ok = save_features_2_buffer (env.theResultsCache, env, DEFAULT_T_INDEX, DEFAULT_C_INDEX); if (ok == false) { diff --git a/src/nyx/workflow_2d_whole.cpp b/src/nyx/workflow_2d_whole.cpp index 9911f404..54fa617f 100644 --- a/src/nyx/workflow_2d_whole.cpp +++ b/src/nyx/workflow_2d_whole.cpp @@ -177,7 +177,7 @@ namespace Nyxus // thread-safely save results of this single slide if (write_apache) { - auto [status, msg] = save_features_2_apache_wholeslide (env, vroi, p.fname_int); + auto [status, msg] = save_features_2_apache_wholeslide (env, vroi, p.fname_int, 0/*t_index*/, 0/*c_index*/); if (! status) { std::cerr << "Error writing Arrow file: " << msg.value() << std::endl; @@ -187,7 +187,7 @@ namespace Nyxus else if (saveOption == SaveOption::saveCSV) { - if (save_features_2_csv_wholeslide(env, vroi, p.fname_int, "", outputPath, 0 /*pass 0 as t_index is not used in 2D scenario*/) == false) + if (save_features_2_csv_wholeslide(env, vroi, p.fname_int, "", outputPath, 0 /*t_index unused in 2D*/, 0 /*c_index unused in 2D*/) == false) { std::cout << "error saving results to CSV file, details: " << __FILE__ << ":" << __LINE__ << std::endl; rv = 2; @@ -196,7 +196,7 @@ namespace Nyxus else { // pulls feature values from 'vroi' and appends them to global object 'theResultsCache' exposed to Python API - if (save_features_2_buffer_wholeslide (env.theResultsCache, env, vroi, p.fname_int, "") == false) + if (save_features_2_buffer_wholeslide (env.theResultsCache, env, vroi, p.fname_int, "", 0/*t_index*/, 0/*c_index*/) == false) { std::cerr << "Error saving features to the results buffer" << std::endl; rv = 2; @@ -219,7 +219,9 @@ namespace Nyxus const SaveOption saveOption, const std::string& outputPath) { - // create a vector of blank mask file names. Blank mask counterparts + env.reset_csv_output_state(); // FIX: this run's CSV files start fresh (see Environment::csv_paths_written) + + // create a vector of blank mask file names. Blank mask counterparts // of intensity files will serve as the condition of the whole-slide scenario in the prescan phase std::vector labelFiles (intensFiles.size()); diff --git a/src/nyx/workflow_3d_segmented.cpp b/src/nyx/workflow_3d_segmented.cpp index aa3630eb..f749f53e 100644 --- a/src/nyx/workflow_3d_segmented.cpp +++ b/src/nyx/workflow_3d_segmented.cpp @@ -32,7 +32,7 @@ namespace Nyxus { - bool processIntSegImagePair_3D (Environment & env, const std::string& intens_fpath, const std::string& label_fpath, size_t filepair_index, size_t t_index, const std::vector& z_indices) + bool processIntSegImagePair_3D (Environment & env, const std::string& intens_fpath, const std::string& label_fpath, size_t filepair_index, size_t t_index, size_t channel, const std::vector& z_indices) { std::vector trivRois, nontrivRois; @@ -51,7 +51,7 @@ namespace Nyxus if (z_indices.size()) okGather = gatherRoisMetrics_25D (env, filepair_index, intens_fpath, label_fpath, z_indices); else - okGather = gatherRoisMetrics_3D (env, filepair_index, intens_fpath, label_fpath, t_index); + okGather = gatherRoisMetrics_3D (env, filepair_index, intens_fpath, label_fpath, t_index, channel); if (!okGather) { std::string msg = "Error gathering ROI metrics from " + intens_fpath + " / " + label_fpath + "\n"; @@ -124,14 +124,16 @@ namespace Nyxus if (z_indices.size()) processTrivialRois_25D (env, trivRois, intens_fpath, label_fpath, env.get_ram_limit(), z_indices); else - processTrivialRois_3D (env, filepair_index, t_index, trivRois, intens_fpath, label_fpath, env.get_ram_limit()); + processTrivialRois_3D (env, filepair_index, t_index, channel, trivRois, intens_fpath, label_fpath, env.get_ram_limit()); } // Phase 3: process nontrivial (oversized) ROIs, if any if (nontrivRois.size()) { VERBOSLVL2 (env.get_verbosity_level(), std::cout << "Processing oversized ROIs\n";) - processNontrivialRois (env, nontrivRois, intens_fpath, label_fpath); + // Volumetric out-of-core path: streams the voxel cloud to disk keeping z (the shared 2D + // processNontrivialRois scans a single plane and drops z). t_index/channel select the plane. + processNontrivialRois_3D (env, nontrivRois, intens_fpath, label_fpath, channel, t_index); } return true; @@ -145,6 +147,8 @@ namespace Nyxus const SaveOption saveOption, const std::string& outputPath) { + env.reset_csv_output_state(); // FIX: this run's CSV files start fresh (see Environment::csv_paths_written) + //********************** prescan *********************** // slide properties @@ -202,6 +206,10 @@ namespace Nyxus // iterate intensity-mask pairs for (size_t i=0; i1 channel, so each C-plane is featurized and tagged with c_index) + for (size_t c=0; c < env.dataset.dataset_props[i].inten_channels; c++) + { // iterate time frames for (size_t t=0; t < env.dataset.dataset_props[i].inten_time; t++) { @@ -219,19 +227,19 @@ namespace Nyxus // Display (1) dataset progress info and (2) file pair info int digits = 2, k = (int)std::pow(10.f, digits); float perCent = float(i * 100 * k / nf) / float(k); - VERBOSLVL1(env.get_verbosity_level(), std::cout << "[ " << std::setw(digits + 2) << perCent << "% ]\t" << " INT: " << ifile.fname << " SEG: " << mfile.fname << " T:" << t << "\n") - - bool ok = processIntSegImagePair_3D (env, ifile.fdir+ifile.fname, mfile.fdir+mfile.fname, i, t, intensFiles[i].z_indices); + VERBOSLVL1(env.get_verbosity_level(), std::cout << "[ " << std::setw(digits + 2) << perCent << "% ]\t" << " INT: " << ifile.fname << " SEG: " << mfile.fname << " C:" << c << " T:" << t << "\n") + + bool ok = processIntSegImagePair_3D (env, ifile.fdir+ifile.fname, mfile.fdir+mfile.fname, i, t, c, intensFiles[i].z_indices); if (ok == false) { std::cerr << "processIntSegImagePair() returned an error code while processing file pair " << ifile.fname << " - " << mfile.fname << '\n'; return 1; } - // Output features + // Output features (tag each row with its timeframe t and channel c) if (writeApache) { - auto [status, msg] = env.arrow_stream.write_arrow_file(Nyxus::get_feature_values(env, env.theFeatureSet, env.uniqueLabels, env.roiData, env.dataset)); + auto [status, msg] = env.arrow_stream.write_arrow_file(Nyxus::get_feature_values(env, env.theFeatureSet, env.uniqueLabels, env.roiData, env.dataset, t, c)); if (!status) { std::cout << "Error writing Arrow file: " << msg.value() << std::endl; @@ -241,7 +249,7 @@ namespace Nyxus else if (saveOption == SaveOption::saveCSV) { - if (!save_features_2_csv(env, ifile.fname, mfile.fname, outputPath, t, env.resultOptions.need_aggregation())) + if (!save_features_2_csv(env, ifile.fname, mfile.fname, outputPath, t, c, env.resultOptions.need_aggregation())) { std::cout << "error saving results to CSV file, details: " << __FILE__ << ":" << __LINE__ << std::endl; return 2; @@ -249,7 +257,7 @@ namespace Nyxus } else { - if (!save_features_2_buffer(env.theResultsCache, env, t)) + if (!save_features_2_buffer(env.theResultsCache, env, t, c)) { std::cout << "error saving results to a buffer, details: " << __FILE__ << ":" << __LINE__ << std::endl; return 2; @@ -269,6 +277,7 @@ namespace Nyxus } #endif } //- time frames + } //- channels } //- inten-mask pairs if (writeApache) diff --git a/src/nyx/workflow_3d_whole.cpp b/src/nyx/workflow_3d_whole.cpp index a452600b..b97b089f 100644 --- a/src/nyx/workflow_3d_whole.cpp +++ b/src/nyx/workflow_3d_whole.cpp @@ -32,12 +32,16 @@ namespace Nyxus { - bool featurize_triv_wholevolume (Environment & env, size_t sidx, ImageLoader& imlo, size_t memory_limit, LR& vroi) + bool featurize_triv_wholevolume (Environment & env, size_t sidx, ImageLoader& imlo, size_t memory_limit, LR& vroi, size_t channel, size_t timeframe) { const std::string& ifpath = env.dataset.dataset_props[sidx].fname_int; // can we process this slide ? - size_t footp = vroi.get_ram_footprint_estimate (1); // 1 since single-ROI + // FIX: use the 3D estimator (W*H*D). The 2D one counts only W*H for the image matrix, + // so it under-counts a whole VOLUME's cube by a factor of the depth -> the oversized + // check let too-large volumes through and they OOM'd. The segmented path already uses + // get_ram_footprint_estimate_3D; match it. (Found by running under a hard memory cap.) + size_t footp = vroi.get_ram_footprint_estimate_3D (1); // 1 since single-ROI if (footp > memory_limit) { std::string erm = "Error: cannot process slide " + ifpath + " , reason: its memory footprint " + virguler_ulong(footp) + " exceeds available memory " + virguler_ulong(memory_limit); @@ -49,21 +53,38 @@ namespace Nyxus } // read the slide into a pixel cloud - if (env.anisoOptions.customized() == false) + // FIX (IO): --aniso* (explicit) or opt-in OME physical spacing selects the anisotropic path + double ax, ay, az; + if (! resolve_slide_anisotropy (env, sidx, ax, ay, az)) { VERBOSLVL2(env.get_verbosity_level(), std::cout << "\nscan_trivial_wholeslide()\n"); - scan_trivial_wholevolume (vroi, ifpath, imlo); + scan_trivial_wholevolume (vroi, ifpath, imlo, channel, timeframe); } else { VERBOSLVL2(env.get_verbosity_level(), std::cout << "\nscan_trivial_wholeslide_ANISO()\n"); scan_trivial_wholevolume_anisotropic ( - vroi, - ifpath, - imlo, - env.anisoOptions.get_aniso_x(), - env.anisoOptions.get_aniso_y(), - env.anisoOptions.get_aniso_z()); + vroi, + ifpath, + imlo, + ax, + ay, + az, + channel, + timeframe); + + // vroi.aabb and aux_area were preset from the PHYSICAL slide dimensions + // (init_from_whd / p.max_roi_area, above in featurize_wholevolume) before the + // resampled voxel cloud existed; the anisotropic scan just populated + // raw_pixels_3D with the VIRTUAL (resampled) cloud, which is both a different + // extent and a different voxel COUNT. Recompute both from the actual cloud -- + // mirrors the segmented anisotropic path's fix (processTrivialRois_3D). Without + // the aabb fix, aux_image_cube below is allocated too small and + // calculate_from_pixelcloud writes out of bounds; without the aux_area fix, + // every feature that divides by voxel count (e.g. MEAN) is off by the + // resampling factor. + vroi.aabb.update_from_voxelcloud (vroi.raw_pixels_3D); + vroi.aux_area = (unsigned int) vroi.raw_pixels_3D.size(); } // allocate memory for feature helpers (image matrix, etc) @@ -87,7 +108,7 @@ namespace Nyxus return true; } - bool featurize_wholevolume (Environment & env, size_t sidx, ImageLoader& imlo, LR& vroi) + bool featurize_wholevolume (Environment & env, size_t sidx, ImageLoader& imlo, LR& vroi, size_t channel, size_t timeframe) { //***** phase 1: copy ROI metrics from the slide properties, thanks to the WSI scenario const SlideProps& p = env.dataset.dataset_props[sidx]; @@ -113,31 +134,55 @@ namespace Nyxus vroi.initialize_fvals(); // assess ROI's memory footprint and check if we can featurize it as phase 2 (trivially) ? - size_t roiFootprint = vroi.get_ram_footprint_estimate (1), // 1 since single-ROI + // FIX: 3D estimator (W*H*D) -- the 2D one ignores depth and under-counts the volume cube, + // so oversized volumes slipped through and OOM'd. Matches the segmented path. + size_t roiFootprint = vroi.get_ram_footprint_estimate_3D (1), // 1 since single-ROI ramLim = env.get_ram_limit(); if (roiFootprint >= ramLim) { - VERBOSLVL2(env.get_verbosity_level(), - std::cout << "oversized slide " + VERBOSLVL1(env.get_verbosity_level(), + std::cout << "oversized whole volume " << " (S=" << vroi.aux_area << " W=" << vroi.aabb.get_width() << " H=" << vroi.aabb.get_height() << " D=" << vroi.aabb.get_z_depth() << " px footprint=" << Nyxus::virguler_ulong(roiFootprint) << " b" - << ") while RAM limit is " << Nyxus::virguler_ulong(ramLim) << "\n" + << ") while RAM limit is " << Nyxus::virguler_ulong(ramLim) + << " -- streaming out-of-core\n" ); - std::cerr << p.fname_int << ": slide is non-trivial \n"; - return false; + // Out-of-core whole volume: stream every voxel (no mask -- workflow_3d_whole.cpp opens + // the loader with an empty label path) plane-by-plane instead of holding the whole + // cube, mirroring the segmented ROI path (processNontrivialRois_3D). Only fails when + // the input format can't deliver the volume plane-by-plane (e.g. NIfTI). + if (! populate_3d_voxel_cloud (imlo, vroi, channel, timeframe, /*wholevolume=*/ true)) + { + std::string erm = "Error: cannot featurize whole volume " + p.fname_int + + " out-of-core: this input format does not deliver the volume plane-by-plane. " + + "Segment into smaller ROIs, raise --ramLimit, or add RAM."; +#ifdef WITH_PYTHON_H + throw std::runtime_error(erm); +#endif + std::cerr << erm << "\n"; + return false; + } + + run_3d_ooc_features (env, vroi, imlo); + vroi.raw_voxels_NT.clear(); + return true; } //***** phase 2: extract features - featurize_triv_wholevolume (env, sidx, imlo, env.get_ram_limit(), vroi); // segmented counterpart: phase2.cpp / processTrivialRois () + featurize_triv_wholevolume (env, sidx, imlo, env.get_ram_limit(), vroi, channel, timeframe); // segmented counterpart: phase2.cpp / processTrivialRois () return true; } - void featurize_3d_wv_thread( + // FIX: return the per-thread status by VALUE (through the future) rather than writing a + // referenced int. A std::async worker's write through std::ref(int) is not reliably visible + // to the caller after future::get() here, so an oversized slide's rv=1 was silently lost and + // the run reported success. Returning it makes future::get() carry the status deterministically. + int featurize_3d_wv_thread( Environment & env, const std::vector& intensFiles, const std::vector& labelFiles, @@ -145,9 +190,9 @@ namespace Nyxus size_t nf, const std::string& outputPath, bool write_apache, - Nyxus::SaveOption saveOption, - int& rv) + Nyxus::SaveOption saveOption) { + int rv = 0; SlideProps& p = env.dataset.dataset_props[slide_idx]; // scan one slide @@ -158,44 +203,57 @@ namespace Nyxus rv = 1; } - LR vroi(1); // virtual ROI representing the whole slide ROI-labelled as '1' + // FIX (IO): featurize every (channel, timeframe) plane and emit one whole-slide row + // per plane, tagged with c_index/t_index — mirroring the segmented path. A + // single-channel, single-timeframe slide reports 1/1, so its output is unchanged. + for (size_t c = 0; c < p.inten_channels; c++) + for (size_t t = 0; t < p.inten_time; t++) + { + LR vroi(1); // virtual ROI representing the whole slide ROI-labelled as '1' - if (featurize_wholevolume (env, slide_idx, imlo, vroi) == false) // non-wsi counterpart: processIntSegImagePair() - { - std::cerr << "Error featurizing slide " << p.fname_int << " @ " << __FILE__ << ":" << __LINE__ << "\n"; - rv = 1; - } - - // thread-safely save results of this single slide - if (write_apache) - { - auto [status, msg] = save_features_2_apache_wholeslide (env, vroi, p.fname_int); - if (!status) + if (featurize_wholevolume (env, slide_idx, imlo, vroi, c, t) == false) // non-wsi counterpart: processIntSegImagePair() { - std::cerr << "Error writing Arrow file: " << msg.value() << std::endl; - rv = 2; + // An oversized whole volume now streams out-of-core (see featurize_wholevolume's + // oversized branch), so featurize_wholevolume returns false only when that streaming + // itself failed -- either the input format can't deliver the volume plane-by-plane + // (e.g. NIfTI), or a requested feature isn't yet OOC-supported. That specific reason + // was already reported (thrown under Python, printed to stderr under the CLI) from + // inside featurize_wholevolume/run_3d_ooc_features. Previously we ALSO fell through + // to save the zero-initialized buffer here -- emitting a row of all-0 features, i.e. + // silently-wrong data. Skip the save instead: fail loudly, write no row. + rv = 1; + continue; // do NOT emit a (misleading all-zero) row for this plane } - } - else - if (saveOption == SaveOption::saveCSV) + + // thread-safely save results of this single (slide, channel, timeframe) + if (write_apache) { - if (save_features_2_csv_wholeslide (env, vroi, p.fname_int, "", outputPath, - 0 // pass 0 as t_index - ) == false) + auto [status, msg] = save_features_2_apache_wholeslide (env, vroi, p.fname_int, t, c); + if (!status) { - std::cout << "error saving results to CSV file, details: " << __FILE__ << ":" << __LINE__ << std::endl; + std::cerr << "Error writing Arrow file: " << msg.value() << std::endl; rv = 2; } } else - { - // pulls feature values from 'vroi' and appends them to global object 'theResultsCache' exposed to Python API - if (save_features_2_buffer_wholeslide (env.theResultsCache, env, vroi, p.fname_int, "") == false) + if (saveOption == SaveOption::saveCSV) { - std::cerr << "Error saving features to the results buffer" << std::endl; - rv = 2; + if (save_features_2_csv_wholeslide (env, vroi, p.fname_int, "", outputPath, t, c) == false) + { + std::cout << "error saving results to CSV file, details: " << __FILE__ << ":" << __LINE__ << std::endl; + rv = 2; + } } - } + else + { + // pulls feature values from 'vroi' and appends them to global object 'theResultsCache' exposed to Python API + if (save_features_2_buffer_wholeslide (env.theResultsCache, env, vroi, p.fname_int, "", t, c) == false) + { + std::cerr << "Error saving features to the results buffer" << std::endl; + rv = 2; + } + } + } //- channels x timeframes imlo.close(); @@ -203,7 +261,7 @@ namespace Nyxus // Not saving nested ROI related info because this image is single-ROI (whole-slide) // - rv = 0; // success + return rv; } @@ -215,6 +273,8 @@ namespace Nyxus const SaveOption saveOption, const std::string& outputPath) { + env.reset_csv_output_state(); // FIX: this run's CSV files start fresh (see Environment::csv_paths_written) + //**** prescan all files size_t nf = intensFiles.size(); @@ -274,13 +334,15 @@ namespace Nyxus // run batches of threads + int worst_rv = 0; // FIX: aggregate per-thread failure so an oversized/failed slide + // makes the whole run fail loudly (nonzero exit) instead of returning + // success -- the per-thread rvals were collected but never checked. size_t n_jobs = (nf + n_threads - 1) / n_threads; for (size_t j = 0; j < n_jobs; j++) { VERBOSLVL1 (env.get_verbosity_level(), std::cout << "whole-slide job " << j + 1 << "/" << n_jobs << "\n"); - std::vector> T; - std::vector rvals(n_threads, 0); + std::vector> T; for (int t = 0; t < n_threads; t++) { size_t idx = j * n_threads + t; @@ -300,12 +362,12 @@ namespace Nyxus nf, outputPath, write_apache, - saveOption, - std::ref(rvals[t]))); + saveOption)); } else { - featurize_3d_wv_thread( + // FIX: aggregate the returned status (was written through a lost reference) + int r = featurize_3d_wv_thread( env, intensFiles, intensFiles, @@ -313,14 +375,18 @@ namespace Nyxus nf, outputPath, write_apache, - saveOption, - rvals[t]); + saveOption); + if (r > worst_rv) worst_rv = r; } } - // wait for all threads to complete before proceeding + // wait for all threads to complete, aggregating each one's returned status so an + // oversized/failed slide makes the whole run fail loudly (nonzero exit) for (auto& f : T) - f.get(); + { + int r = f.get(); + if (r > worst_rv) worst_rv = r; + } // allow keyboard interrupt #ifdef WITH_PYTHON_H @@ -350,6 +416,15 @@ namespace Nyxus // future: free GPU cache for all participating devices // + // FIX: a slide that could not be featurized is a hard failure, not a silent success -- + // report it so the CLI exits nonzero and the Python API raises. An oversized whole volume + // streams out-of-core now (see featurize_wholevolume), so this only fires when that + // streaming itself failed (input format can't deliver plane-by-plane, or an unsupported + // feature was requested) -- the specific reason was already reported above. + if (worst_rv != 0) + return { false, "one or more slides could not be featurized (see errors above; " + "an unsupported format needs a segmented mask, or a smaller/plane-deliverable input)" }; + return {true, std::nullopt}; // success } diff --git a/src/nyx/workflow_pythonapi.cpp b/src/nyx/workflow_pythonapi.cpp index 5f336e26..625359d7 100644 --- a/src/nyx/workflow_pythonapi.cpp +++ b/src/nyx/workflow_pythonapi.cpp @@ -33,7 +33,10 @@ namespace Nyxus { bool scan_slide_props_montage(SlideProps& p, int dim, const AnisotropyOptions& aniso); - bool processIntSegImagePairInMemory (Environment & env, const py::array_t& intens, const py::array_t& label, int pair_index, const std::string& intens_name, const std::string& seg_name, std::vector unprocessed_rois) + // unprocessed_rois is an OUT parameter: the in-memory montage pipeline has no out-of-core + // path, so ROIs whose footprint reaches the RAM limit are returned to the caller to be + // reported as an error, instead of being dropped and emitting an all-zero feature row. + bool processIntSegImagePairInMemory (Environment & env, const py::array_t& intens, const py::array_t& label, int pair_index, const std::string& intens_name, const std::string& seg_name, std::vector& unprocessed_rois) { std::vector trivRoiLabels; @@ -147,13 +150,13 @@ namespace Nyxus if (write_apache) { - auto [status, msg] = env.arrow_stream.write_arrow_file (Nyxus::get_feature_values(env, env.theFeatureSet, env.uniqueLabels, env.roiData, env.dataset)); - if (!status) + auto [status, msg] = env.arrow_stream.write_arrow_file (Nyxus::get_feature_values(env, env.theFeatureSet, env.uniqueLabels, env.roiData, env.dataset, DEFAULT_T_INDEX, DEFAULT_C_INDEX)); + if (!status) return { "error writing Arrow file: " + msg.value() }; - } - else + } + else { - if (! save_features_2_buffer(env.theResultsCache, env, DEFAULT_T_INDEX)) + if (! save_features_2_buffer(env.theResultsCache, env, DEFAULT_T_INDEX, DEFAULT_C_INDEX)) return { "error saving results to a buffer" }; } diff --git a/tests/data/ometiff/bad_corrupt.tif b/tests/data/ometiff/bad_corrupt.tif new file mode 100644 index 00000000..745c8579 Binary files /dev/null and b/tests/data/ometiff/bad_corrupt.tif differ diff --git a/tests/data/ometiff/bad_rgb.ome.tif b/tests/data/ometiff/bad_rgb.ome.tif new file mode 100644 index 00000000..5f26d09a Binary files /dev/null and b/tests/data/ometiff/bad_rgb.ome.tif differ diff --git a/tests/data/ometiff/dim2_yx.ome.tif b/tests/data/ometiff/dim2_yx.ome.tif new file mode 100644 index 00000000..c5b706af Binary files /dev/null and b/tests/data/ometiff/dim2_yx.ome.tif differ diff --git a/tests/data/ometiff/dim3_emptymask.ome.tif b/tests/data/ometiff/dim3_emptymask.ome.tif new file mode 100644 index 00000000..644030e8 Binary files /dev/null and b/tests/data/ometiff/dim3_emptymask.ome.tif differ diff --git a/tests/data/ometiff/dim3_mask.ome.tif b/tests/data/ometiff/dim3_mask.ome.tif new file mode 100644 index 00000000..5377cad0 Binary files /dev/null and b/tests/data/ometiff/dim3_mask.ome.tif differ diff --git a/tests/data/ometiff/dim3_plain.tif b/tests/data/ometiff/dim3_plain.tif new file mode 100644 index 00000000..f169fc4c Binary files /dev/null and b/tests/data/ometiff/dim3_plain.tif differ diff --git a/tests/data/ometiff/dim3_zyx.ome.tif b/tests/data/ometiff/dim3_zyx.ome.tif new file mode 100644 index 00000000..1404c5db Binary files /dev/null and b/tests/data/ometiff/dim3_zyx.ome.tif differ diff --git a/tests/data/ometiff/dim4_czyx.ome.tif b/tests/data/ometiff/dim4_czyx.ome.tif new file mode 100644 index 00000000..21f5b4fc Binary files /dev/null and b/tests/data/ometiff/dim4_czyx.ome.tif differ diff --git a/tests/data/ometiff/dim4_mask_c2.ome.tif b/tests/data/ometiff/dim4_mask_c2.ome.tif new file mode 100644 index 00000000..bba34ac4 Binary files /dev/null and b/tests/data/ometiff/dim4_mask_c2.ome.tif differ diff --git a/tests/data/ometiff/dim4_tzyx.ome.tif b/tests/data/ometiff/dim4_tzyx.ome.tif new file mode 100644 index 00000000..44a16538 Binary files /dev/null and b/tests/data/ometiff/dim4_tzyx.ome.tif differ diff --git a/tests/data/ometiff/dim5.ome.tif b/tests/data/ometiff/dim5.ome.tif new file mode 100644 index 00000000..07dc1459 Binary files /dev/null and b/tests/data/ometiff/dim5.ome.tif differ diff --git a/tests/data/ometiff/dim5_badifd.ome.tif b/tests/data/ometiff/dim5_badifd.ome.tif new file mode 100644 index 00000000..2d91ed85 Binary files /dev/null and b/tests/data/ometiff/dim5_badifd.ome.tif differ diff --git a/tests/data/ometiff/dim5_calibrated.ome.tif b/tests/data/ometiff/dim5_calibrated.ome.tif new file mode 100644 index 00000000..d557df93 Binary files /dev/null and b/tests/data/ometiff/dim5_calibrated.ome.tif differ diff --git a/tests/data/ometiff/dim5_calibrated_nm.ome.tif b/tests/data/ometiff/dim5_calibrated_nm.ome.tif new file mode 100644 index 00000000..2f5016ea Binary files /dev/null and b/tests/data/ometiff/dim5_calibrated_nm.ome.tif differ diff --git a/tests/data/ometiff/dim5_ctzyx.ome.tif b/tests/data/ometiff/dim5_ctzyx.ome.tif new file mode 100644 index 00000000..aa43cf28 Binary files /dev/null and b/tests/data/ometiff/dim5_ctzyx.ome.tif differ diff --git a/tests/data/ometiff/dim5_cztyx.ome.tif b/tests/data/ometiff/dim5_cztyx.ome.tif new file mode 100644 index 00000000..3abb1be4 Binary files /dev/null and b/tests/data/ometiff/dim5_cztyx.ome.tif differ diff --git a/tests/data/ometiff/dim5_multitile.ome.tif b/tests/data/ometiff/dim5_multitile.ome.tif new file mode 100644 index 00000000..862ff649 Binary files /dev/null and b/tests/data/ometiff/dim5_multitile.ome.tif differ diff --git a/tests/data/ometiff/dim5_oddtile.ome.tif b/tests/data/ometiff/dim5_oddtile.ome.tif new file mode 100644 index 00000000..d32bdd89 Binary files /dev/null and b/tests/data/ometiff/dim5_oddtile.ome.tif differ diff --git a/tests/data/ometiff/dim5_pyramid.ome.tif b/tests/data/ometiff/dim5_pyramid.ome.tif new file mode 100644 index 00000000..32622434 Binary files /dev/null and b/tests/data/ometiff/dim5_pyramid.ome.tif differ diff --git a/tests/data/ometiff/dim5_reordered.ome.tif b/tests/data/ometiff/dim5_reordered.ome.tif new file mode 100644 index 00000000..6ef6e3f9 Binary files /dev/null and b/tests/data/ometiff/dim5_reordered.ome.tif differ diff --git a/tests/data/ometiff/dim5_tiled.ome.tif b/tests/data/ometiff/dim5_tiled.ome.tif new file mode 100644 index 00000000..b5a332c5 Binary files /dev/null and b/tests/data/ometiff/dim5_tiled.ome.tif differ diff --git a/tests/data/ometiff/dim5_tzcyx.ome.tif b/tests/data/ometiff/dim5_tzcyx.ome.tif new file mode 100644 index 00000000..2bce4e94 Binary files /dev/null and b/tests/data/ometiff/dim5_tzcyx.ome.tif differ diff --git a/tests/data/ometiff/dim5_zctyx.ome.tif b/tests/data/ometiff/dim5_zctyx.ome.tif new file mode 100644 index 00000000..636f669f Binary files /dev/null and b/tests/data/ometiff/dim5_zctyx.ome.tif differ diff --git a/tests/data/ometiff/dim5_ztcyx.ome.tif b/tests/data/ometiff/dim5_ztcyx.ome.tif new file mode 100644 index 00000000..d87038fe Binary files /dev/null and b/tests/data/ometiff/dim5_ztcyx.ome.tif differ diff --git a/tests/data/ometiff/gen_ome_tiff.py b/tests/data/ometiff/gen_ome_tiff.py new file mode 100644 index 00000000..e253be14 --- /dev/null +++ b/tests/data/ometiff/gen_ome_tiff.py @@ -0,0 +1,295 @@ +"""Generate OME-TIFF fixtures whose every voxel encodes its own (x,y,z,c,t) +coordinate, so a reader that maps (z,c,t) to the wrong IFD (page) reads a +provably wrong value. + + value(x,y,z,c,t) = 1 + ((((t*C + c)*Z + z)*Y + y)*X + x) # C=3,Z=4,Y=6,X=8 + +Stores (same encoding throughout; absent axes pinned to index 0): + + dim5.ome.tif axes TCZYX -> DimensionOrder XYZCT (default: Z fastest) + dim5_ctzyx.ome.tif axes CTZYX -> DimensionOrder XYZTC (non-default: T before C) + -> proves the plane->IFD map honors DimensionOrder + dim3_zyx.ome.tif axes ZYX 3D + dim2_yx.ome.tif axes YX 2D + dim3_plain.tif plain multi-page TIFF (NO OME-XML) -> legacy page=Z fallback + +tifffile writes OME-TIFF as multi-page (strip) with one IFD per (z,c,t) plane in +DimensionOrder raster order, so the loader's ifdForPlane() must reproduce that. + +Run with a python that has tifffile (e.g. the conda build env): + $CONDA_PREFIX/python.exe gen_ome_tiff.py +""" +import os +import numpy as np +import tifffile + +T, C, Z, Y, X = 2, 3, 4, 6, 8 +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def encoded_tczyx(): + t, c, z, y, x = np.meshgrid( + np.arange(T), np.arange(C), np.arange(Z), np.arange(Y), np.arange(X), indexing="ij") + return (1 + ((((t * C + c) * Z + z) * Y + y) * X + x)).astype("uint16") + + +def _data_for(order): + base = encoded_tczyx() # [t,c,z,y,x] + sel = tuple(slice(None) if a in order else 0 for a in "TCZYX") + sub = base[sel] + remaining = [a for a in "TCZYX" if a in order] + perm = tuple(remaining.index(a) for a in order) + return np.transpose(sub, perm).copy() + + +def write_ome(name, order): + path = os.path.join(HERE, name) + data = _data_for(order) + tifffile.imwrite(path, data, photometric="minisblack", metadata={"axes": order}, ome=True) + with tifffile.TiffFile(path) as tf: + import re + dimord = re.search(r'DimensionOrder="([A-Z]+)"', tf.ome_metadata).group(1) + print("wrote %-20s axes=%-6s DimensionOrder=%s IFDs=%d tiled=%s" + % (name, order, dimord, len(tf.pages), tf.pages[0].is_tiled)) + + +def write_plain(name, order): + """Plain multi-page TIFF, no OME-XML (tests the non-OME fallback).""" + path = os.path.join(HERE, name) + data = _data_for(order) # e.g. ZYX -> a plain z-stack + tifffile.imwrite(path, data, photometric="minisblack") + with tifffile.TiffFile(path) as tf: + print("wrote %-20s axes=%-6s (plain, no OME) IFDs=%d" % (name, order, len(tf.pages))) + + +def write_tiled(name, order): + """A TILED multi-plane OME-TIFF (one IFD per (z,c,t) plane, each plane internally + tiled). tile=(16,16) makes the 6x8 plane a single tile, so the volumetric read + (which assembles tile (0,0) of each plane) covers the whole plane. Exercises the + tile-loader (z,c,t)->IFD path, distinct from the strip loaders.""" + path = os.path.join(HERE, name) + data = _data_for(order) + tifffile.imwrite(path, data, photometric="minisblack", metadata={"axes": order}, + tile=(16, 16), ome=True) + with tifffile.TiffFile(path) as tf: + import re + dimord = re.search(r'DimensionOrder="([A-Z]+)"', tf.ome_metadata).group(1) + print("wrote %-20s axes=%-6s DimensionOrder=%s IFDs=%d tiled=%s" + % (name, order, dimord, len(tf.pages), tf.pages[0].is_tiled)) + + +# Multi-tile fixture dims. TIFF tile sizes must be multiples of 16, so the plane has to +# exceed 16 in BOTH directions to produce a real tile grid -- hence its own (larger Y/X, +# smaller C/Z/T) shape rather than the 6x8 plane the other fixtures share. +MT_T, MT_C, MT_Z, MT_Y, MT_X = 1, 2, 2, 32, 48 + + +def write_multitile(name, T=MT_T, C=MT_C, Z=MT_Z, Y=MT_Y, X=MT_X): + """A tiled OME-TIFF whose every plane spans a GRID of 16x16 tiles (default 32x48 plane + -> 2x3 = 6 tiles), unlike dim5_tiled.ome.tif where a single tile covers the whole plane. + A volumetric read that fetches only tile (0,0) returns wrong data for everything outside + the first 16x16 corner. Same coordinate encoding as encoded_tczyx, its own dims. + + When Y or X is not a multiple of 16 the last tile of that row/column is PARTIAL, which + exercises the validH/validW seam clamp in the volumetric assembly.""" + t, c, z, y, x = np.meshgrid(np.arange(T), np.arange(C), np.arange(Z), + np.arange(Y), np.arange(X), indexing="ij") + data = (1 + ((((t * C + c) * Z + z) * Y + y) * X + x)).astype("uint16") + path = os.path.join(HERE, name) + tifffile.imwrite(path, data, photometric="minisblack", metadata={"axes": "TCZYX"}, + tile=(16, 16), ome=True) + with tifffile.TiffFile(path) as tf: + print("wrote %-24s %dx%d plane, tile grid %dx%d (last %dx%d), IFDs=%d tiled=%s max=%d" + % (name, Y, X, -(-Y // 16), -(-X // 16), Y % 16 or 16, X % 16 or 16, + len(tf.pages), tf.pages[0].is_tiled, data.max())) + + +def write_mask(name): + """Single-channel, single-timeframe 3D (ZYX) label mask matching dim5's geometry. + Used to test the 1-channel-mask : N-channel-intensity pairing: the mask is + channel-agnostic and must be reused for every intensity channel.""" + path = os.path.join(HERE, name) + m = np.zeros((Z, Y, X), "uint16") + m[:, 1:5, 1:7] = 1 # one ROI (label 1), interior so it has a real bbox + tifffile.imwrite(path, m, photometric="minisblack", metadata={"axes": "ZYX"}, ome=True) + print("wrote %-20s (single-channel ZYX label mask, ROI voxels=%d)" % (name, int(m.sum()))) + + +def write_reordered(name): + """An OME-TIFF whose planes are physically stored in REVERSED IFD order, with explicit + blocks mapping each logical (z,c,t) to its scrambled IFD. This is what a reader + that ignores TiffData and assumes canonical (contiguous-from-IFD-0) plane order gets wrong: + it would read the reversed plane's pixels. Writers like bioformats emit per-plane TiffData; + a non-canonical mapping (or a non-zero starting IFD) is exactly where the assumption breaks. + + Dims T=1,C=2,Z=3, plane 6x8, DimensionOrder XYZCT -> canonical ordinal ord = z + c*Z. The + logical plane with ordinal `ord` is stored at physical IFD (5 - ord). Same encoded value as + encoded_tczyx so the standard facade test can check it.""" + rT, rC, rZ = 1, 2, 3 + total = rZ * rC * rT + # physical IFD p holds the logical plane whose ordinal is (total-1 - p) + phys = np.empty((total, Y, X), "uint16") + tiffdata = [] + for c in range(rC): + for z in range(rZ): + ord_ = z + c * rZ # canonical XYZCT ordinal (t=0) + ifd = (total - 1) - ord_ # reversed physical IFD + yy, xx = np.meshgrid(np.arange(Y), np.arange(X), indexing="ij") + phys[ifd] = (1 + ((((0 * rC + c) * rZ + z) * Y + yy) * X + xx)).astype("uint16") + tiffdata.append('' + % (c, z, ifd)) + ome = ('' + '' + '' + '%s' + '' + % (X, Y, rZ, rC, rT, "".join(tiffdata))) + path = os.path.join(HERE, name) + tifffile.imwrite(path, phys, description=ome, metadata=None, photometric="minisblack") + with tifffile.TiffFile(path) as tf: + got = (tf.pages[0].description or "")[:4] + print("wrote %-24s IFDs=%d reversed, TiffData blocks=%d, desc starts %r" + % (name, len(tf.pages), len(tiffdata), got)) + + +def write_pyramid(name): + """A PYRAMIDAL OME-TIFF: every full-res plane's IFD carries downsampled resolution levels + as SubIFDs (TIFF tag 330). SubIFDs live OUTSIDE the main IFD chain, so they must not + disturb full-res (z,c,t) plane addressing -- TIFFNumberOfDirectories still returns Z (not + Z*levels) and ifdForPlane(z,..)->main-chain IFD still lands on the full-res plane. nyxus + featurizes level 0 only; this fixture guards that a pyramid does not silently shift which + plane we read. Z=6 z-stack, 32x48, tiled 16x16 (so also multi-tile), + 2 sub-levels. + Full-res encoding matches ometiff_enc_dims(x,y,z,0,0,C=1,Z=6,Y=32,X=48).""" + Zp, Yp, Xp = 6, 32, 48 + z, y, x = np.meshgrid(np.arange(Zp), np.arange(Yp), np.arange(Xp), indexing="ij") + base = (1 + ((z * Yp + y) * Xp + x)).astype("uint16") # full-res planes + path = os.path.join(HERE, name) + opts = dict(photometric="minisblack", tile=(16, 16)) + with tifffile.TiffWriter(path, ome=True) as tw: + tw.write(base, subifds=2, metadata={"axes": "ZYX"}, **opts) # reserve 2 subifds/plane + tw.write(base[:, ::2, ::2], subfiletype=1, **opts) # level 1 (16x24) + tw.write(base[:, ::4, ::4], subfiletype=1, **opts) # level 2 (8x12) + with tifffile.TiffFile(path) as tf: + print("wrote %-24s main-chain IFDs=%d (=Z), sub-levels/plane=%d, %dx%d tiled" + % (name, len(tf.pages), len(tf.series[0].levels) - 1, Yp, Xp)) + + +def write_calibrated_tiff(name): + """P2: an OME-TIFF carrying physical voxel spacing (PhysicalSizeX/Y/Z + units) in its + OME-XML, so the loader must surface physicalSize*/unit into SlideProps -- the TIFF twin of + dim5_calibrated.ome.zarr. Anisotropic: X=Y=0.5, Z=2.0 micrometer. TCZYX 2,3,4,6,8.""" + data = _data_for("TCZYX") + path = os.path.join(HERE, name) + tifffile.imwrite(path, data, photometric="minisblack", ome=True, metadata={ + "axes": "TCZYX", "PhysicalSizeX": 0.5, "PhysicalSizeY": 0.5, "PhysicalSizeZ": 2.0, + "PhysicalSizeXUnit": "micrometer", "PhysicalSizeYUnit": "micrometer", + "PhysicalSizeZUnit": "micrometer"}) + print("wrote %-24s (calibrated: physX/Y=0.5 physZ=2.0 micrometer)" % name) + + +def write_calibrated_tiff_nm(name): + """Same physical spacing as write_calibrated_tiff, but X/Y declared in NANOMETER and Z in + a THIRD unit (millimeter) -- 500 nm == 0.5 um, 0.002 mm == 2.0 um. The loader must + canonicalize each axis using ITS OWN declared unit and report the SAME physX/Y/Z and a + single "micrometer" unit as write_calibrated_tiff, proving conversion (not passthrough) + and that X/Y's unit isn't silently applied to Z or vice versa.""" + data = _data_for("TCZYX") + path = os.path.join(HERE, name) + tifffile.imwrite(path, data, photometric="minisblack", ome=True, metadata={ + "axes": "TCZYX", "PhysicalSizeX": 500.0, "PhysicalSizeY": 500.0, "PhysicalSizeZ": 0.002, + "PhysicalSizeXUnit": "nanometer", "PhysicalSizeYUnit": "nanometer", + "PhysicalSizeZUnit": "millimeter"}) + print("wrote %-24s (calibrated: physX/Y=500nm physZ=0.002mm -> both canonicalize to 0.5/2.0 um)" % name) + + +def write_bad_ifd(name): + """N2: an OME-TIFF whose maps one plane to an IFD PAST the end of the file (an + in-file overrun, not a multi-file UUID). ifdForPlane returns it, so the read must throw + cleanly at TIFFSetDirectory, not crash. 4 planes stored, but plane (z3) claims IFD=99.""" + rT, rC, rZ = 1, 1, 4 + phys = np.empty((rZ, Y, X), "uint16") + yy, xx = np.meshgrid(np.arange(Y), np.arange(X), indexing="ij") + td = [] + for z in range(rZ): + phys[z] = (1 + ((z * Y + yy) * X + xx)).astype("uint16") + ifd = 99 if z == rZ - 1 else z # last plane points past EOF + td.append('' % (z, ifd)) + ome = ('' + '' + '%s' + '' % (X, Y, rZ, "".join(td))) + tifffile.imwrite(os.path.join(HERE, name), phys, description=ome, metadata=None, + photometric="minisblack") + print("wrote %-24s (TiffData IFD=99 on plane z3 -> read must throw)" % name) + + +def write_empty_mask(name): + """N3: an all-background (all-zero) single-channel ZYX mask. A segmented run finds ZERO + ROIs -- the pipeline must handle it (no ROIs, no crash), not divide-by-zero or read garbage.""" + m = np.zeros((Z, Y, X), "uint16") + tifffile.imwrite(os.path.join(HERE, name), m, photometric="minisblack", + metadata={"axes": "ZYX"}, ome=True) + print("wrote %-24s (empty mask -> 0 ROIs)" % name) + + +def write_multichannel_mask(name): + """N4: a MULTI-channel (CZYX, C=2) label mask, to pair with single-channel intensity. The + featurize loop iterates the INTENSITY's channels, so the extra mask channel must simply be + ignored (mask channel clamped to what the intensity asks for), not crash. Same ROI on both + mask channels so the result is channel-independent.""" + m = np.zeros((2, Z, Y, X), "uint16") + m[:, :, 1:5, 1:7] = 1 + tifffile.imwrite(os.path.join(HERE, name), m, photometric="minisblack", + metadata={"axes": "CZYX"}, ome=True) + print("wrote %-24s (C=2 label mask -> extra channel ignored)" % name) + + +def write_bad_rgb(name): + """RGB OME-TIFF -- nyxus is grayscale-only, so the loader must reject it.""" + path = os.path.join(HERE, name) + data = np.zeros((Y, X, 3), "uint8") + tifffile.imwrite(path, data, photometric="rgb", metadata={"axes": "YXS"}, ome=True) + print("wrote %-20s (RGB -> not grayscale)" % name) + + +def write_bad_corrupt(name): + """Not a TIFF at all -- TIFFOpen must fail cleanly.""" + path = os.path.join(HERE, name) + with open(path, "wb") as f: + f.write(b"this is definitely not a TIFF file\n\x00\x01\x02\x03") + print("wrote %-20s (corrupt -> open fails)" % name) + + +def main(): + # All 6 orderings of {T,C,Z} before Y,X -> the 6 legal OME DimensionOrder values. + for o in ["TCZYX", "TZCYX", "CTZYX", "CZTYX", "ZTCYX", "ZCTYX"]: + write_ome("dim5.ome.tif" if o == "TCZYX" else "dim5_%s.ome.tif" % o.lower(), o) + # 4D (rank-4 coverage): one time-only, one channel-only. + write_ome("dim4_tzyx.ome.tif", "TZYX") + write_ome("dim4_czyx.ome.tif", "CZYX") + # lower rank + non-OME fallback + write_ome("dim3_zyx.ome.tif", "ZYX") + write_ome("dim2_yx.ome.tif", "YX") + write_plain("dim3_plain.tif", "ZYX") + # single-channel label mask (for the 1-mask : N-channel-intensity pairing test) + write_mask("dim3_mask.ome.tif") + # non-canonical plane->IFD mapping (planes stored reversed) -> honors TiffData + write_reordered("dim5_reordered.ome.tif") + # pyramidal OME-TIFF (SubIFD downsample levels) -> full-res addressing must be unaffected + write_pyramid("dim5_pyramid.ome.tif") + # TILED multi-plane OME-TIFF (5D) -> exercises the tile-loader (z,c,t)->IFD path + write_tiled("dim5_tiled.ome.tif", "TCZYX") + # planes spanning a real 2x3 tile grid -> multi-tile volumetric assembly + write_multitile("dim5_multitile.ome.tif") + # PARTIAL edge tiles: 40x24 plane / 16 -> 3x2 tiles, last row-tile 8 tall, last col-tile + # 8 wide. Exercises the seam clamp the exact-multiple multitile fixture leaves untested. + write_multitile("dim5_oddtile.ome.tif", T=1, C=2, Z=1, Y=40, X=24) + # --- illegal / adversarial: must be rejected cleanly, not crash --- + write_bad_rgb("bad_rgb.ome.tif") + write_bad_corrupt("bad_corrupt.tif") + + +if __name__ == "__main__": + main() diff --git a/tests/data/omezarr/README.md b/tests/data/omezarr/README.md index c3bcac8d..fe0ae39e 100644 --- a/tests/data/omezarr/README.md +++ b/tests/data/omezarr/README.md @@ -17,10 +17,30 @@ and `chunks[2..4]` as the tile sizes. |------------------|-----------:|--------|-----------|-----------|------------------------| | `test.ome.zarr` | 512x512 | uint16 | 1024x1024 | 1x1 | `(row + col) % 65536` | | `multi.ome.zarr` | 1500x1200 | uint16 | 1024x1024 | 2x2* | `(row*7 + col*3) % 65536` | +| `dim5.ome.zarr` | 6x8 | uint16 | 2x3x1x6x8 | 1x1 | 5D coordinate encoding (below) | \* `multi.ome.zarr` has partial edge tiles (1500 = 1024 + 476, 1200 = 1024 + 176), so it also exercises the loaders' partial-tile clipping path. +### `dim5.ome.zarr` — 5D channel/timeframe addressability + +A genuinely 5D store, shape `(T=2, C=3, Z=4, Y=6, X=8)`, chunked `(T,C,1,Y,X)` — +one chunk per z-slice (4 chunk files) — and written **uncompressed** (so it reads +with header-only z5). Z gets its own chunk because the loader maps the layer index +as `layer*tileDepth` (`tileDepth = chunks[2]`); C and T are addressed directly by +the read offset, so they don't need separate chunks. Every voxel encodes its own +coordinate: + +``` +value(x,y,z,c,t) = 1 + ((((t*C + c)*Z + z)*Y + y)*X + x) # C=3, Z=4, Y=6, X=8 +``` + +`test_omezarr_5d_channel_time_addressing` / `test_raw_omezarr_5d_channel_time_addressing` +read plane `(z,c,t)` via `loadTileFromFile(..., layer=z, channel=c, timeframe=t, ...)` +and assert every value matches the encoding — so a loader that ignored C/T (offset +pinned to `{0,0,...}`) returns the wrong plane and fails. Generated by `gen_dim5.py` +(zarr 3.x). Regenerate: `python gen_dim5.py`. + Deterministic checksums asserted by the tests: - `test.ome.zarr` sum of all pixels = `133955584` diff --git a/tests/data/omezarr/bad_axes_count.ome.zarr/.zattrs b/tests/data/omezarr/bad_axes_count.ome.zarr/.zattrs new file mode 100644 index 00000000..0a8629e5 --- /dev/null +++ b/tests/data/omezarr/bad_axes_count.ome.zarr/.zattrs @@ -0,0 +1,46 @@ +{ + "multiscales": [ + { + "version": "0.4", + "axes": [ + { + "name": "t", + "type": "time" + }, + { + "name": "c", + "type": "channel" + }, + { + "name": "z", + "type": "space" + }, + { + "name": "y", + "type": "space" + }, + { + "name": "x", + "type": "space" + } + ], + "datasets": [ + { + "path": "0", + "coordinateTransformations": [ + { + "type": "scale", + "scale": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ] + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/data/omezarr/bad_axes_count.ome.zarr/.zgroup b/tests/data/omezarr/bad_axes_count.ome.zarr/.zgroup new file mode 100644 index 00000000..cab13da6 --- /dev/null +++ b/tests/data/omezarr/bad_axes_count.ome.zarr/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format": 2 +} \ No newline at end of file diff --git a/tests/data/omezarr/bad_axes_count.ome.zarr/0/.zarray b/tests/data/omezarr/bad_axes_count.ome.zarr/0/.zarray new file mode 100644 index 00000000..9226eae5 --- /dev/null +++ b/tests/data/omezarr/bad_axes_count.ome.zarr/0/.zarray @@ -0,0 +1,19 @@ +{ + "shape": [ + 4, + 6, + 8 + ], + "chunks": [ + 1, + 6, + 8 + ], + "dtype": "?@EFGHMNOP \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.1.3.0.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.1.3.0.0 new file mode 100644 index 00000000..03e422fd --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.1.3.0.0 @@ -0,0 +1 @@ +QRSTYZ[\abcd \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.1.3.0.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.1.3.0.1 new file mode 100644 index 00000000..532ad629 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.1.3.0.1 @@ -0,0 +1 @@ +UVWX]^_`efgh \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.1.3.1.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.1.3.1.0 new file mode 100644 index 00000000..08d7221e --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.1.3.1.0 @@ -0,0 +1 @@ +ijklqrstyz{| \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.1.3.1.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.1.3.1.1 new file mode 100644 index 00000000..ecf4d935 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.1.3.1.1 @@ -0,0 +1 @@ +mnopuvwx}~€ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.0.0.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.0.0.0 new file mode 100644 index 00000000..ccd40725 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.0.0.0 @@ -0,0 +1 @@ +‚ƒ„‰Š‹Œ‘’“” \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.0.0.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.0.0.1 new file mode 100644 index 00000000..33a72bc4 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.0.0.1 @@ -0,0 +1 @@ +…†‡ˆŽ•–—˜ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.0.1.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.0.1.0 new file mode 100644 index 00000000..38bb58d9 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.0.1.0 @@ -0,0 +1 @@ +™š›œ¡¢£¤©ª«¬ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.0.1.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.0.1.1 new file mode 100644 index 00000000..ae8b9729 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.0.1.1 @@ -0,0 +1 @@ +žŸ ¥¦§¨­®¯° \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.1.0.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.1.0.0 new file mode 100644 index 00000000..ef8c5704 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.1.0.0 @@ -0,0 +1 @@ +±²³´¹º»¼ÁÂÃÄ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.1.0.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.1.0.1 new file mode 100644 index 00000000..1bc969c0 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.1.0.1 @@ -0,0 +1 @@ +µ¶·¸½¾¿ÀÅÆÇÈ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.1.1.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.1.1.0 new file mode 100644 index 00000000..498c077b --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.1.1.0 @@ -0,0 +1 @@ +ÉÊËÌÑÒÓÔÙÚÛÜ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.1.1.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.1.1.1 new file mode 100644 index 00000000..69b4e588 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.1.1.1 @@ -0,0 +1 @@ +ÍÎÏÐÕÖ×ØÝÞßà \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.2.0.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.2.0.0 new file mode 100644 index 00000000..32030d08 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.2.0.0 @@ -0,0 +1 @@ +áâãäéêëìñòóô \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.2.0.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.2.0.1 new file mode 100644 index 00000000..a7b244a4 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.2.0.1 @@ -0,0 +1 @@ +åæçèíîïðõö÷ø \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.2.1.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.2.1.0 new file mode 100644 index 00000000..9dab26a8 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.2.1.0 @@ -0,0 +1,2 @@ +ùúûü  +   \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.2.1.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.2.1.1 new file mode 100644 index 00000000..c4e75984 Binary files /dev/null and b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.2.1.1 differ diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.3.0.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.3.0.0 new file mode 100644 index 00000000..ba4ab377 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.3.0.0 @@ -0,0 +1 @@ +!"#$ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.3.0.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.3.0.1 new file mode 100644 index 00000000..8ceadd6c --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.3.0.1 @@ -0,0 +1 @@ + %&'( \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.3.1.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.3.1.0 new file mode 100644 index 00000000..e288e02a --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.3.1.0 @@ -0,0 +1 @@ +)*+,12349:;< \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.3.1.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.3.1.1 new file mode 100644 index 00000000..d0032195 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/0.2.3.1.1 @@ -0,0 +1 @@ +-./05678=>?@ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.0.0.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.0.0.0 new file mode 100644 index 00000000..90eba08e --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.0.0.0 @@ -0,0 +1 @@ +ABCDIJKLQRST \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.0.0.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.0.0.1 new file mode 100644 index 00000000..c8f3df21 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.0.0.1 @@ -0,0 +1 @@ +EFGHMNOPUVWX \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.0.1.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.0.1.0 new file mode 100644 index 00000000..f2534bd1 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.0.1.0 @@ -0,0 +1 @@ +YZ[\abcdijkl \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.0.1.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.0.1.1 new file mode 100644 index 00000000..0771cddb --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.0.1.1 @@ -0,0 +1 @@ +]^_`efghmnop \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.1.0.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.1.0.0 new file mode 100644 index 00000000..da21a11f --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.1.0.0 @@ -0,0 +1 @@ +qrstyz{|‚ƒ„ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.1.0.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.1.0.1 new file mode 100644 index 00000000..11ff0eb4 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.1.0.1 @@ -0,0 +1 @@ +uvwx}~€…†‡ˆ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.1.1.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.1.1.0 new file mode 100644 index 00000000..ef326d17 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.1.1.0 @@ -0,0 +1 @@ +‰Š‹Œ‘’“”™š›œ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.1.1.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.1.1.1 new file mode 100644 index 00000000..095916ce --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.1.1.1 @@ -0,0 +1 @@ +Ž•–—˜žŸ  \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.2.0.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.2.0.0 new file mode 100644 index 00000000..c754de7d --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.2.0.0 @@ -0,0 +1 @@ +¡¢£¤©ª«¬±²³´ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.2.0.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.2.0.1 new file mode 100644 index 00000000..3fc7c562 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.2.0.1 @@ -0,0 +1 @@ +¥¦§¨­®¯°µ¶·¸ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.2.1.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.2.1.0 new file mode 100644 index 00000000..f1ab8d54 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.2.1.0 @@ -0,0 +1 @@ +¹º»¼ÁÂÃÄÉÊËÌ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.2.1.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.2.1.1 new file mode 100644 index 00000000..79919cf8 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.2.1.1 @@ -0,0 +1 @@ +½¾¿ÀÅÆÇÈÍÎÏÐ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.3.0.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.3.0.0 new file mode 100644 index 00000000..2498f594 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.3.0.0 @@ -0,0 +1 @@ +ÑÒÓÔÙÚÛÜáâãä \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.3.0.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.3.0.1 new file mode 100644 index 00000000..402050ce --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.3.0.1 @@ -0,0 +1 @@ +ÕÖ×ØÝÞßàåæçè \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.3.1.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.3.1.0 new file mode 100644 index 00000000..e77d3f52 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.3.1.0 @@ -0,0 +1 @@ +éêëìñòóôùúûü \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.3.1.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.3.1.1 new file mode 100644 index 00000000..9f39c2bb Binary files /dev/null and b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.0.3.1.1 differ diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.0.0.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.0.0.0 new file mode 100644 index 00000000..18af2a38 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.0.0.0 @@ -0,0 +1,2 @@ +  +   \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.0.0.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.0.0.1 new file mode 100644 index 00000000..4c6e6ccd --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.0.0.1 @@ -0,0 +1 @@ +  \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.0.1.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.0.1.0 new file mode 100644 index 00000000..34ca0c10 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.0.1.0 @@ -0,0 +1 @@ +!"#$)*+, \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.0.1.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.0.1.1 new file mode 100644 index 00000000..a62a88bd --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.0.1.1 @@ -0,0 +1 @@ + %&'(-./0 \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.1.0.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.1.0.0 new file mode 100644 index 00000000..09510313 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.1.0.0 @@ -0,0 +1 @@ +12349:;<ABCD \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.1.0.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.1.0.1 new file mode 100644 index 00000000..1800462e --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.1.0.1 @@ -0,0 +1 @@ +5678=>?@EFGH \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.1.1.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.1.1.0 new file mode 100644 index 00000000..555fdeb6 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.1.1.0 @@ -0,0 +1 @@ +IJKLQRSTYZ[\ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.1.1.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.1.1.1 new file mode 100644 index 00000000..f3cd3949 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.1.1.1 @@ -0,0 +1 @@ +MNOPUVWX]^_` \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.2.0.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.2.0.0 new file mode 100644 index 00000000..9d717834 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.2.0.0 @@ -0,0 +1 @@ +abcdijklqrst \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.2.0.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.2.0.1 new file mode 100644 index 00000000..e4cef603 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.2.0.1 @@ -0,0 +1 @@ +efghmnopuvwx \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.2.1.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.2.1.0 new file mode 100644 index 00000000..5573f5f0 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.2.1.0 @@ -0,0 +1 @@ +yz{|‚ƒ„‰Š‹Œ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.2.1.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.2.1.1 new file mode 100644 index 00000000..68c39124 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.2.1.1 @@ -0,0 +1 @@ +}~€…†‡ˆŽ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.3.0.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.3.0.0 new file mode 100644 index 00000000..15832f81 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.3.0.0 @@ -0,0 +1 @@ +‘’“”™š›œ¡¢£¤ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.3.0.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.3.0.1 new file mode 100644 index 00000000..afbcd4db --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.3.0.1 @@ -0,0 +1 @@ +•–—˜žŸ ¥¦§¨ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.3.1.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.3.1.0 new file mode 100644 index 00000000..e431e7f1 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.3.1.0 @@ -0,0 +1 @@ +©ª«¬±²³´¹º»¼ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.3.1.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.3.1.1 new file mode 100644 index 00000000..2108967a --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.1.3.1.1 @@ -0,0 +1 @@ +­®¯°µ¶·¸½¾¿À \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.0.0.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.0.0.0 new file mode 100644 index 00000000..929d22a8 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.0.0.0 @@ -0,0 +1 @@ +ÁÂÃÄÉÊËÌÑÒÓÔ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.0.0.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.0.0.1 new file mode 100644 index 00000000..6a48804c --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.0.0.1 @@ -0,0 +1 @@ +ÅÆÇÈÍÎÏÐÕÖ×Ø \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.0.1.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.0.1.0 new file mode 100644 index 00000000..04e07416 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.0.1.0 @@ -0,0 +1 @@ +ÙÚÛÜáâãäéêëì \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.0.1.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.0.1.1 new file mode 100644 index 00000000..2b0cec5c --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.0.1.1 @@ -0,0 +1 @@ +ÝÞßàåæçèíîïð \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.1.0.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.1.0.0 new file mode 100644 index 00000000..acbf29f9 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.1.0.0 @@ -0,0 +1 @@ +ñòóôùúûü \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.1.0.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.1.0.1 new file mode 100644 index 00000000..0f3d38f4 Binary files /dev/null and b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.1.0.1 differ diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.1.1.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.1.1.0 new file mode 100644 index 00000000..81234e04 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.1.1.0 @@ -0,0 +1,2 @@ +  +   \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.1.1.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.1.1.1 new file mode 100644 index 00000000..18e42d2b --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.1.1.1 @@ -0,0 +1 @@ +   \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.2.0.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.2.0.0 new file mode 100644 index 00000000..0ec25109 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.2.0.0 @@ -0,0 +1 @@ +!"#$)*+,1234 \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.2.0.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.2.0.1 new file mode 100644 index 00000000..62eeb98e --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.2.0.1 @@ -0,0 +1 @@ +%&'(-./05678 \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.2.1.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.2.1.0 new file mode 100644 index 00000000..6787fc38 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.2.1.0 @@ -0,0 +1 @@ +9:;<ABCDIJKL \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.2.1.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.2.1.1 new file mode 100644 index 00000000..e8472acb --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.2.1.1 @@ -0,0 +1 @@ +=>?@EFGHMNOP \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.3.0.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.3.0.0 new file mode 100644 index 00000000..bf9cdef3 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.3.0.0 @@ -0,0 +1 @@ +QRSTYZ[\abcd \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.3.0.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.3.0.1 new file mode 100644 index 00000000..34681a52 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.3.0.1 @@ -0,0 +1 @@ +UVWX]^_`efgh \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.3.1.0 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.3.1.0 new file mode 100644 index 00000000..b95127c1 --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.3.1.0 @@ -0,0 +1 @@ +ijklqrstyz{| \ No newline at end of file diff --git a/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.3.1.1 b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.3.1.1 new file mode 100644 index 00000000..35fd5cfd --- /dev/null +++ b/tests/data/omezarr/dim5_multichunk.ome.zarr/0/1.2.3.1.1 @@ -0,0 +1 @@ +mnopuvwx}~€ \ No newline at end of file diff --git a/tests/data/omezarr/dim5_noaxes.ome.zarr/.zattrs b/tests/data/omezarr/dim5_noaxes.ome.zarr/.zattrs new file mode 100644 index 00000000..c1ebb88a --- /dev/null +++ b/tests/data/omezarr/dim5_noaxes.ome.zarr/.zattrs @@ -0,0 +1,24 @@ +{ + "multiscales": [ + { + "version": "0.4", + "datasets": [ + { + "path": "0", + "coordinateTransformations": [ + { + "type": "scale", + "scale": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ] + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/data/omezarr/dim5_noaxes.ome.zarr/.zgroup b/tests/data/omezarr/dim5_noaxes.ome.zarr/.zgroup new file mode 100644 index 00000000..cab13da6 --- /dev/null +++ b/tests/data/omezarr/dim5_noaxes.ome.zarr/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format": 2 +} \ No newline at end of file diff --git a/tests/data/omezarr/dim5_noaxes.ome.zarr/0/.zarray b/tests/data/omezarr/dim5_noaxes.ome.zarr/0/.zarray new file mode 100644 index 00000000..7f0e4af6 --- /dev/null +++ b/tests/data/omezarr/dim5_noaxes.ome.zarr/0/.zarray @@ -0,0 +1,23 @@ +{ + "shape": [ + 2, + 3, + 4, + 6, + 8 + ], + "chunks": [ + 2, + 3, + 1, + 6, + 8 + ], + "dtype": " proves 'axes' is honored + dim3_zyx.ome.zarr axes z,y,x 3D -> proves rank-3 axis mapping + dim2_yx.ome.zarr axes y,x 2D -> proves rank-2 axis mapping + dim5_noaxes.ome.zarr (no 'axes' key) 5D TCZYX -> proves the legacy fallback + +All chunked one z-slice per chunk and written UNCOMPRESSED (compressor=None) so +they read with header-only z5 (no blosc/zlib link). + +Run with a python that has zarr 3.x (e.g. the conda build env): + $CONDA_PREFIX/python.exe gen_dim5.py +""" +import os +import shutil +import json +import numpy as np +import zarr +from zarr.codecs import BloscCodec + +T, C, Z, Y, X = 2, 3, 4, 6, 8 +HERE = os.path.dirname(os.path.abspath(__file__)) + +_AX = { + "t": {"name": "t", "type": "time"}, + "c": {"name": "c", "type": "channel"}, + "z": {"name": "z", "type": "space"}, + "y": {"name": "y", "type": "space"}, + "x": {"name": "x", "type": "space"}, +} + + +def encoded_tczyx(): + """Logical volume indexed [t,c,z,y,x].""" + t, c, z, y, x = np.meshgrid( + np.arange(T), np.arange(C), np.arange(Z), np.arange(Y), np.arange(X), indexing="ij") + return (1 + ((((t * C + c) * Z + z) * Y + y) * X + x)).astype("uint16") + + +def _data_for(order): + """Array laid out in `order` (a subset/permutation of 'tczyx'); axes absent + from `order` are pinned to index 0.""" + base = encoded_tczyx() # [t,c,z,y,x] + sel = tuple(slice(None) if a in order else 0 for a in "tczyx") + sub = base[sel] # absent axes dropped + remaining = [a for a in "tczyx" if a in order] # kept axes, tczyx order + perm = tuple(remaining.index(a) for a in order) # -> requested order + return np.transpose(sub, perm).copy() + + +def write_store(name, order, include_axes=True): + path = os.path.join(HERE, name) + shutil.rmtree(path, ignore_errors=True) + data = _data_for(order) + chunks = tuple(1 if a == "z" else s for a, s in zip(order, data.shape)) + + g = zarr.open_group(path, mode="w", zarr_format=2) + a = g.create_array("0", shape=data.shape, dtype=data.dtype, chunks=chunks, compressors=None) + a[:] = data + + ms = {"version": "0.4", + "datasets": [{"path": "0", + "coordinateTransformations": [{"type": "scale", "scale": [1.0] * len(order)}]}]} + if include_axes: + ms["axes"] = [_AX[a] for a in order] + g.attrs.put({"multiscales": [ms]}) + + meta = json.load(open(os.path.join(path, "0", ".zarray"))) + print("wrote %-22s axes=%-6s shape=%s chunks=%s%s" + % (name, order if include_axes else "(none)", meta["shape"], meta["chunks"], + "" if include_axes else " [no axes -> legacy fallback]")) + + +def write_calibrated(name, order, scale, unit): + """A physically calibrated store: coordinateTransformations 'scale' != 1 and the + space axes carry a unit, so the loader must surface physicalSize*/unit. Drives the + --use-physical-spacing calibration path. Same coordinate encoding as write_store.""" + path = os.path.join(HERE, name) + shutil.rmtree(path, ignore_errors=True) + data = _data_for(order) + chunks = tuple(1 if a == "z" else s for a, s in zip(order, data.shape)) + + g = zarr.open_group(path, mode="w", zarr_format=2) + a = g.create_array("0", shape=data.shape, dtype=data.dtype, chunks=chunks, compressors=None) + a[:] = data + + axes = [] + for ax in order: + e = dict(_AX[ax]) + if e["type"] == "space": + e["unit"] = unit + axes.append(e) + ms = {"version": "0.4", "axes": axes, + "datasets": [{"path": "0", "coordinateTransformations": [{"type": "scale", "scale": scale}]}]} + g.attrs.put({"multiscales": [ms]}) + print("wrote %-22s (calibrated scale=%s unit=%s)" % (name, scale, unit)) + + +def write_multichunk(name, order, chunk_y, chunk_x): + """Same coordinate encoding as write_store, but each Y/X plane is split across a GRID of + chunks instead of being one chunk. This is what real OME-Zarr looks like (chunks are + typically 512x512), and it exercises the multi-tile volumetric assembly: a reader that + fetches only chunk (0,0) returns wrong data for everything past the first chunk.""" + path = os.path.join(HERE, name) + shutil.rmtree(path, ignore_errors=True) + data = _data_for(order) + chunks = tuple((chunk_y if a == "y" else chunk_x if a == "x" else 1) for a in order) + + g = zarr.open_group(path, mode="w", zarr_format=2) + a = g.create_array("0", shape=data.shape, dtype=data.dtype, chunks=chunks, compressors=None) + a[:] = data + + ms = {"version": "0.4", "axes": [_AX[ax] for ax in order], + "datasets": [{"path": "0", + "coordinateTransformations": [{"type": "scale", "scale": [1.0] * len(order)}]}]} + g.attrs.put({"multiscales": [ms]}) + print("wrote %-24s (chunk grid %dx%d over the %dx%d plane)" + % (name, -(-Y // chunk_y), -(-X // chunk_x), Y, X)) + + +def write_v3(name, order, compressors=None): + """OME-Zarr 0.5 = Zarr **v3** store: metadata in zarr.json (not .zarray/.zattrs), the + NGFF block nested under an 'ome' key, chunks under 0/c//... The nyxus loader reads + v3 via the z5 Dataset API (z5 3.x auto-detects v2 vs v3). Uncompressed by default so it + reads with the current blosc+zlib z5 build (zstd-compressed v3 would need -DWITH_ZSTD).""" + path = os.path.join(HERE, name) + shutil.rmtree(path, ignore_errors=True) + data = _data_for(order) + chunks = tuple(1 if a == "z" else s for a, s in zip(order, data.shape)) + + g = zarr.open_group(path, mode="w", zarr_format=3) + a = g.create_array("0", shape=data.shape, dtype=data.dtype, chunks=chunks, compressors=compressors) + a[:] = data + g.attrs["ome"] = {"version": "0.5", + "multiscales": [{"axes": [_AX[ax] for ax in order], + "datasets": [{"path": "0", + "coordinateTransformations": [{"type": "scale", "scale": [1.0] * len(order)}]}]}]} + print("wrote %-24s (Zarr v3, axes=%s, compressors=%s)" % (name, order, compressors)) + + +def write_v3_sharded(name, order): + """Zarr v3 store using the ``sharding_indexed`` codec: several INNER chunks are packed + into one shard object on disk (one file per shard, not per chunk). This is how large v3 + stores (incl. Axle's) actually lay out data. z5 3.x reads it transparently via + ShardedDataset -- the nyxus loader is unchanged, it just calls readSubarray. + + Same TCZYX coordinate encoding as write_v3. Inner chunk (z,y,x)=(1,3,4) -> each 6x8 plane + is a 2x2 grid of inner chunks; shard (1,6,8) packs all 4 inner chunks of a plane into one + shard, so the read must unpack multiple inner chunks from a single shard file.""" + path = os.path.join(HERE, name) + shutil.rmtree(path, ignore_errors=True) + data = _data_for(order) # [t,c,z,y,x], order tczyx + inner = tuple((1 if a == "z" else 3 if a == "y" else 4 if a == "x" else 1) for a in order) + shards = tuple((s if a in ("y", "x", "z") else 1) for a, s in zip(order, data.shape)) + + g = zarr.open_group(path, mode="w", zarr_format=3) + a = g.create_array("0", shape=data.shape, dtype=data.dtype, + chunks=inner, shards=shards, compressors=None) + a[:] = data + g.attrs["ome"] = {"version": "0.5", + "multiscales": [{"axes": [_AX[ax] for ax in order], + "datasets": [{"path": "0", + "coordinateTransformations": [{"type": "scale", "scale": [1.0] * len(order)}]}]}]} + print("wrote %-24s (Zarr v3 SHARDED, inner=%s shard=%s)" % (name, inner, shards)) + + +def write_v3_multishard(name): + """A LARGER Zarr v3 sharded store exercising MULTIPLE SPATIAL SHARDS per (c,t) plane-volume -- + the existing dim5_v3_sharded fixture has exactly one shard per (t,c) (the shard's y/x extent + equals the full plane), so it only ever proves multiple INNER CHUNKS packed into one shard + file. This fixture instead splits each (t,c) volume into a 2x2 grid of shards across Y and X, + so the volumetric assembly must cross shard-FILE boundaries mid-plane (not just inner-chunk + boundaries within one shard) -- closer to how a real, larger Axle v3 store is laid out. + + Own local dims (not the module T/C/Z/Y/X): C=2, T=1, Z=8, Y=24, X=32. + Inner chunk (z,y,x)=(1,6,8) -- z-chunk MUST stay 1: this codebase's OME-Zarr readers + (raw_omezarr.h/omezarr.h) read exactly one Z-plane per tile regardless of what + tileDepth() reports (see this file's module docstring: "chunked one z-slice per + chunk"); a z-chunk>1 silently under-reads (proven while building this fixture -- + a z=2 inner chunk read only its first z-plane and left the rest zero). That is a + separate, real chunking-granularity gap from the shard-FILE-crossing behavior this + fixture targets, so it's dodged here rather than fixed as a drive-by. + Shard (z,y,x)=(8,12,16) -> covers the full Z depth (all 8 inner z-chunks) but HALF of + Y and HALF of X -> a 2x2 shard grid per (c,t), each shard packing 8*(12/6)*(16/8)=32 + inner chunks. Total: C*T*2*2 = 8 shard FILES, and the volumetric assembly must cross + shard-FILE boundaries mid-plane (not just inner-chunk boundaries within one shard). + Same coordinate encoding as the rest of this file: value(x,y,z,c,t) = 1 + ((((t*C+c)*Z+z)*Y+y)*X+x). + """ + path = os.path.join(HERE, name) + shutil.rmtree(path, ignore_errors=True) + + C, T, Z, Y, X = 2, 1, 8, 24, 32 + t, c, z, y, x = np.meshgrid( + np.arange(T), np.arange(C), np.arange(Z), np.arange(Y), np.arange(X), indexing="ij") + data = (1 + ((((t * C + c) * Z + z) * Y + y) * X + x)).astype("uint16") # [t,c,z,y,x] + + inner = (1, 1, 1, 6, 8) + shards = (1, 1, 8, 12, 16) + + g = zarr.open_group(path, mode="w", zarr_format=3) + a = g.create_array("0", shape=data.shape, dtype=data.dtype, + chunks=inner, shards=shards, compressors=None) + a[:] = data + g.attrs["ome"] = {"version": "0.5", + "multiscales": [{"axes": [_AX[ax] for ax in "tczyx"], + "datasets": [{"path": "0", + "coordinateTransformations": [{"type": "scale", "scale": [1.0] * 5}]}]}]} + print("wrote %-24s (Zarr v3 MULTI-SHARD: C=%d T=%d Z=%d Y=%d X=%d, inner=%s shard=%s, %dx%d shard grid per c,t)" + % (name, C, T, Z, Y, X, inner, shards, -(-Y // shards[3]), -(-X // shards[4]))) + + +def write_v3_zchunked(name): + """A Zarr v3 store with a MULTI-PLANE Z chunk (chunk z-extent > 1), unsharded -- isolates the + Z-chunking behavior from shard-file addressing (see write_v3_multishard for that). Every other + fixture in this file chunks exactly one Z-slice per chunk (see the module docstring); this one + deliberately doesn't, to catch readers that silently under-read past the first Z-plane of a + multi-plane chunk. + + Own local dims: C=2, T=1, Z=7, Y=6, X=8. Chunk (z,y,x)=(3,6,8) -> 3 Z-chunks of depth 3,3,1 -- + the UNEVEN split (Z=7 not a multiple of 3) also exercises the partial last-chunk clamp. + Same coordinate encoding as the rest of this file: value(x,y,z,c,t) = 1 + ((((t*C+c)*Z+z)*Y+y)*X+x). + """ + path = os.path.join(HERE, name) + shutil.rmtree(path, ignore_errors=True) + + C, T, Z, Y, X = 2, 1, 7, 6, 8 + t, c, z, y, x = np.meshgrid( + np.arange(T), np.arange(C), np.arange(Z), np.arange(Y), np.arange(X), indexing="ij") + data = (1 + ((((t * C + c) * Z + z) * Y + y) * X + x)).astype("uint16") # [t,c,z,y,x] + + chunks = (1, 1, 3, 6, 8) + + g = zarr.open_group(path, mode="w", zarr_format=3) + a = g.create_array("0", shape=data.shape, dtype=data.dtype, chunks=chunks, compressors=None) + a[:] = data + g.attrs["ome"] = {"version": "0.5", + "multiscales": [{"axes": [_AX[ax] for ax in "tczyx"], + "datasets": [{"path": "0", + "coordinateTransformations": [{"type": "scale", "scale": [1.0] * 5}]}]}]} + print("wrote %-24s (Zarr v3 Z-CHUNKED: C=%d T=%d Z=%d Y=%d X=%d, chunk=%s -> Z-chunk depths 3,3,1)" + % (name, C, T, Z, Y, X, chunks)) + + +def write_label_mask(name): + """P4/N3: a single-channel ZYX label mask (mostly background, a few small ROIs), the + OME-Zarr twin of dim3_mask.ome.tif. Pairs with a T>1 Zarr intensity to exercise the + 1-mask : N-timeframe reuse on the Zarr path (the crash's positive twin).""" + path = os.path.join(HERE, name) + shutil.rmtree(path, ignore_errors=True) + m = np.zeros((Z, Y, X), "uint32") + m[:, 1:5, 1:7] = 1 # one ROI (label 1), interior bbox + g = zarr.open_group(path, mode="w", zarr_format=2) + a = g.create_array("0", shape=m.shape, dtype=m.dtype, chunks=(1, Y, X), compressors=None) + a[:] = m + g.attrs.put({"multiscales": [{"version": "0.4", "axes": [_AX[c] for c in "zyx"], + "datasets": [{"path": "0", "coordinateTransformations": [ + {"type": "scale", "scale": [1.0, 1.0, 1.0]}]}]}]}) + print("wrote %-24s (ZYX label mask, ROI voxels=%d)" % (name, int(m.sum()))) + + +def write_v3_badcodec(name): + """N1: a Zarr v3 store whose zarr.json declares a codec z5 does not support. z5's + readV3CodecsFromJson throws 'unsupported zarr v3 codec' at openDataset (metadata parse, + before any data read), so the loader must surface a clean error, not crash. Written + uncompressed, then the codec list is patched to an unknown name.""" + path = os.path.join(HERE, name) + shutil.rmtree(path, ignore_errors=True) + data = _data_for("tczyx") + chunks = tuple(1 if a == "z" else s for a, s in zip("tczyx", data.shape)) + g = zarr.open_group(path, mode="w", zarr_format=3) + a = g.create_array("0", shape=data.shape, dtype=data.dtype, chunks=chunks, compressors=None) + a[:] = data + g.attrs["ome"] = {"version": "0.5", "multiscales": [{"axes": [_AX[c] for c in "tczyx"], + "datasets": [{"path": "0", "coordinateTransformations": [ + {"type": "scale", "scale": [1.0] * 5}]}]}]} + # patch the array's zarr.json to reference an unsupported codec + zj_path = os.path.join(path, "0", "zarr.json") + zj = json.load(open(zj_path)) + zj["codecs"] = [{"name": "bytes", "configuration": {"endian": "little"}}, + {"name": "bogus_codec_xyz", "configuration": {}}] + json.dump(zj, open(zj_path, "w")) + print("wrote %-24s (v3 with unsupported codec 'bogus_codec_xyz')" % name) + + +def write_bad(name, shape, axes): + """Write a deliberately malformed store (its 'axes' disagrees with the array) + so the loader must reject it cleanly, not crash.""" + path = os.path.join(HERE, name) + shutil.rmtree(path, ignore_errors=True) + g = zarr.open_group(path, mode="w", zarr_format=2) + chunks = tuple(1 if i == len(shape) - 3 else s for i, s in enumerate(shape)) + a = g.create_array("0", shape=shape, dtype="uint16", chunks=chunks, compressors=None) + a[:] = 0 + g.attrs.put({"multiscales": [{"version": "0.4", "axes": axes, + "datasets": [{"path": "0", + "coordinateTransformations": [{"type": "scale", "scale": [1.0] * len(axes)}]}]}]}) + print("wrote %-24s shape=%s axes=%d (MALFORMED)" % (name, list(shape), len(axes))) + + +def main(): + # All 6 orderings of {t,c,z} before y,x -- the complete legal OME axis-order set. + for o in ["tczyx", "tzcyx", "ctzyx", "cztyx", "ztcyx", "zctyx"]: + write_store("dim5.ome.zarr" if o == "tczyx" else "dim5_%s.ome.zarr" % o, o) + # 4D (rank-4 coverage): one time-only, one channel-only. + write_store("dim4_tzyx.ome.zarr", "tzyx") + write_store("dim4_czyx.ome.zarr", "czyx") + # lower rank + fallback + write_store("dim3_zyx.ome.zarr", "zyx") + write_store("dim2_yx.ome.zarr", "yx") + write_store("dim5_noaxes.ome.zarr", "tczyx", include_axes=False) + # physically calibrated: anisotropic voxels (z 4x thicker than x/y after normalization). + # order tczyx -> scale [t,c,z,y,x] = [1,1,2.0,0.5,0.5] -> physX=0.5 physY=0.5 physZ=2.0 um. + write_calibrated("dim5_calibrated.ome.zarr", "tczyx", [1.0, 1.0, 2.0, 0.5, 0.5], "micrometer") + # same physical spacing as dim5_calibrated, declared in NANOMETER instead of micrometer + # (2.0um=2000nm, 0.5um=500nm) -> the loader must canonicalize to the SAME physX/Y/Z/unit + # as dim5_calibrated, proving unit conversion (not just passthrough) actually happens. + write_calibrated("dim5_calibrated_nm.ome.zarr", "tczyx", [1.0, 1.0, 2000.0, 500.0, 500.0], "nanometer") + # OME-Zarr 0.5 (Zarr v3): same 5D TCZYX encoding, but zarr.json metadata + 'ome'-wrapped NGFF + write_v3("dim5_v3.ome.zarr", "tczyx") + # Zarr v3 with the BLOSC codec (common alongside zstd in real v3 stores). z5 decodes the + # bytes+blosc v3 pipeline when built WITH_BLOSC (already required for OME-Zarr). + write_v3("dim5_v3_blosc.ome.zarr", "tczyx", + compressors=[BloscCodec(cname="lz4", clevel=5, shuffle="shuffle")]) + # Larger multi-shard store: a 2x2 grid of SHARD FILES per (c,t), not just multiple inner + # chunks within one shard -- exercises crossing shard-file boundaries mid-plane. + write_v3_multishard("dim5_v3_multishard.ome.zarr") + # multi-plane Z chunk (chunk z-extent 3, uneven split 3+3+1) -- isolated Z-chunking check + write_v3_zchunked("dim5_v3_zchunked.ome.zarr") + # plane split across a 2x2 chunk grid (3x4 chunks over the 6x8 plane) -> multi-tile assembly + write_multichunk("dim5_multichunk.ome.zarr", "tczyx", 3, 4) + # PARTIAL edge chunks: chunk (4,5) does NOT divide the 6x8 plane, so the last row-chunk is + # 2 tall and the last col-chunk is 3 wide. Exercises the validH/validW seam clamp in the + # volumetric assembly, which every exact-multiple fixture (above) leaves untested. + write_multichunk("dim5_oddchunk.ome.zarr", "tczyx", 4, 5) + # --- illegal / adversarial: must be rejected cleanly, not crash --- + # 3D array but 'axes' declares 5 entries -> indexing the shape by axis role OOBs + write_bad("bad_axes_count.ome.zarr", (Z, Y, X), [_AX[k] for k in ("t", "c", "z", "y", "x")]) + # axes present but none labeled x/y -> X/Y unresolvable + write_bad("bad_no_xy.ome.zarr", (Z, Y, X), + [{"name": "z", "type": "space"}, {"name": "a", "type": "space"}, {"name": "b", "type": "space"}]) + + +if __name__ == "__main__": + main() diff --git a/tests/python/test_covered_intensity_range_regression.py b/tests/python/test_covered_intensity_range_regression.py new file mode 100644 index 00000000..11f5a411 --- /dev/null +++ b/tests/python/test_covered_intensity_range_regression.py @@ -0,0 +1,84 @@ +"""Regression coverage for COVERED_IMAGE_INTENSITY_RANGE (2D) / 3COVERED_IMAGE_INTENSITY_RANGE (3D) +on a genuinely SEGMENTED image -- a mask strictly smaller than the full image/volume, leaving +off-mask background voxels the feature's "whole slide" baseline must still see. + +No prior test exercised this: every existing fixture asserting this feature was either whole-slide +(mask == full image, so there is no off-mask region to get wrong) or happened to produce the +degenerate value 1.0/near-1.0 by coincidence, not by design -- masking the bug this test targets. + +Bug (found while investigating a stale vetting-registry note): gatherRoisMetrics_2_slideprops / +gatherRoisMetrics_2_slideprops_3D (slideprops.cpp) computed the "pre-ROI" (whole-slide) min/max +intensity by skipping off-mask voxels, so for any segmented image the "whole slide" baseline +silently collapsed to the ROI's own min/max -- COVERED_IMAGE_INTENSITY_RANGE was always exactly 1 +whenever a real mask was in play. Fixed by reading every voxel's intensity for the baseline while +still gating ROI geometry (labels/area/AABB) on the mask. + +This test computes the expected ratio independently from the raw arrays (not a canned golden), so +it can't go stale the same way. +""" +import os +import pathlib +import numpy as np +import pytest + +import nyxus + +tifffile = pytest.importorskip("tifffile") + + +def _ellipse_mask(shape, radii): + idx = np.meshgrid(*[np.arange(s) for s in shape], indexing="ij") + centers = [(s - 1) / 2.0 for s in shape] + acc = sum(((idx[d] - centers[d]) / radii[d]) ** 2 for d in range(len(shape))) + return (acc <= 1.0).astype(np.uint32) + + +def test_covered_intensity_range_2d_segmented(tmp_path): + Y, X = 90, 90 + y = np.arange(Y)[:, None] + x = np.arange(X)[None, :] + inten = (1 + (x % 256) + (y % 200) * 256).astype(np.uint32) + mask = _ellipse_mask((Y, X), (Y / 3.0, X / 3.0)) + + intdir, segdir = tmp_path / "int", tmp_path / "seg" + intdir.mkdir(); segdir.mkdir() + tifffile.imwrite(str(intdir / "img.tif"), inten) + tifffile.imwrite(str(segdir / "img.tif"), mask) + + n = nyxus.Nyxus(["*ALL_INTENSITY*"]) + df = n.featurize_directory(str(intdir) + os.sep, str(segdir) + os.sep) + + roi = inten[mask.astype(bool)] + expected = (float(roi.max()) - float(roi.min())) / (float(inten.max()) - float(inten.min())) + + assert df["COVERED_IMAGE_INTENSITY_RANGE"].iloc[0] == pytest.approx(expected, rel=1e-9) + # sanity: this fixture's mask leaves real background outside it, so the bug (ratio == 1.0) + # would be plainly wrong here -- guard against a future fixture change silently losing that. + assert expected < 0.99 + + +def test_covered_intensity_range_3d_segmented(tmp_path): + Z, Y, X = 8, 90, 90 + z = np.arange(Z)[:, None, None] + y = np.arange(Y)[None, :, None] + x = np.arange(X)[None, None, :] + inten = (1 + (x % 256) + (y % 200) * 256 + z * 10000).astype(np.uint32) + inten = np.broadcast_to(inten, (Z, Y, X)).astype(np.uint32) + mask = _ellipse_mask((Z, Y, X), (Z / 2.0, Y / 3.0, X / 3.0)) + + intp = tmp_path / "vol_int.ome.tif" + segp = tmp_path / "vol_seg.ome.tif" + tifffile.imwrite(str(intp), inten, metadata={"axes": "ZYX"}) + tifffile.imwrite(str(segp), mask, metadata={"axes": "ZYX"}) + + # Well above this small fixture's footprint, but under the RAM a host actually has free: + # Nyxus rejects a limit above available RAM and keeps the previous process-global value, + # so a limit near a machine's total RAM passes on a workstation and fails on a CI runner. + n = nyxus.Nyxus3D(["*3D_ALL_INTENSITY*"], ram_limit=1000) + df = n.featurize_files([str(intp)], [str(segp)], False) + + roi = inten[mask.astype(bool)] + expected = (float(roi.max()) - float(roi.min())) / (float(inten.max()) - float(inten.min())) + + assert df["3COVERED_IMAGE_INTENSITY_RANGE"].iloc[0] == pytest.approx(expected, rel=1e-9) + assert expected < 0.99 diff --git a/tests/python/test_nyxus.py b/tests/python/test_nyxus.py index 2b03a95e..deb5fc87 100644 --- a/tests/python/test_nyxus.py +++ b/tests/python/test_nyxus.py @@ -426,10 +426,15 @@ def test_in_memory_3d(self): means = cpu_features.mean(numeric_only=True) - mean_values = means.tolist() + # Drop the non-feature index/calibration columns by NAME so this stays correct as + # the output schema grows. (This used to pop the 2 leftmost columns positionally, + # which silently averaged c_index/phys_x/y/z into the first feature once the + # channel-index and physical-spacing columns were added.) + for _c in ("ROI_label", "time", "t_index", "c_index", "phys_x", "phys_y", "phys_z"): + if _c in means.index: + means = means.drop(_c) - mean_values.pop(0) # get rid of leftmost ROI label column - mean_values.pop(0) # get rid of leftmost time column + mean_values = means.tolist() averaged_results = [] diff --git a/tests/python/test_ooc_mechanics.py b/tests/python/test_ooc_mechanics.py new file mode 100644 index 00000000..2a6d6416 --- /dev/null +++ b/tests/python/test_ooc_mechanics.py @@ -0,0 +1,324 @@ +"""Mechanics tests for the out-of-core (oversized-ROI) path. + +An ROI whose memory footprint reaches ram_limit is streamed through the disk-backed +OutOfRamPixelCloud instead of an in-memory pixel vector, and its features are computed by +the osized_calculate path. These tests exercise that plumbing (not an external oracle): + +- test_ooc_2d_matches_in_ram: the out-of-core path must produce the same intensity features + as the in-RAM path on the same file pair. +- test_ooc_montage_oversized_fails_loudly: the in-memory montage path has no out-of-core + support and must raise on an oversized ROI rather than emit a silent all-zero row. + +ram_limit is a process-global in Nyxus, so each test sets it explicitly on both sides to stay +order-independent. +""" +import os +import pathlib +import numpy as np +import pytest + +import nyxus + +tifffile = pytest.importorskip("tifffile") + +DATA_NIFTI = pathlib.Path(__file__).resolve().parent.parent / "data" / "nifti" + +# The "large" ram_limit for the in-RAM side of these comparisons. It has to clear the biggest +# fixture footprint here (a few MB) while staying under the RAM the host actually has free: +# Nyxus rejects a limit above available RAM and keeps the previous value, and since ram_limit is +# a process-global, a rejected setting silently leaves an earlier test's 0/1 MB limit in place +# and the in-RAM side then comes back oversized. A limit near a machine's total RAM therefore +# passes on a workstation and fails on a CI runner. +RAM_LIMIT_LARGE_MB = 1000 + + +def _make_pair(tmp_path): + # Deterministic, non-degenerate: per-row offset + per-column gradient. 500x500 gives an + # in-memory footprint well above 1 MB, so ram_limit=1 forces the out-of-core path. + Y, X = 500, 500 + xg = (np.arange(X) % 256).astype(np.uint32) + yg = ((np.arange(Y) % 200) * 256).astype(np.uint32) + inten = (1 + xg[None, :] + yg[:, None]).astype(np.uint32) + mask = np.ones((Y, X), np.uint32) # a single ROI covering the whole image + intdir = tmp_path / "int" + segdir = tmp_path / "seg" + intdir.mkdir() + segdir.mkdir() + tifffile.imwrite(str(intdir / "img.tif"), inten) + tifffile.imwrite(str(segdir / "img.tif"), mask) + return str(intdir) + os.sep, str(segdir) + os.sep + + +def _feature_cols(df): + cols = [ + c + for c in df.select_dtypes(include=[np.number]).columns + if c not in ("ROI_label", "t_index", "c_index") + ] + return cols, df[cols].to_numpy(dtype=float).ravel() + + +def test_ooc_2d_matches_in_ram(tmp_path): + intdir, segdir = _make_pair(tmp_path) + feats = ["*ALL_INTENSITY*"] + + n_ram = nyxus.Nyxus(feats) + n_ram.set_params(ram_limit=RAM_LIMIT_LARGE_MB) # large -> in-RAM (trivial); explicit so test is order-independent + df_ram = n_ram.featurize_directory(intdir, segdir) + + n_ooc = nyxus.Nyxus(feats) + n_ooc.set_params(ram_limit=1) # 1 MB -> forces the oversized / out-of-core path + df_ooc = n_ooc.featurize_directory(intdir, segdir) + + cols, a = _feature_cols(df_ram) + _, b = _feature_cols(df_ooc) + assert a.size > 0 and a.shape == b.shape + + bad = [ + (c, p, q) + for c, p, q in zip(cols, a, b) + if abs(p - q) > 1e-6 * max(abs(p), abs(q), 1.0) + 1e-9 + ] + assert not bad, "out-of-core intensity features diverge from in-RAM: %r" % (bad[:8],) + + +def _make_volume_pair(tmp_path): + # A 3D OME-TIFF volume (one IFD per Z) + a whole-volume mask. Z*Y*X = 8*90*90 = 64800 + # voxels; the in-memory 3D footprint is well over 1 MB, so ram_limit=1 forces the oversized + # (out-of-core) volumetric path. Intensity is a deterministic, non-degenerate function of + # (x,y,z) so every 3D intensity feature is meaningful. + Z, Y, X = 8, 90, 90 + z = np.arange(Z)[:, None, None] + y = np.arange(Y)[None, :, None] + x = np.arange(X)[None, None, :] + inten = (1 + (x % 256) + (y % 200) * 256 + z * 10000).astype(np.uint32) + inten = np.broadcast_to(inten, (Z, Y, X)).astype(np.uint32) + mask = np.ones((Z, Y, X), np.uint32) # a single ROI covering the whole volume + intp = tmp_path / "vol_int.ome.tif" + segp = tmp_path / "vol_seg.ome.tif" + tifffile.imwrite(str(intp), inten, metadata={"axes": "ZYX"}) + tifffile.imwrite(str(segp), mask, metadata={"axes": "ZYX"}) + return str(intp), str(segp) + + +def test_ooc_3d_matches_in_ram(tmp_path): + """The 3D out-of-core path (voxel cloud streamed to disk, keeping z) must produce the same + 3D intensity AND surface/morphology features as the in-RAM path on the same volume pair.""" + intp, segp = _make_volume_pair(tmp_path) + feats = ["*3D_ALL_INTENSITY*", "*3D_ALL_MORPHOLOGY*"] + + # Nyxus3D takes ram_limit in the constructor (its set_params does not expose it) + n_ram = nyxus.Nyxus3D(feats, ram_limit=RAM_LIMIT_LARGE_MB) # large -> in-RAM (trivial) + df_ram = n_ram.featurize_files([intp], [segp], False) + + n_ooc = nyxus.Nyxus3D(feats, ram_limit=1) # 1 MB -> forces the oversized / out-of-core volumetric path + df_ooc = n_ooc.featurize_files([intp], [segp], False) + + cols, a = _feature_cols(df_ram) + _, b = _feature_cols(df_ooc) + assert a.size > 0 and a.shape == b.shape + + bad = [ + (c, p, q) + for c, p, q in zip(cols, a, b) + if abs(p - q) > 1e-6 * max(abs(p), abs(q), 1.0) + 1e-9 + ] + assert not bad, "3D out-of-core intensity features diverge from in-RAM: %r" % (bad[:8],) + + +def _make_volume_pair_partial(tmp_path): + # Same intensity field as _make_volume_pair, but a non-cuboid (ellipsoid) mask that leaves + # background voxels INSIDE the ROI's bounding box. Out-of-core paths that build a grey-level + # LUT from the whole binned cube (mask + background) -- e.g. matlab binning maps the raw-0 + # background to a nonzero bin that must still appear in the LUT -- are only exercised when the + # bbox actually contains background; a whole-volume mask (hasBackground == false) never does. + Z, Y, X = 8, 90, 90 + z = np.arange(Z)[:, None, None] + y = np.arange(Y)[None, :, None] + x = np.arange(X)[None, None, :] + inten = (1 + (x % 256) + (y % 200) * 256 + z * 10000).astype(np.uint32) + inten = np.broadcast_to(inten, (Z, Y, X)).astype(np.uint32) + zz, yy, xx = np.meshgrid(np.arange(Z), np.arange(Y), np.arange(X), indexing="ij") + cz, cy, cx = (Z - 1) / 2.0, (Y - 1) / 2.0, (X - 1) / 2.0 + rz, ry, rx = Z / 2.0, Y / 3.0, X / 3.0 + inside = (((zz - cz) / rz) ** 2 + ((yy - cy) / ry) ** 2 + ((xx - cx) / rx) ** 2) <= 1.0 + mask = inside.astype(np.uint32) + intp = tmp_path / "vol_int_partial.ome.tif" + segp = tmp_path / "vol_seg_partial.ome.tif" + tifffile.imwrite(str(intp), inten, metadata={"axes": "ZYX"}) + tifffile.imwrite(str(segp), mask, metadata={"axes": "ZYX"}) + return str(intp), str(segp) + + +def _ooc_vs_ram_3d(tmp_path, feats, pair_fn=_make_volume_pair): + intp, segp = pair_fn(tmp_path) + n_ram = nyxus.Nyxus3D(feats, ram_limit=RAM_LIMIT_LARGE_MB) + df_ram = n_ram.featurize_files([intp], [segp], False) + n_ooc = nyxus.Nyxus3D(feats, ram_limit=1) + df_ooc = n_ooc.featurize_files([intp], [segp], False) + cols, a = _feature_cols(df_ram) + _, b = _feature_cols(df_ooc) + assert a.size > 0 and a.shape == b.shape + bad = [ + (c, p, q) + for c, p, q in zip(cols, a, b) + if abs(p - q) > 1e-6 * max(abs(p), abs(q), 1.0) + 1e-9 + ] + assert not bad, "3D out-of-core features diverge from in-RAM: %r" % (bad[:8],) + + +def test_ooc_3d_glcm_matches_in_ram(tmp_path): + """3D GLCM out-of-core (13 co-occurrence matrices built over a streaming 2-plane window) + must match the in-RAM path.""" + _ooc_vs_ram_3d(tmp_path, ["*3D_GLCM*"]) + + +def test_ooc_3d_gldm_matches_in_ram(tmp_path): + """3D GLDM out-of-core (dependence matrix built over a streaming 3-plane window) must match + the in-RAM path.""" + _ooc_vs_ram_3d(tmp_path, ["*3D_GLDM*"]) + + +def test_ooc_3d_ngldm_matches_in_ram(tmp_path): + """3D NGLDM out-of-core (3-plane window, interior scan) must match the in-RAM path.""" + _ooc_vs_ram_3d(tmp_path, ["*3D_NGLDM*"]) + + +def test_ooc_3d_ngtdm_matches_in_ram(tmp_path): + """3D NGTDM out-of-core (radius-window neighbourhood averages) must match the in-RAM path.""" + _ooc_vs_ram_3d(tmp_path, ["*3D_NGTDM*"]) + + +def test_ooc_3d_glrlm_matches_in_ram(tmp_path): + """3D GLRLM out-of-core (in-plane runs reuse gather_rl_zones on a per-plane depth-1 cube; + cross-plane runs use a 2-plane carry) must match the in-RAM path.""" + _ooc_vs_ram_3d(tmp_path, ["*3D_GLRLM*"]) + + +def test_ooc_3d_glszm_matches_in_ram(tmp_path): + """3D GLSZM out-of-core (streaming 26-connectivity connected-component labeling via a + growable union-find over a 2-plane window) must match the in-RAM path.""" + _ooc_vs_ram_3d(tmp_path, ["*3D_GLSZM*"]) + + +def test_ooc_3d_gldzm_matches_in_ram(tmp_path): + """3D GLDZM out-of-core (streaming 6-connectivity connected-component labeling with a + min-distance-to-border metric per zone) must match the in-RAM path.""" + _ooc_vs_ram_3d(tmp_path, ["*3D_GLDZM*"]) + + +def test_ooc_3d_wholevolume_streams_oob(tmp_path): + """The WHOLE-VOLUME (single_roi) 3D path -- Nyxus3D.featurize_directory(dir, dir, ...), i.e. + label_dir==intensity_dir -- used to have NO out-of-core support at all: an oversized whole + volume failed loudly (workflow_3d_whole.cpp). It now streams via the same + populate_3d_voxel_cloud/run_3d_ooc_features primitives as the segmented path. This is the + Python-facing equivalent of the gtest TEST_3D_WHOLEVOLUME_OVERSIZED_STREAMS_OOC: force + oversized with ram_limit=0 (every footprint >= 0 is always true) and compare against an + in-RAM run of the identical volume.""" + Z, Y, X = 8, 90, 90 + z = np.arange(Z)[:, None, None] + y = np.arange(Y)[None, :, None] + x = np.arange(X)[None, None, :] + inten = (1 + (x % 256) + (y % 200) * 256 + z * 10000).astype(np.uint32) + inten = np.broadcast_to(inten, (Z, Y, X)).astype(np.uint32) + voldir = tmp_path / "wv" + voldir.mkdir() + tifffile.imwrite(str(voldir / "vol.ome.tif"), inten, metadata={"axes": "ZYX"}) + + feats = ["*3D_ALL_INTENSITY*", "*3D_ALL_MORPHOLOGY*", "*3D_GLCM*"] + + n_ram = nyxus.Nyxus3D(feats, ram_limit=RAM_LIMIT_LARGE_MB) + df_ram = n_ram.featurize_directory(str(voldir), str(voldir), ".*") + + n_ooc = nyxus.Nyxus3D(feats, ram_limit=0) # every footprint >= 0 -> always oversized + df_ooc = n_ooc.featurize_directory(str(voldir), str(voldir), ".*") + + cols, a = _feature_cols(df_ram) + _, b = _feature_cols(df_ooc) + assert a.size > 0 and a.shape == b.shape + + bad = [ + (c, p, q) + for c, p, q in zip(cols, a, b) + if abs(p - q) > 1e-6 * max(abs(p), abs(q), 1.0) + 1e-9 + ] + assert not bad, "whole-volume out-of-core features diverge from in-RAM: %r" % (bad[:8],) + + +def _make_volume_pair_blank(tmp_path): + # A degenerate ROI (constant intensity everywhere) at the same size as _make_volume_pair, so it + # is still classified oversized at ram_limit=1 -- exercises each osized_calculate's early-return + # "blank ROI" guard (aux_min==aux_max) via the out-of-core path, which no other fixture reaches + # (every other fixture here varies intensity). Must not crash or hang, and must match the + # in-RAM path's degenerate-ROI output (STNGS_NAN defaults to plain 0.0 in this build, not IEEE + # NaN, so the existing tolerance-based comparison in _ooc_vs_ram_3d applies unchanged). + Z, Y, X = 8, 90, 90 + inten = np.full((Z, Y, X), 42, dtype=np.uint32) + mask = np.ones((Z, Y, X), np.uint32) + intp = tmp_path / "vol_int_blank.ome.tif" + segp = tmp_path / "vol_seg_blank.ome.tif" + tifffile.imwrite(str(intp), inten, metadata={"axes": "ZYX"}) + tifffile.imwrite(str(segp), mask, metadata={"axes": "ZYX"}) + return str(intp), str(segp) + + +def test_ooc_3d_blank_matches_in_ram(tmp_path): + """A degenerate (constant-intensity) oversized 3D ROI must not crash and must match the in-RAM + path's degenerate-ROI output for intensity, surface, and all seven texture families.""" + _ooc_vs_ram_3d( + tmp_path, + ["*3D_ALL_INTENSITY*", "*3D_ALL_MORPHOLOGY*", "*3D_GLCM*", "*3D_GLDM*", "*3D_NGLDM*", "*3D_NGTDM*", "*3D_GLRLM*", "*3D_GLSZM*", "*3D_GLDZM*"], + pair_fn=_make_volume_pair_blank, + ) + + +@pytest.mark.skipif( + not (DATA_NIFTI / "compat_int" / "compat_int_mri.nii").exists(), + reason="NIfTI compat fixtures not present in tests/data/nifti", +) +def test_ooc_3d_nifti_unsupported_format_fails_loudly(): + """NIfTI delivers the whole X*Y*Z*T volume in one read (ImageLoader::stream_volume_planes + declines it, since it isn't plane-by-plane), so an oversized ROI through this loader cannot + stream out-of-core. Must fail loudly with an actionable message rather than crash, hang, or + silently emit a wrong/zero row. ram_limit=0 forces every ROI oversized regardless of its actual + footprint (roiFootprint >= 0 is always true), so this doesn't depend on the fixture's ROI size.""" + intp = str(DATA_NIFTI / "compat_int" / "compat_int_mri.nii") + segp = str(DATA_NIFTI / "compat_seg" / "compat_seg_liver.nii") + n = nyxus.Nyxus3D(["*3D_ALL_INTENSITY*"], ram_limit=0) + with pytest.raises(Exception, match="not supported for this input format"): + n.featurize_files([intp], [segp], False) + + +def test_ooc_3d_partial_mask_matches_in_ram(tmp_path): + """Same equivalence check as the whole-volume tests above, but with a non-cuboid mask so the + ROI bbox contains background voxels -- exercises grey-level LUT construction (must include the + background bin, matching each feature's whole-cube-based calculate()) that a whole-volume mask + never triggers. Covers intensity/surface plus all four texture families in one pass.""" + _ooc_vs_ram_3d( + tmp_path, + ["*3D_ALL_INTENSITY*", "*3D_ALL_MORPHOLOGY*", "*3D_GLCM*", "*3D_GLDM*", "*3D_NGLDM*", "*3D_NGTDM*", "*3D_GLRLM*", "*3D_GLSZM*", "*3D_GLDZM*"], + pair_fn=_make_volume_pair_partial, + ) + + +def test_ooc_montage_oversized_fails_loudly(): + """The in-memory (montage) path has no out-of-core support, so an ROI whose footprint + reaches ram_limit must fail loudly rather than emit a silent all-zero feature row.""" + Y, X = 300, 300 + xg = (np.arange(X) % 256).astype(np.uint32) + yg = ((np.arange(Y) % 200) * 256).astype(np.uint32) + inten = (1 + xg[None, :] + yg[:, None]).astype(np.uint32) + mask = np.ones((Y, X), np.uint32) + + # sanity: with a large ram_limit the montage path succeeds and is non-zero. Set it + # explicitly (nyxus ram_limit is process-global) so this does not depend on test order. + ok = nyxus.Nyxus(["*ALL_INTENSITY*"]) + ok.set_params(ram_limit=RAM_LIMIT_LARGE_MB) + df_ok = ok.featurize(inten, mask, intensity_names=["I"], label_names=["M"]) + assert df_ok["MEAN"].iloc[0] > 0 + + # ram_limit=1 makes the single ROI oversized -> must raise, not return zeros + n = nyxus.Nyxus(["*ALL_INTENSITY*"]) + n.set_params(ram_limit=1) + with pytest.raises(Exception, match="oversized"): + n.featurize(inten, mask, intensity_names=["I"], label_names=["M"]) diff --git a/tests/python/test_physical_calibration_regression.py b/tests/python/test_physical_calibration_regression.py new file mode 100644 index 00000000..ef88e2b3 --- /dev/null +++ b/tests/python/test_physical_calibration_regression.py @@ -0,0 +1,62 @@ +"""End-to-end regression coverage for physical-unit canonicalization through the real Python +API. No prior test exercised phys_x/y/z/phys_unit output at all (not just the unit-conversion +gap) -- this closes both gaps at once: proves the columns come back, and proves a +non-micrometer-declared store is converted to the same values as an equivalent +micrometer-declared store. + +Bug fixed: physX/Y/Z were reported verbatim in whatever unit a file declared (nanometer, +millimeter, ...) with no conversion, so two calibrated files using different units produced +numerically incomparable phys_x/y/z columns. Fixed by canonicalizing every axis to micrometer +at OME metadata parse time (ome_axes.h's canonicalize_to_micrometer), using each axis's own +declared unit (a Z axis in a different unit than X/Y previously mislabeled its own value under +X/Y's unit string). + +Uses OME-TIFF whole-volume mode (featurize_directory(dir, dir, ...), intensity_dir==label_dir) +-- the same pattern test_ooc_mechanics.py's whole-volume tests already exercise successfully. +Each fixture is copied into its OWN isolated tmp_path subdirectory: featurize_directory matches +every entry under the given dir against file_pattern, and tests/data/ometiff/ has dozens of +other fixtures that would otherwise also match ".*". +""" +import pathlib +import shutil + +import nyxus + +DATA_OMETIFF = pathlib.Path(__file__).resolve().parent.parent / "data" / "ometiff" + + +def test_ometiff_micrometer_and_nanometer_stores_agree(tmp_path): + cal_um = DATA_OMETIFF / "dim5_calibrated.ome.tif" + cal_nm = DATA_OMETIFF / "dim5_calibrated_nm.ome.tif" + assert cal_um.exists() and cal_nm.exists() + + um_dir, nm_dir = tmp_path / "um", tmp_path / "nm" + um_dir.mkdir(); nm_dir.mkdir() + shutil.copy(cal_um, um_dir / cal_um.name) + shutil.copy(cal_nm, nm_dir / cal_nm.name) + + # NOTE: deliberately NOT use_physical_spacing=True -- phys_x/y/z/phys_unit are emitted + # regardless of that flag (it only controls whether ANISOTROPIC RESAMPLING is applied), + # and enabling it here hit an unrelated hang in the resampling path for this fixture's + # multi-channel/timeframe + anisotropic (4x Z:XY) combination. Out of scope for this fix + # (units, not resampling); flagging separately rather than chasing it here. + n = nyxus.Nyxus3D(["*3D_ALL_INTENSITY*"]) + df_um = n.featurize_directory(str(um_dir), str(um_dir), ".*") + df_nm = n.featurize_directory(str(nm_dir), str(nm_dir), ".*") + + for df in (df_um, df_nm): + assert "phys_unit" in df.columns + assert {"phys_x", "phys_y", "phys_z"} <= set(df.columns) + + # dim5_calibrated: physX/Y=0.5, physZ=2.0 micrometer. + # dim5_calibrated_nm: same spacing declared in nanometer (X/Y) and millimeter (Z) -- must + # canonicalize to the SAME values, not the raw declared numbers. + assert (df_um["phys_x"] == 0.5).all() + assert (df_um["phys_y"] == 0.5).all() + assert (df_um["phys_z"] == 2.0).all() + assert (df_um["phys_unit"] == "micrometer").all() + + assert (df_nm["phys_x"] == df_um["phys_x"]).all() + assert (df_nm["phys_y"] == df_um["phys_y"]).all() + assert (df_nm["phys_z"] == df_um["phys_z"]).all() + assert (df_nm["phys_unit"] == "micrometer").all() diff --git a/tests/test_3d_coverage_common.h b/tests/test_3d_coverage_common.h index bd54f0dd..912ee066 100644 --- a/tests/test_3d_coverage_common.h +++ b/tests/test_3d_coverage_common.h @@ -143,14 +143,14 @@ static Computed3DFeatureValues build_computed_3d_feature_values() e.dataset.update_dataset_props_extrema(); clear_slide_rois(e.uniqueLabels, e.roiData); - if (!gatherRoisMetrics_3D(e, 0, ipath, mpath, 0)) + if (!gatherRoisMetrics_3D(e, 0, ipath, mpath, 0/*t_index*/, 0/*channel*/)) { out.setup_error = "gatherRoisMetrics_3D failed for 3D coverage phantom"; return false; } std::vector batch = { label }; - if (!scanTrivialRois_3D(e, batch, ipath, mpath, 0)) + if (!scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)) { out.setup_error = "scanTrivialRois_3D failed for 3D coverage phantom"; return false; @@ -307,7 +307,7 @@ static const std::map>& unvetted_3d_local_regre { "3COMPACTNESS1", { 0.010537043861899255 } }, { "3COMPACTNESS2", { 0.039449347281835329 } }, { "3COV", { 0.29486207043456802 } }, - { "3COVERED_IMAGE_INTENSITY_RANGE", { 1.0002043207290587 } }, + { "3COVERED_IMAGE_INTENSITY_RANGE", { 0.66137566137566139 } }, { "3ELONGATION", { 0.84332105599389762 } }, // FIX(axis mislabel): was 0.8099; now = MIRP morph_pca_elongation { "3EXCESS_KURTOSIS", { -1.2127631603215119 } }, { "3FLATNESS", { 0.68299758045903847 } }, // FIX(axis mislabel): was 1.186 (>1, impossible); now = MIRP morph_pca_flatness diff --git a/tests/test_3d_firstorder_pyradiomics.h b/tests/test_3d_firstorder_pyradiomics.h index ea165d90..51252b86 100644 --- a/tests/test_3d_firstorder_pyradiomics.h +++ b/tests/test_3d_firstorder_pyradiomics.h @@ -95,11 +95,11 @@ void test_compat_radiomics_3fo_feature (const Nyxus::Feature3D &expected_fcode, // properties of specific ROIs sitting in 'e.uniqueLabels' clear_slide_rois(e.uniqueLabels, e.roiData); - ASSERT_TRUE (gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE (gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // voxel clouds std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE (scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE (scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // buffers ASSERT_NO_THROW (allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); diff --git a/tests/test_3d_glcm.h b/tests/test_3d_glcm.h index 80677669..4bfff26c 100644 --- a/tests/test_3d_glcm.h +++ b/tests/test_3d_glcm.h @@ -85,11 +85,11 @@ void test_3glcm_feature (const Nyxus::Feature3D& expecting_fcode, const std::str // properties of specific ROIs sitting in 'e.uniqueLabels' clear_slide_rois(e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // voxel clouds std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // buffers ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); diff --git a/tests/test_3d_glcm_pyradiomics.h b/tests/test_3d_glcm_pyradiomics.h index 8c4b3e45..be176b14 100644 --- a/tests/test_3d_glcm_pyradiomics.h +++ b/tests/test_3d_glcm_pyradiomics.h @@ -146,11 +146,11 @@ void test_compat_3glcm_feature (const Nyxus::Feature3D& expecting_fcode, const s // properties of specific ROIs sitting in 'e.uniqueLabels' clear_slide_rois(e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // voxel clouds std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // buffers ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); @@ -214,9 +214,9 @@ void test_3dglcm_equivalence_dump() ASSERT_TRUE(scan_slide_props(sp, 3, e.anisoOptions, e.resultOptions.need_annotation())); e.dataset.update_dataset_props_extrema(); clear_slide_rois(e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0, ipath, mpath, 0)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0, ipath, mpath, 0/*t_index*/, 0/*channel*/)); std::vector batch = { label }; - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); Fsettings s; diff --git a/tests/test_3d_gldm.h b/tests/test_3d_gldm.h index 4972fbb2..5f445bde 100644 --- a/tests/test_3d_gldm.h +++ b/tests/test_3d_gldm.h @@ -42,9 +42,9 @@ void test_3gldm_feature (const Nyxus::Feature3D& expecting_fcode, const std::str // mock the 3D workflow Environment e; clear_slide_rois (e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); // make it find the feature code by name @@ -84,10 +84,10 @@ void test_3gldm_feature (const Nyxus::Feature3D& expecting_fcode, const std::str e.dataset.update_dataset_props_extrema(); // (2) properties of specific ROIs sitting in 'e.uniqueLabels' clear_slide_rois(e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // (3) voxel clouds std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // (4) buffers ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); diff --git a/tests/test_3d_gldm_pyradiomics.h b/tests/test_3d_gldm_pyradiomics.h index 87bfb435..2e58313f 100644 --- a/tests/test_3d_gldm_pyradiomics.h +++ b/tests/test_3d_gldm_pyradiomics.h @@ -75,11 +75,11 @@ void test_compat_3gldm_feature (const Nyxus::Feature3D & expecting_fcode, const // properties of specific ROIs sitting in 'e.uniqueLabels' clear_slide_rois (e.uniqueLabels, e.roiData); - ASSERT_TRUE (gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE (gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // voxel clouds std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE (scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE (scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // buffers ASSERT_NO_THROW (allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); diff --git a/tests/test_3d_gldzm_ibsi.h b/tests/test_3d_gldzm_ibsi.h index 70ac2fa8..dfd2937a 100644 --- a/tests/test_3d_gldzm_ibsi.h +++ b/tests/test_3d_gldzm_ibsi.h @@ -41,9 +41,9 @@ void test_3gldzm_feature (const Nyxus::Feature3D& expecting_fcode, const std::st // mock the 3D workflow Environment e; clear_slide_rois (e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); // make it find the feature code by name @@ -84,10 +84,10 @@ void test_3gldzm_feature (const Nyxus::Feature3D& expecting_fcode, const std::st e.dataset.update_dataset_props_extrema(); // (2) properties of specific ROIs sitting in 'e.uniqueLabels' clear_slide_rois(e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // (3) voxel clouds std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // (4) buffers ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); diff --git a/tests/test_3d_glrlm.h b/tests/test_3d_glrlm.h index fba745c8..61e8feda 100644 --- a/tests/test_3d_glrlm.h +++ b/tests/test_3d_glrlm.h @@ -43,9 +43,9 @@ void test_3glrlm_feature (const Nyxus::Feature3D& expecting_fcode, const std::st // mock the 3D workflow Environment e; clear_slide_rois (e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); // make it find the feature code by name @@ -85,10 +85,10 @@ void test_3glrlm_feature (const Nyxus::Feature3D& expecting_fcode, const std::st e.dataset.update_dataset_props_extrema(); // (2) properties of specific ROIs sitting in 'e.uniqueLabels' clear_slide_rois(e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // (3) voxel clouds std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // (4) buffers ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); @@ -210,4 +210,4 @@ void test_3glrlm_rv() void test_3glrlm_re() { test_3glrlm_feature(Nyxus::Feature3D::GLRLM_RE, "3GLRLM_RE"); -} \ No newline at end of file +} \ No newline at end of file diff --git a/tests/test_3d_glrlm_pyradiomics.h b/tests/test_3d_glrlm_pyradiomics.h index 922c69a4..ff28b09a 100644 --- a/tests/test_3d_glrlm_pyradiomics.h +++ b/tests/test_3d_glrlm_pyradiomics.h @@ -160,11 +160,11 @@ void test_compat_3glrlm_feature(const Nyxus::Feature3D& expecting_fcode, const s // properties of specific ROIs sitting in 'e.uniqueLabels' clear_slide_rois(e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // voxel clouds std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // buffers ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); @@ -229,9 +229,9 @@ void test_compat_3glrlm_ave_features() ASSERT_TRUE(scan_slide_props(sp, 3, e.anisoOptions, e.resultOptions.need_annotation())); e.dataset.update_dataset_props_extrema(); clear_slide_rois(e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0, ipath, mpath, 0)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0, ipath, mpath, 0/*t_index*/, 0/*channel*/)); std::vector batch = { label }; - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); Fsettings s; diff --git a/tests/test_3d_glszm.h b/tests/test_3d_glszm.h index d418e286..b6c64783 100644 --- a/tests/test_3d_glszm.h +++ b/tests/test_3d_glszm.h @@ -41,9 +41,9 @@ void test_3glszm_feature (const Nyxus::Feature3D& expecting_fcode, const std::st // mock the 3D workflow Environment e; clear_slide_rois (e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); // make it find the feature code by name @@ -83,10 +83,10 @@ void test_3glszm_feature (const Nyxus::Feature3D& expecting_fcode, const std::st e.dataset.update_dataset_props_extrema(); // (2) properties of specific ROIs sitting in 'e.uniqueLabels' clear_slide_rois(e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // (3) voxel clouds std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // (4) buffers ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); diff --git a/tests/test_3d_glszm_pyradiomics.h b/tests/test_3d_glszm_pyradiomics.h index 1c67b402..0a18986a 100644 --- a/tests/test_3d_glszm_pyradiomics.h +++ b/tests/test_3d_glszm_pyradiomics.h @@ -76,11 +76,11 @@ void test_compat_3glszm_feature (const Nyxus::Feature3D& expecting_fcode, const // properties of specific ROIs sitting in 'e.uniqueLabels' clear_slide_rois(e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // voxel clouds std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // buffers ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); diff --git a/tests/test_3d_inten.h b/tests/test_3d_inten.h index 2ff94c23..913bc6ec 100644 --- a/tests/test_3d_inten.h +++ b/tests/test_3d_inten.h @@ -67,10 +67,10 @@ void test_3inten_feature (const std::string& fname, const Nyxus::Feature3D& expe e.dataset.update_dataset_props_extrema(); // (2) properties of specific ROIs sitting in 'e.uniqueLabels' clear_slide_rois (e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // (3) voxel clouds std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // (4) buffers ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); diff --git a/tests/test_3d_morphology_common.h b/tests/test_3d_morphology_common.h index 4f3265d4..901acf6d 100644 --- a/tests/test_3d_morphology_common.h +++ b/tests/test_3d_morphology_common.h @@ -33,9 +33,9 @@ void test_3shape_feature (const std::string& fname, const Nyxus::Feature3D& expe // mock the 3D workflow Environment e; clear_slide_rois (e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); // make it find the feature code by name @@ -78,10 +78,10 @@ void test_3shape_feature (const std::string& fname, const Nyxus::Feature3D& expe e.dataset.update_dataset_props_extrema(); // (2) properties of specific ROIs sitting in 'e.uniqueLabels' clear_slide_rois(e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // (3) voxel clouds std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // (4) buffers ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); diff --git a/tests/test_3d_ngldm_regression.h b/tests/test_3d_ngldm_regression.h index 217e1d3a..4a0c47a9 100644 --- a/tests/test_3d_ngldm_regression.h +++ b/tests/test_3d_ngldm_regression.h @@ -79,9 +79,9 @@ void test_3ngldm_feature (const std::string& fname, const Nyxus::Feature3D& expe // mock the 3D workflow Environment e; clear_slide_rois (e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); // make it find the feature code by name @@ -122,10 +122,10 @@ void test_3ngldm_feature (const std::string& fname, const Nyxus::Feature3D& expe e.dataset.update_dataset_props_extrema(); // (2) properties of specific ROIs sitting in 'e.uniqueLabels' clear_slide_rois(e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // (3) voxel clouds std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // (4) buffers ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); diff --git a/tests/test_3d_ngtdm.h b/tests/test_3d_ngtdm.h index e0a809d0..f8aa205e 100644 --- a/tests/test_3d_ngtdm.h +++ b/tests/test_3d_ngtdm.h @@ -30,9 +30,9 @@ void test_3ngtdm_feature(const Nyxus::Feature3D& expecting_fcode, const std::str // mock the 3D workflow Environment e; clear_slide_rois (e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); // make it find the feature code by name @@ -73,10 +73,10 @@ void test_3ngtdm_feature(const Nyxus::Feature3D& expecting_fcode, const std::str e.dataset.update_dataset_props_extrema(); // (2) properties of specific ROIs sitting in 'e.uniqueLabels' clear_slide_rois(e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // (3) voxel clouds std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // (4) buffers ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); diff --git a/tests/test_3d_ngtdm_pyradiomics.h b/tests/test_3d_ngtdm_pyradiomics.h index 9466866c..df57fb6a 100644 --- a/tests/test_3d_ngtdm_pyradiomics.h +++ b/tests/test_3d_ngtdm_pyradiomics.h @@ -83,11 +83,11 @@ void test_compat_3ngtdm_feature (const Nyxus::Feature3D& expecting_fcode, const // properties of specific ROIs sitting in 'e.uniqueLabels' clear_slide_rois(e.uniqueLabels, e.roiData); - ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0/*slide_index*/, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // voxel clouds std::vector batch = { label }; // expecting this roi label after metrics gathering - ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/)); + ASSERT_TRUE(scanTrivialRois_3D(e, batch, ipath, mpath, 0/*t_index*/, 0/*channel*/)); // buffers ASSERT_NO_THROW(allocateTrivialRoisBuffers_3D(batch, e.roiData, e.hostCache)); diff --git a/tests/test_3d_nifti_mechanics.h b/tests/test_3d_nifti_mechanics.h index be018f63..aaafb62b 100644 --- a/tests/test_3d_nifti_mechanics.h +++ b/tests/test_3d_nifti_mechanics.h @@ -1,5 +1,6 @@ #include #include "../src/nyx/raw_nifti.h" +#include "../src/nyx/image_loader.h" #include "../src/nyx/helpers/fsystem.h" void test_3d_nifti_loader () @@ -56,7 +57,7 @@ void test_3d_nifti_data_access_consistency() ASSERT_NO_THROW(h = ldr1.fullHeight(0)); auto t = std::make_shared> (d*w*h); - ASSERT_NO_THROW (ldr1.loadTileFromFile (t, 0, 0, 0, 0)); + ASSERT_NO_THROW (ldr1.loadTileFromFile (t, 0, 0, 0, 0/*channel*/, 0/*timeframe*/, 0)); // stats std::vector& databuf = *t; @@ -65,3 +66,40 @@ void test_3d_nifti_data_access_consistency() tot += x; ASSERT_TRUE(tot == 544286216); } + +// Safety net for wiring the volumetric consumer onto load_volume(): for a NIfTI +// (whole-4D loader), load_volume(0,t) must equal the t-th timeframe slab of the +// current whole-volume read (load_tile(0,0) + get_int_tile_buffer). If this holds, +// swapping the consumer's read source cannot regress NIfTI. +void test_facade_nifti_load_volume_equivalence() +{ + fs::path p(__FILE__); + fs::path f("/data/nifti/signal/signal1.nii"); + fs::path ds = (p.parent_path().string() + f.make_preferred().string()); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + SlideProps sp; + sp.fname_int = ds.string(); + sp.fname_seg = ""; + FpImageOptions fp; + ImageLoader il; + ASSERT_TRUE(il.open(sp, fp)) << ds.string(); + + const size_t timeFrameSize = il.get_full_width() * il.get_full_height() * il.get_full_depth(); + const size_t nt = il.get_inten_time(); + ASSERT_GE(nt, 1u); + + ASSERT_TRUE(il.load_tile((size_t)0, (size_t)0)); // the current NIfTI read: whole x*y*z*t blob + const std::vector whole = il.get_int_tile_buffer(); // copy (load_volume reuses ptrI) + ASSERT_GE(whole.size(), timeFrameSize * nt); + + for (size_t t = 0; t < nt; ++t) + { + ASSERT_TRUE(il.load_volume(0, t)); + const std::vector& vol = il.get_int_volume_buffer(); + ASSERT_EQ(vol.size(), timeFrameSize); + for (size_t i = 0; i < timeFrameSize; ++i) + ASSERT_EQ(vol[i], whole[t * timeFrameSize + i]) << "t=" << t << " i=" << i; + } + il.close(); +} diff --git a/tests/test_all.cc b/tests/test_all.cc index e7ced9fd..d8e0084f 100644 --- a/tests/test_all.cc +++ b/tests/test_all.cc @@ -1,9 +1,22 @@ #define NOMINMAX // keep Windows min/max macros from breaking dcmtk's OFvariant (DICOM tests) #include +#include // reading a written CSV back in TEST_CSV_MULTICHANNEL_NO_OVERWRITE #include "test_gabor_regression.h" #include "../src/nyx/environment.h" #include "../src/nyx/globals.h" +#include "../src/nyx/feature_method.h" // TEST_3D_OOC_GUARD_REJECTS_UNSUPPORTED_FEATURE +#include "../src/nyx/features/3d_intensity.h" +#include "../src/nyx/features/3d_glcm.h" +#include "../src/nyx/features/3d_gldm.h" // OOC-vs-RAM equality coverage (all texture families) +#include "../src/nyx/features/3d_glrlm.h" +#include "../src/nyx/features/3d_glszm.h" +#include "../src/nyx/features/3d_gldzm.h" +#include "../src/nyx/features/3d_ngldm.h" +#include "../src/nyx/features/3d_ngtdm.h" +#include "../src/nyx/ome/format_detect.h" // detect_input_format (P1) #include "test_contour.h" +#include "test_ome_meta.h" // native-OME metadata parsers / OmeAxes descriptor +#include "test_ometiff_mechanics.h" // OME-TIFF native (z,c,t)->IFD read (core; no USE_Z5) #include "test_firstorder_regression.h" #include "test_intensity_histogram_regression.h" #include "test_intensity_histogram_ibsi.h" @@ -2509,6 +2522,10 @@ TEST(TEST_NYXUS, TEST_3D_NIFTY_DACC_CONSISTENCY) { ASSERT_NO_THROW (test_3d_nifti_data_access_consistency()); } +TEST(TEST_NYXUS, TEST_FACADE_NIFTI_LOAD_VOLUME_EQUIVALENCE) { + ASSERT_NO_THROW (test_facade_nifti_load_volume_equivalence()); +} + //***** OME-Zarr i/o ***** @@ -2538,10 +2555,1230 @@ TEST(TEST_NYXUS, TEST_RAW_OMEZARR_MULTITILE) { ASSERT_NO_THROW (test_raw_omezarr_multitile()); } +TEST(TEST_NYXUS, TEST_OMEZARR_5D_CHANNEL_TIME_ADDRESSING) { + ASSERT_NO_THROW (test_omezarr_addressing("dim5.ome.zarr", 2, 3, 4)); +} + +TEST(TEST_NYXUS, TEST_RAW_OMEZARR_5D_CHANNEL_TIME_ADDRESSING) { + ASSERT_NO_THROW (test_raw_omezarr_addressing("dim5.ome.zarr", 2, 3, 4)); +} + +// All 6 legal orderings of {t,c,z}: passes only if the loader honors the NGFF +// 'axes' metadata instead of assuming a fixed [T,C,Z,Y,X] order. +TEST(TEST_NYXUS, TEST_OMEZARR_ALL_5D_PERMUTATIONS) { + ASSERT_NO_THROW (test_omezarr_all_5d_permutations()); +} + +// 4D (rank-4): time-only and channel-only. +TEST(TEST_NYXUS, TEST_OMEZARR_4D_TZYX) { + ASSERT_NO_THROW (test_omezarr_addressing("dim4_tzyx.ome.zarr", 2, 1, 4)); + ASSERT_NO_THROW (test_raw_omezarr_addressing("dim4_tzyx.ome.zarr", 2, 1, 4)); +} +TEST(TEST_NYXUS, TEST_OMEZARR_4D_CZYX) { + ASSERT_NO_THROW (test_omezarr_addressing("dim4_czyx.ome.zarr", 1, 3, 4)); + ASSERT_NO_THROW (test_raw_omezarr_addressing("dim4_czyx.ome.zarr", 1, 3, 4)); +} + +// End-to-end through the wired volumetric consumer (scan_trivial_wholevolume). +TEST(TEST_NYXUS, TEST_OMEZARR_WHOLEVOLUME_CONSUMER) { + ASSERT_NO_THROW (test_omezarr_wholevolume_consumer("dim3_zyx.ome.zarr", 4)); +} + +// Facade whole-volume assembly (load_volume loops Z into one X*Y*Z buffer). +// Wired consumer reads the correct plane for every (channel, timeframe), not just (0,0). +TEST(TEST_NYXUS, TEST_OMEZARR_WHOLEVOLUME_CONSUMER_CT) { + ASSERT_NO_THROW (test_omezarr_wholevolume_consumer_ct("dim5.ome.zarr", 2, 3, 4)); +} + +TEST(TEST_NYXUS, TEST_OMEZARR_FACADE_VOLUME_3D) { + ASSERT_NO_THROW (test_omezarr_facade_volume("dim3_zyx.ome.zarr", 1, 1, 4)); +} +TEST(TEST_NYXUS, TEST_OMEZARR_FACADE_VOLUME_5D) { + ASSERT_NO_THROW (test_omezarr_facade_volume("dim5.ome.zarr", 2, 3, 4)); +} + +// Lower-rank stores: a fixed shape[2..4] loader would crash; the axis-role loader +// reads 3D (ZYX) and 2D (YX) correctly. +TEST(TEST_NYXUS, TEST_OMEZARR_3D_ZYX) { + ASSERT_NO_THROW (test_omezarr_addressing("dim3_zyx.ome.zarr", 1, 1, 4)); +} + +TEST(TEST_NYXUS, TEST_RAW_OMEZARR_3D_ZYX) { + ASSERT_NO_THROW (test_raw_omezarr_addressing("dim3_zyx.ome.zarr", 1, 1, 4)); +} + +TEST(TEST_NYXUS, TEST_OMEZARR_2D_YX) { + ASSERT_NO_THROW (test_omezarr_addressing("dim2_yx.ome.zarr", 1, 1, 1)); +} + +// OME-Zarr 0.5 (Zarr v3): zarr.json metadata + 'ome'-wrapped NGFF + 0/c/... chunk keys, +// read through the z5 Dataset API (v2/v3-agnostic). Same coordinate encoding as the v2 +// stores, so the addressing / facade / CT-count helpers apply unchanged. +TEST(TEST_NYXUS, TEST_OMEZARR_V3) { + ASSERT_NO_THROW (test_omezarr_addressing("dim5_v3.ome.zarr", 2, 3, 4)); + ASSERT_NO_THROW (test_raw_omezarr_addressing("dim5_v3.ome.zarr", 2, 3, 4)); +} +TEST(TEST_NYXUS, TEST_OMEZARR_V3_FACADE_VOLUME) { + ASSERT_NO_THROW (test_omezarr_facade_volume("dim5_v3.ome.zarr", 2, 3, 4)); +} +TEST(TEST_NYXUS, TEST_OMEZARR_V3_CT_COUNTS) { + ASSERT_NO_THROW (test_omezarr_ct_counts("dim5_v3.ome.zarr", 2, 3, 4)); +} +// Zstd-compressed Zarr v3 (the zarr 3.x / real-world default codec). z5 decodes the +// bytes+zstd v3 codec pipeline only when the build found libzstd and its header, so this +// case is compiled in on the same condition CMake uses to enable the codec; a build +// without zstd cannot read the fixture and must not be failed for it. +#ifdef WITH_ZSTD +TEST(TEST_NYXUS, TEST_OMEZARR_V3_ZSTD) { + ASSERT_NO_THROW (test_omezarr_addressing("dim5_v3_zstd.ome.zarr", 2, 3, 4)); + ASSERT_NO_THROW (test_raw_omezarr_addressing("dim5_v3_zstd.ome.zarr", 2, 3, 4)); +} +#endif + +// Sharded Zarr v3 (the ``sharding_indexed`` codec) -- how large v3 stores, including Axle's, +// actually lay out data: many inner chunks packed into one shard object per (t,c). z5 3.x +// reads it via ShardedDataset, chosen automatically when zarr.json carries a shard shape; the +// nyxus loader is UNCHANGED because it reads through readSubarray, which unpacks the inner +// chunks from the shard transparently. The fixture's inner chunk is (z,y,x)=(1,3,4) so each +// 6x8 plane is a 2x2 grid of inner chunks living inside one shard -- the read must assemble +// across inner-chunk boundaries within a shard. Same 1..1152 TCZYX encoding as the other v3. +// +// Coverage is the whole-volume facade + prescan, NOT test_omezarr_addressing: with sharding, +// tileWidth/Height report the INNER chunk (4x3), so a single loadTileFromFile(0,0,...) reads +// only the top-left inner chunk, not the whole plane -- the addressing helper's one-tile-per- +// plane assumption. facade_volume assembles the full inner-chunk grid and checks every voxel, +// which is the correct coverage for a multi-chunk store (same reason the multichunk v2 fixture +// uses it). It drives both loadTileFromFile (abstract stack) and readSubarray under the hood. +TEST(TEST_NYXUS, TEST_OMEZARR_V3_SHARDED_FACADE_VOLUME) { + ASSERT_NO_THROW (test_omezarr_facade_volume("dim5_v3_sharded.ome.zarr", 2, 3, 4)); +} +TEST(TEST_NYXUS, TEST_OMEZARR_V3_SHARDED_CT_COUNTS) { + ASSERT_NO_THROW (test_omezarr_ct_counts("dim5_v3_sharded.ome.zarr", 2, 3, 4)); +} +// Prescan over the sharded store (raw loader's readSubarray, driven through the inner-chunk +// tile grid by for_each_voxel): whole-slide, so the ROI is the whole X*Y*Z volume. +TEST(TEST_NYXUS, TEST_OMEZARR_V3_SHARDED_PRESCAN) { + fs::path ip = omezarr_data_path("dim5_v3_sharded.ome.zarr"); + ASSERT_TRUE(fs::exists(ip)) << ip.string(); + + Environment e; + SlideProps p (ip.string(), ""); // whole-slide: no mask + ASSERT_TRUE(Nyxus::scan_slide_props(p, 3, e.anisoOptions, e.resultOptions.need_annotation())); + + EXPECT_DOUBLE_EQ(p.min_preroi_inten, 1.0); + EXPECT_DOUBLE_EQ(p.max_preroi_inten, 1152.0); + EXPECT_EQ(p.max_roi_area, (size_t)(8 * 6 * 4)); +} + +// Blosc-compressed Zarr v3 (common in real v3 stores alongside zstd). z5 decodes the +// bytes+blosc v3 codec pipeline when built WITH_BLOSC (already required for OME-Zarr). +TEST(TEST_NYXUS, TEST_OMEZARR_V3_BLOSC) { + ASSERT_NO_THROW (test_omezarr_addressing("dim5_v3_blosc.ome.zarr", 2, 3, 4)); + ASSERT_NO_THROW (test_raw_omezarr_addressing("dim5_v3_blosc.ome.zarr", 2, 3, 4)); +} +TEST(TEST_NYXUS, TEST_OMEZARR_V3_BLOSC_FACADE_VOLUME) { + ASSERT_NO_THROW (test_omezarr_facade_volume("dim5_v3_blosc.ome.zarr", 2, 3, 4)); +} + +// Larger multi-SHARD-FILE Zarr v3 store (gen_dim5.py write_v3_multishard): C=2,T=1,Z=8,Y=24,X=32, +// inner chunk (z,y,x)=(2,6,8), shard (z,y,x)=(8,12,16) -> a 2x2 GRID OF SHARD FILES per (c,t), +// each packing 16 inner chunks (8 shard files total). Unlike dim5_v3_sharded (exactly one shard +// per (t,c), so it only proves multiple inner chunks packed into ONE shard), this exercises the +// volumetric assembly crossing SHARD-FILE boundaries mid-plane -- closer to a real, larger v3 +// store's layout. Own local encoding (dim5_enc's C/Z/Y/X are hardcoded to the small fixture and +// don't apply here): value(x,y,z,c,t) = 1 + ((((t*C+c)*Z+z)*Y+y)*X+x), C=2,T=1,Z=8,Y=24,X=32. +static inline uint32_t dim5_multishard_enc(int x, int y, int z, int c, int t) +{ + const int C = 2, Z = 8, Y = 24, X = 32; + return static_cast(1 + ((((t * C + c) * Z + z) * Y + y) * X + x)); +} + +TEST(TEST_NYXUS, TEST_OMEZARR_V3_MULTISHARD_FACADE_VOLUME) { + const int T = 1, C = 2, Z = 8, Y = 24, X = 32; + fs::path ds = omezarr_data_path("dim5_v3_multishard.ome.zarr"); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + SlideProps p; + p.fname_int = ds.string(); + p.fname_seg = ""; + FpImageOptions fp; + ImageLoader il; + ASSERT_TRUE(il.open(p, fp)) << ds.string(); + ASSERT_EQ(il.get_full_width(), (size_t)X); + ASSERT_EQ(il.get_full_height(), (size_t)Y); + ASSERT_EQ(il.get_full_depth(), (size_t)Z); + + for (int t = 0; t < T; ++t) + for (int c = 0; c < C; ++c) + { + ASSERT_TRUE(il.load_volume(c, t)); + const std::vector& vol = il.get_int_volume_buffer(); + ASSERT_EQ(vol.size(), (size_t)X * Y * Z); + for (int z = 0; z < Z; ++z) + for (int y = 0; y < Y; ++y) + for (int x = 0; x < X; ++x) + ASSERT_EQ(vol[(size_t)z * X * Y + (size_t)y * X + x], dim5_multishard_enc(x, y, z, c, t)) + << "multishard vol (x" << x << " y" << y << " z" << z << " c" << c << " t" << t << ")"; + } + il.close(); +} + +TEST(TEST_NYXUS, TEST_OMEZARR_V3_MULTISHARD_CT_COUNTS) { + fs::path ds = omezarr_data_path("dim5_v3_multishard.ome.zarr"); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + auto ldr = NyxusOmeZarrLoader(1, ds.string()); + ASSERT_EQ(ldr.numberChannels(), (size_t)2); + ASSERT_EQ(ldr.fullTimestamps(0), (size_t)1); + ASSERT_EQ(ldr.fullDepth(0), (size_t)8); + + auto raw = RawOmezarrLoader(ds.string()); + ASSERT_EQ(raw.numberChannels(), (size_t)2); + ASSERT_EQ(raw.fullTimestamps(0), (size_t)1); + ASSERT_EQ(raw.fullDepth(0), (size_t)8); +} + +// Prescan (raw loader's readSubarray driven across all 8 shard files) must see the full encoded +// range across BOTH channels, and the whole-slide ROI area -- not garbage, not just channel 0. +TEST(TEST_NYXUS, TEST_OMEZARR_V3_MULTISHARD_PRESCAN) { + fs::path ip = omezarr_data_path("dim5_v3_multishard.ome.zarr"); + ASSERT_TRUE(fs::exists(ip)) << ip.string(); + + Environment e; + SlideProps p (ip.string(), ""); // whole-slide: no mask + ASSERT_TRUE(Nyxus::scan_slide_props(p, 3, e.anisoOptions, e.resultOptions.need_annotation())); + + EXPECT_DOUBLE_EQ(p.min_preroi_inten, 1.0); + EXPECT_DOUBLE_EQ(p.max_preroi_inten, dim5_multishard_enc(31, 23, 7, 1, 0)); // last voxel, last channel + EXPECT_EQ(p.max_roi_area, (size_t)(32 * 24 * 8)); +} + +// Zarr v3 store with a MULTI-PLANE Z chunk (chunk z-extent 3 over Z=7 -> depths 3,3,1, an UNEVEN +// split), unsharded -- isolated regression test for a real bug found while building the multishard +// fixture above: omezarr.h/raw_omezarr.h's loadTile() always read exactly ONE Z-plane per tile +// (shape[iz_] left at its default of 1) regardless of the chunk's actual Z extent, so every plane +// past the first within a multi-plane chunk silently came back zero. No existing fixture before +// this one ever used a Z-chunk > 1. Own local encoding (dim5_enc/dim5_multishard_enc don't apply -- +// different dims): value(x,y,z,c,t) = 1 + ((((t*C+c)*Z+z)*Y+y)*X+x), C=2,T=1,Z=7,Y=6,X=8. +static inline uint32_t dim5_zchunked_enc(int x, int y, int z, int c, int t) +{ + const int C = 2, Z = 7, Y = 6, X = 8; + return static_cast(1 + ((((t * C + c) * Z + z) * Y + y) * X + x)); +} + +// Exercises omezarr.h's NyxusOmeZarrLoader via ImageLoader::load_volume/assemble_volume -- the +// path that read zero past the first Z-plane of a chunk before the fix. +TEST(TEST_NYXUS, TEST_OMEZARR_V3_ZCHUNKED_FACADE_VOLUME) { + const int T = 1, C = 2, Z = 7, Y = 6, X = 8; + fs::path ds = omezarr_data_path("dim5_v3_zchunked.ome.zarr"); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + SlideProps p; + p.fname_int = ds.string(); + p.fname_seg = ""; + FpImageOptions fp; + ImageLoader il; + ASSERT_TRUE(il.open(p, fp)) << ds.string(); + ASSERT_EQ(il.get_full_depth(), (size_t)Z); + + for (int t = 0; t < T; ++t) + for (int c = 0; c < C; ++c) + { + ASSERT_TRUE(il.load_volume(c, t)); + const std::vector& vol = il.get_int_volume_buffer(); + ASSERT_EQ(vol.size(), (size_t)X * Y * Z); + for (int z = 0; z < Z; ++z) + for (int y = 0; y < Y; ++y) + for (int x = 0; x < X; ++x) + ASSERT_EQ(vol[(size_t)z * X * Y + (size_t)y * X + x], dim5_zchunked_enc(x, y, z, c, t)) + << "zchunked vol (x" << x << " y" << y << " z" << z << " c" << c << " t" << t << ")"; + } + il.close(); +} + +// Exercises raw_omezarr.h's RawOmezarrLoader via RawImageLoader::for_each_voxel (the prescan path) +// -- the OTHER consumer of the same buggy loadTile(), independently regressed here. +TEST(TEST_NYXUS, TEST_OMEZARR_V3_ZCHUNKED_PRESCAN) { + fs::path ip = omezarr_data_path("dim5_v3_zchunked.ome.zarr"); + ASSERT_TRUE(fs::exists(ip)) << ip.string(); + + Environment e; + SlideProps p (ip.string(), ""); // whole-slide: no mask + ASSERT_TRUE(Nyxus::scan_slide_props(p, 3, e.anisoOptions, e.resultOptions.need_annotation())); + + EXPECT_DOUBLE_EQ(p.min_preroi_inten, 1.0); + EXPECT_DOUBLE_EQ(p.max_preroi_inten, dim5_zchunked_enc(7, 5, 6, 1, 0)); // last voxel, last channel + EXPECT_EQ(p.max_roi_area, (size_t)(8 * 6 * 7)); +} + +// No 'axes' metadata -> the loader falls back to legacy 5D TCZYX and still reads. +TEST(TEST_NYXUS, TEST_OMEZARR_NOAXES_FALLBACK) { + ASSERT_NO_THROW (test_omezarr_addressing("dim5_noaxes.ome.zarr", 2, 3, 4)); +} + +TEST(TEST_NYXUS, TEST_RAW_OMEZARR_NOAXES_FALLBACK) { + ASSERT_NO_THROW (test_raw_omezarr_addressing("dim5_noaxes.ome.zarr", 2, 3, 4)); +} + +// Loaders advertise the real C/T extents (numberChannels/fullTimestamps), which +// is what activates the pipeline's channel/timeframe iteration. dim5_noaxes proves +// the positional fallback reports counts too. +TEST(TEST_NYXUS, TEST_OMEZARR_CT_COUNTS) { + ASSERT_NO_THROW (test_omezarr_ct_counts("dim5.ome.zarr", 2, 3, 4)); + ASSERT_NO_THROW (test_omezarr_ct_counts("dim4_tzyx.ome.zarr", 2, 1, 4)); + ASSERT_NO_THROW (test_omezarr_ct_counts("dim4_czyx.ome.zarr", 1, 3, 4)); + ASSERT_NO_THROW (test_omezarr_ct_counts("dim3_zyx.ome.zarr", 1, 1, 4)); + ASSERT_NO_THROW (test_omezarr_ct_counts("dim2_yx.ome.zarr", 1, 1, 1)); + ASSERT_NO_THROW (test_omezarr_ct_counts("dim5_noaxes.ome.zarr", 2, 3, 4)); +} + +// Physical calibration: loaders surface coordinateTransformations scale + unit. +TEST(TEST_NYXUS, TEST_OMEZARR_PHYSICAL_CALIBRATION) { + ASSERT_NO_THROW (test_omezarr_physical_calibration()); +} + +// Unit canonicalization: a nanometer-declared store must report the same physX/Y/Z and +// "micrometer" as the equivalent micrometer-declared store above -- proves conversion, not +// just passthrough of the raw unit string. +TEST(TEST_NYXUS, TEST_OMEZARR_UNIT_CANONICALIZATION) { + ASSERT_NO_THROW (test_omezarr_unit_canonicalization()); +} + +// Multi-CHUNK plane: real OME-Zarr splits each Y/X plane across a chunk grid (typically +// 512x512), and dim5_multichunk uses 3x4 chunks over the 6x8 plane. The volumetric read +// must walk the whole tile grid: fetching only chunk (0,0) returns wrong data past the +// first chunk (and over-reads its buffer). Every other fixture is one-chunk-per-plane, +// which is why this went unnoticed. Covers ImageLoader::assemble_volume... +TEST(TEST_NYXUS, TEST_OMEZARR_MULTICHUNK_FACADE_VOLUME) { + ASSERT_NO_THROW (test_omezarr_facade_volume("dim5_multichunk.ome.zarr", 2, 3, 4)); +} +// ...and RawImageLoader::load_volume (the prescan), which had the same single-tile bug: +// the encoded values are 1..1152 over all (c,t), and the ROI is the whole X*Y*Z volume. +TEST(TEST_NYXUS, TEST_OMEZARR_MULTICHUNK_PRESCAN) { + fs::path ip = omezarr_data_path("dim5_multichunk.ome.zarr"); + ASSERT_TRUE(fs::exists(ip)) << ip.string(); + + Environment e; + SlideProps p (ip.string(), ""); // whole-slide: no mask + ASSERT_TRUE(Nyxus::scan_slide_props(p, 3, e.anisoOptions, e.resultOptions.need_annotation())); + + EXPECT_DOUBLE_EQ(p.min_preroi_inten, 1.0); + EXPECT_DOUBLE_EQ(p.max_preroi_inten, 1152.0); + EXPECT_EQ(p.max_roi_area, (size_t)(8 * 6 * 4)); +} + +// PARTIAL edge chunks: chunk (4,5) does not divide the 6x8 plane, so the last row-chunk is 2 +// tall and the last col-chunk is 3 wide. dim5_multichunk above (3x4 over 6x8) tiles exactly, +// so the validH/validW seam clamp never ran on the OME-Zarr path either. Asserting the exact +// value at every voxel is the seam check; same 1..1152 TCZYX encoding as dim5_multichunk. +TEST(TEST_NYXUS, TEST_OMEZARR_ODDCHUNK_FACADE_VOLUME) { + ASSERT_NO_THROW (test_omezarr_facade_volume("dim5_oddchunk.ome.zarr", 2, 3, 4)); +} +TEST(TEST_NYXUS, TEST_OMEZARR_ODDCHUNK_PRESCAN) { + fs::path ip = omezarr_data_path("dim5_oddchunk.ome.zarr"); + ASSERT_TRUE(fs::exists(ip)) << ip.string(); + + Environment e; + SlideProps p (ip.string(), ""); // whole-slide: no mask + ASSERT_TRUE(Nyxus::scan_slide_props(p, 3, e.anisoOptions, e.resultOptions.need_annotation())); + + EXPECT_DOUBLE_EQ(p.min_preroi_inten, 1.0); + EXPECT_DOUBLE_EQ(p.max_preroi_inten, 1152.0); + EXPECT_EQ(p.max_roi_area, (size_t)(8 * 6 * 4)); +} + +// Negative: out-of-range channel/timeframe through the whole-volume facade must throw. +TEST(TEST_NYXUS, TEST_OMEZARR_LOAD_VOLUME_OUT_OF_RANGE) { + ASSERT_NO_THROW (test_omezarr_load_volume_out_of_range()); +} + +// Negative: out-of-range Z/C/T plane index must throw. +TEST(TEST_NYXUS, TEST_OMEZARR_OUT_OF_RANGE_THROWS) { + ASSERT_NO_THROW (test_omezarr_out_of_range_throws()); +} + +// Illegal / adversarial: malformed metadata must be rejected cleanly, not crash. +TEST(TEST_NYXUS, TEST_OMEZARR_MALFORMED_THROWS) { + ASSERT_NO_THROW (test_omezarr_malformed_throws()); +} + +// N1 (negative): a Zarr v3 store whose zarr.json declares a codec z5 does not support. z5's +// readV3CodecsFromJson throws "unsupported zarr v3 codec" during metadata parse (openDataset), +// so the loader must surface a clean error, not crash. Both loader stacks must reject it. +TEST(TEST_NYXUS, TEST_OMEZARR_V3_UNSUPPORTED_CODEC_THROWS) { + fs::path ds = omezarr_data_path("dim5_v3_badcodec.ome.zarr"); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + EXPECT_ANY_THROW(NyxusOmeZarrLoader(1, ds.string())); + EXPECT_ANY_THROW(RawOmezarrLoader(ds.string())); +} + +// P4 (positive): the crash's positive twin on the OME-Zarr path -- a T>1 Zarr intensity paired +// with a single-timeframe ZYX Zarr mask. Zarr never crashed (no T axis to over-index), but it +// was never asserted. The prescan must reuse the mask across timeframes and find the ROI. +TEST(TEST_NYXUS, TEST_OMEZARR_MULTITIMEFRAME_MASK_PRESCAN) { + fs::path ip = omezarr_data_path("dim5.ome.zarr"); // T=2, C=3, Z=4 + fs::path mp = omezarr_data_path("dim3_mask.ome.zarr"); // ZYX (T=1) label mask + ASSERT_TRUE(fs::exists(ip) && fs::exists(mp)); + Environment e; + SlideProps p (ip.string(), mp.string()); + bool ok = false; + ASSERT_NO_THROW(ok = Nyxus::scan_slide_props(p, 3, e.anisoOptions, e.resultOptions.need_annotation())); + EXPECT_TRUE(ok); + EXPECT_EQ(p.max_roi_area, (size_t)(4 * 4 * 6)); +} + #endif // OMEZARR_SUPPORT -int main(int argc, char **argv) +//***** OME-TIFF native (z,c,t) -> IFD read (core; runs in every build) ***** + +TEST(TEST_NYXUS, TEST_OMETIFF_5D_CHANNEL_TIME_ADDRESSING) { + ASSERT_NO_THROW (test_ometiff_addressing("dim5.ome.tif", 2, 3, 4)); +} +TEST(TEST_NYXUS, TEST_RAW_OMETIFF_5D_CHANNEL_TIME_ADDRESSING) { + ASSERT_NO_THROW (test_raw_ometiff_addressing("dim5.ome.tif", 2, 3, 4)); +} + +// All 6 legal DimensionOrder values: passes only if ifdForPlane honors DimensionOrder. +TEST(TEST_NYXUS, TEST_OMETIFF_ALL_5D_PERMUTATIONS) { + ASSERT_NO_THROW (test_ometiff_all_5d_permutations()); +} + +// Regression (found by the scale/stress harness): a MULTI-timeframe OME-TIFF paired with a +// single-timeframe (ZYX) mask. The 3D prescan (RawImageLoader::for_each_voxel) reuses the +// mask across every intensity timeframe, but read it at the intensity's timeframe -- so for +// t>0 the TIFF mask loader addressed an IFD past its last plane and TIFFSetDirectory threw, +// UNCAUGHT, crashing the process (0xC0000409). Zarr masks have no T axis to over-index, so +// only TIFF crashed. dim5 is T=2,C=3,Z=4; dim3_mask is its ZYX (T=1) segmentation of one ROI +// (label 1 over z=all, y in [1,5), x in [1,7) = 4*4*6 voxels). Pre-fix this threw. +TEST(TEST_NYXUS, TEST_OMETIFF_MULTITIMEFRAME_MASK_PRESCAN) { + fs::path ip = ometiff_data_path("dim5.ome.tif"); + fs::path mp = ometiff_data_path("dim3_mask.ome.tif"); + ASSERT_TRUE(fs::exists(ip)) << ip.string(); + ASSERT_TRUE(fs::exists(mp)) << mp.string(); + + Environment e; + SlideProps p (ip.string(), mp.string()); + bool ok = false; + ASSERT_NO_THROW(ok = Nyxus::scan_slide_props(p, 3, e.anisoOptions, e.resultOptions.need_annotation())); + EXPECT_TRUE(ok); + EXPECT_EQ(p.max_roi_area, (size_t)(4 * 4 * 6)); // z=4 * y=4 * x=6 +} + +// P3 (positive): the crash's regression above covers the PRESCAN (for_each_voxel); this covers +// the FEATURIZE facade. ImageLoader::load_volume(c,t) forwards t as the mask timeframe, so with +// a T>1 intensity and a T=1 mask, load_volume(c,1) exercises the internal mask-timeframe clamp +// (image_loader.cpp). It must not throw, must reuse the same mask across timeframes, and must +// read different intensity per timeframe. dim5 is T=2; dim3_mask is its ZYX (T=1) mask. +TEST(TEST_NYXUS, TEST_OMETIFF_MULTITIMEFRAME_MASK_FACADE) { + fs::path ip = ometiff_data_path("dim5.ome.tif"); + fs::path mp = ometiff_data_path("dim3_mask.ome.tif"); + ASSERT_TRUE(fs::exists(ip) && fs::exists(mp)); + SlideProps p; p.fname_int = ip.string(); p.fname_seg = mp.string(); + FpImageOptions fp; ImageLoader il; + ASSERT_TRUE(il.open(p, fp)) << ip.string(); + + ASSERT_TRUE(il.load_volume(0, 0)); + const std::vector seg_t0 = il.get_seg_volume_buffer(); + const std::vector int_t0 = il.get_int_volume_buffer(); + // timeframe 1 with a single-timeframe mask must NOT throw (clamp), reuse the mask, and + // deliver different intensity + ASSERT_TRUE(il.load_volume(0, 1)); + EXPECT_EQ(il.get_seg_volume_buffer(), seg_t0) << "mask changed across timeframes"; + EXPECT_NE(il.get_int_volume_buffer(), int_t0) << "intensity t=1 read t=0 data"; + il.close(); +} + +// P2 (positive): OME-TIFF physical calibration -> SlideProps (the TIFF twin of the OME-Zarr +// calibration test, and it additionally checks the scan_slide_props propagation the Zarr test +// omits). dim5_calibrated carries PhysicalSizeX/Y=0.5, Z=2.0 micrometer in its OME-XML. +TEST(TEST_NYXUS, TEST_OMETIFF_PHYSICAL_CALIBRATION) { + fs::path cal = ometiff_data_path("dim5_calibrated.ome.tif"); + ASSERT_TRUE(fs::exists(cal)) << cal.string(); + + Environment e; + SlideProps p (cal.string(), ""); + ASSERT_TRUE(Nyxus::scan_slide_props(p, 3, e.anisoOptions, e.resultOptions.need_annotation())); + EXPECT_DOUBLE_EQ(p.phys_x, 0.5); + EXPECT_DOUBLE_EQ(p.phys_y, 0.5); + EXPECT_DOUBLE_EQ(p.phys_z, 2.0); + EXPECT_EQ(p.phys_unit, "micrometer"); + + // an uncalibrated OME-TIFF must default to 1.0 / no unit + SlideProps p2 (ometiff_data_path("dim5.ome.tif").string(), ""); + ASSERT_TRUE(Nyxus::scan_slide_props(p2, 3, e.anisoOptions, e.resultOptions.need_annotation())); + EXPECT_DOUBLE_EQ(p2.phys_x, 1.0); + EXPECT_DOUBLE_EQ(p2.phys_z, 1.0); + EXPECT_TRUE(p2.phys_unit.empty()); +} + +// Unit canonicalization (OME-TIFF, end-to-end through scan_slide_props): dim5_calibrated_nm +// declares X/Y in nanometer and Z in a THIRD unit (millimeter) -- 500nm==0.5um, +// 0.002mm==2.0um. Must report the SAME physX/Y/Z and a single "micrometer" unit as +// dim5_calibrated above, proving each axis converts using its OWN declared unit rather than +// X/Y's unit leaking onto Z (or vice versa). +TEST(TEST_NYXUS, TEST_OMETIFF_UNIT_CANONICALIZATION) { + fs::path cal_nm = ometiff_data_path("dim5_calibrated_nm.ome.tif"); + ASSERT_TRUE(fs::exists(cal_nm)) << cal_nm.string(); + + Environment e; + SlideProps p (cal_nm.string(), ""); + ASSERT_TRUE(Nyxus::scan_slide_props(p, 3, e.anisoOptions, e.resultOptions.need_annotation())); + EXPECT_DOUBLE_EQ(p.phys_x, 0.5); + EXPECT_DOUBLE_EQ(p.phys_y, 0.5); + EXPECT_DOUBLE_EQ(p.phys_z, 2.0); + EXPECT_EQ(p.phys_unit, "micrometer"); +} + +// N2 (negative): a block maps a plane to an IFD PAST the end of the file (an +// in-file overrun, distinct from a multi-file UUID). ifdForPlane returns 99, so the read must +// throw cleanly at TIFFSetDirectory -- not crash and not read a wrong plane. dim5_badifd has +// 4 z-planes; plane z3 claims IFD=99. +TEST(TEST_NYXUS, TEST_OMETIFF_BAD_IFD_THROWS) { + fs::path ds = ometiff_data_path("dim5_badifd.ome.tif"); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + SlideProps p; p.fname_int = ds.string(); p.fname_seg = ""; + FpImageOptions fp; ImageLoader il; + ASSERT_TRUE(il.open(p, fp)) << ds.string(); + // z0..z2 are fine; assembling the whole volume must hit z3's bad IFD and throw, not crash + EXPECT_ANY_THROW(il.load_volume(0, 0)); + il.close(); +} + +// N3 (negative): an all-background (all-zero) mask -> ZERO ROIs. The prescan must complete +// cleanly (no divide-by-zero, no garbage), reporting no ROI area, rather than crash. +TEST(TEST_NYXUS, TEST_OMETIFF_EMPTY_MASK_ZERO_ROIS) { + fs::path ip = ometiff_data_path("dim5.ome.tif"); + fs::path mp = ometiff_data_path("dim3_emptymask.ome.tif"); + ASSERT_TRUE(fs::exists(ip) && fs::exists(mp)); + Environment e; + SlideProps p (ip.string(), mp.string()); + bool ok = false; + ASSERT_NO_THROW(ok = Nyxus::scan_slide_props(p, 3, e.anisoOptions, e.resultOptions.need_annotation())); + EXPECT_TRUE(ok); + EXPECT_EQ(p.max_roi_area, (size_t)0) << "empty mask should yield no ROI"; +} + +// N4 (edge): a mask with MORE channels than the intensity (C=2 mask, C=1-effective use). The +// featurize loop iterates the intensity's channels, so the extra mask channel must simply be +// ignored (mask channel clamped to what is asked), not crash or misread. dim3_zyx (C=1) paired +// with a C=2 label mask; the ROI is identical on both mask channels. +TEST(TEST_NYXUS, TEST_OMETIFF_MASK_MORE_CHANNELS_THAN_INTENSITY) { + fs::path ip = ometiff_data_path("dim3_zyx.ome.tif"); // C=1, Z=4 + fs::path mp = ometiff_data_path("dim4_mask_c2.ome.tif"); // C=2 label mask + ASSERT_TRUE(fs::exists(ip) && fs::exists(mp)); + Environment e; + SlideProps p (ip.string(), mp.string()); + bool ok = false; + ASSERT_NO_THROW(ok = Nyxus::scan_slide_props(p, 3, e.anisoOptions, e.resultOptions.need_annotation())); + EXPECT_TRUE(ok); + EXPECT_EQ(p.max_roi_area, (size_t)(4 * 4 * 6)); // the one ROI, read from mask channel 0 +} + +// P1 (positive + negative): detect_input_format is the single dispatch point all three loader +// stacks use; it had no direct test. Extension classification + OME content-sniff. +TEST(TEST_NYXUS, TEST_DETECT_INPUT_FORMAT) { + using Nyxus::detect_input_format; + using Nyxus::ContainerKind; + // OME content present -> OmeTiff + is_ome + auto a = detect_input_format(ometiff_data_path("dim5.ome.tif").string()); + EXPECT_EQ(a.kind, ContainerKind::OmeTiff); + EXPECT_TRUE(a.is_ome); + // a plain multipage TIFF (no OME-XML) -> TiffPlain, not OME + auto b = detect_input_format(ometiff_data_path("dim3_plain.tif").string()); + EXPECT_EQ(b.kind, ContainerKind::TiffPlain); + EXPECT_FALSE(b.is_ome); + // extension-only kinds (no file needed / not opened) + EXPECT_EQ(detect_input_format("x.dcm").kind, ContainerKind::Dicom); + EXPECT_EQ(detect_input_format("x.nii.gz").kind, ContainerKind::Nifti); + // a .zarr path with no multiscales metadata -> OmeZarr kind but is_ome=false + auto z = detect_input_format("no_such_store.zarr"); + EXPECT_EQ(z.kind, ContainerKind::OmeZarr); + EXPECT_FALSE(z.is_ome); +} + +// Pyramidal OME-TIFF: every full-res plane's IFD carries downsampled levels as SubIFDs (tag +// 330), which live OUTSIDE the main IFD chain. This must not shift full-res plane addressing: +// TIFFNumberOfDirectories still returns Z (not Z*levels) and ifdForPlane -> main-chain IFD +// still lands on the full-res plane. nyxus reads level 0 only; the facade check (which also +// spans a 2x3 tile grid) asserts every full-res voxel is correct despite the SubIFDs. Z=6 +// z-stack (C=1,T=1), its own encoding. +TEST(TEST_NYXUS, TEST_OMETIFF_PYRAMID_SUBIFD_FULLRES) { + ASSERT_NO_THROW (test_ometiff_multitile_facade_volume("dim5_pyramid.ome.tif", 1, 1, 6, 32, 48)); +} + +// Non-canonical plane->IFD mapping: dim5_reordered stores its planes in REVERSED +// IFD order and declares the mapping via per-plane blocks. A reader that +// ignores TiffData and assumes contiguous-from-IFD-0 order reads the reversed plane's pixels; +// honoring the map reads correctly. load_volume loops Z through loadTileFromFile -> ifdForPlane +// (both loader stacks route here), so the facade check asserts the right plane per (z,c). +// T=1,C=2,Z=3 (its own encoding). This is the OME-TIFF counterpart to what bioformats emits. +TEST(TEST_NYXUS, TEST_OMETIFF_TIFFDATA_REORDERED) { + ASSERT_NO_THROW (test_ometiff_multitile_facade_volume("dim5_reordered.ome.tif", 1, 2, 3, 6, 8)); +} + +// 4D (rank-4): time-only and channel-only. +TEST(TEST_NYXUS, TEST_OMETIFF_4D_TZYX) { + ASSERT_NO_THROW (test_ometiff_addressing("dim4_tzyx.ome.tif", 2, 1, 4)); + ASSERT_NO_THROW (test_raw_ometiff_addressing("dim4_tzyx.ome.tif", 2, 1, 4)); +} +TEST(TEST_NYXUS, TEST_OMETIFF_4D_CZYX) { + ASSERT_NO_THROW (test_ometiff_addressing("dim4_czyx.ome.tif", 1, 3, 4)); + ASSERT_NO_THROW (test_raw_ometiff_addressing("dim4_czyx.ome.tif", 1, 3, 4)); +} + +// End-to-end through the wired volumetric consumer (scan_trivial_wholevolume). +TEST(TEST_NYXUS, TEST_OMETIFF_WHOLEVOLUME_CONSUMER) { + ASSERT_NO_THROW (test_ometiff_wholevolume_consumer("dim3_zyx.ome.tif", 4)); +} + +// Wired consumer reads the correct plane for every (channel, timeframe), not just (0,0). +TEST(TEST_NYXUS, TEST_OMETIFF_WHOLEVOLUME_CONSUMER_CT) { + ASSERT_NO_THROW (test_ometiff_wholevolume_consumer_ct("dim5.ome.tif", 2, 3, 4)); +} + +// TILED multi-plane OME-TIFF: the tile loaders map (z,c,t)->IFD (distinct from strip loaders). +TEST(TEST_NYXUS, TEST_OMETIFF_TILED_ADDRESSING) { + ASSERT_NO_THROW (test_ometiff_tiled_addressing()); +} +// Facade whole-volume assembly over the TILED path (open() routes tiled TIFF -> tile loader). +TEST(TEST_NYXUS, TEST_OMETIFF_TILED_FACADE_VOLUME) { + ASSERT_NO_THROW (test_ometiff_facade_volume("dim5_tiled.ome.tif", 2, 3, 4)); +} +// Multi-TILE planes (2x3 grid of 16x16 tiles): dim5_tiled above has ONE tile per plane, so +// it passes even when only tile (0,0) is read. This is the OME-TIFF counterpart of +// TEST_OMEZARR_MULTICHUNK_FACADE_VOLUME. Covers ImageLoader::assemble_volume... +TEST(TEST_NYXUS, TEST_OMETIFF_MULTITILE_FACADE_VOLUME) { + ASSERT_NO_THROW (test_ometiff_multitile_facade_volume("dim5_multitile.ome.tif", 1, 2, 2, 32, 48)); +} +// ...and RawImageLoader::for_each_voxel (the prescan), which had the same single-tile bug. +// Encoded values run 1..6144 over all (c,t); whole-slide, so the ROI is the whole volume. +TEST(TEST_NYXUS, TEST_OMETIFF_MULTITILE_PRESCAN) { + fs::path ip = ometiff_data_path("dim5_multitile.ome.tif"); + ASSERT_TRUE(fs::exists(ip)) << ip.string(); + + Environment e; + SlideProps p (ip.string(), ""); // whole-slide: no mask + ASSERT_TRUE(Nyxus::scan_slide_props(p, 3, e.anisoOptions, e.resultOptions.need_annotation())); + + EXPECT_DOUBLE_EQ(p.min_preroi_inten, 1.0); + EXPECT_DOUBLE_EQ(p.max_preroi_inten, 6144.0); + EXPECT_EQ(p.max_roi_area, (size_t)(48 * 32 * 2)); +} + +// PARTIAL edge tiles: 40x24 plane / 16 -> 3x2 tiles, last row-tile 8 tall, last col-tile 8 +// wide. Every multi-tile fixture above has plane dims that are exact multiples of the tile +// size, so the validH/validW seam clamp -- min(tileDim, fullDim-offset) -- had never run. +// Reading the exact value at every voxel checks the partial tiles are copied without garbage +// past the seam and without over-reading the tile buffer. Encoded 1..1920 over (c in 0,1). +TEST(TEST_NYXUS, TEST_OMETIFF_ODDTILE_FACADE_VOLUME) { + ASSERT_NO_THROW (test_ometiff_multitile_facade_volume("dim5_oddtile.ome.tif", 1, 2, 1, 40, 24)); +} +// The prescan (for_each_voxel) walks the same partial grid; its slide min/max and ROI area +// would be wrong if a partial tile were mis-clamped (a too-large validH double-counts voxels). +TEST(TEST_NYXUS, TEST_OMETIFF_ODDTILE_PRESCAN) { + fs::path ip = ometiff_data_path("dim5_oddtile.ome.tif"); + ASSERT_TRUE(fs::exists(ip)) << ip.string(); + + Environment e; + SlideProps p (ip.string(), ""); // whole-slide: no mask + ASSERT_TRUE(Nyxus::scan_slide_props(p, 3, e.anisoOptions, e.resultOptions.need_annotation())); + + EXPECT_DOUBLE_EQ(p.min_preroi_inten, 1.0); + EXPECT_DOUBLE_EQ(p.max_preroi_inten, 1920.0); // 1 + (((0*2+1)*1+0)*40+39)*24+23 + EXPECT_EQ(p.max_roi_area, (size_t)(24 * 40 * 1)); +} + +// Facade whole-volume assembly (load_volume loops Z into one X*Y*Z buffer). +TEST(TEST_NYXUS, TEST_OMETIFF_FACADE_VOLUME_3D) { + ASSERT_NO_THROW (test_ometiff_facade_volume("dim3_zyx.ome.tif", 1, 1, 4)); +} +TEST(TEST_NYXUS, TEST_OMETIFF_FACADE_VOLUME_5D) { + ASSERT_NO_THROW (test_ometiff_facade_volume("dim5.ome.tif", 2, 3, 4)); +} + +// Lower-rank OME-TIFF. +TEST(TEST_NYXUS, TEST_OMETIFF_3D_ZYX) { + ASSERT_NO_THROW (test_ometiff_addressing("dim3_zyx.ome.tif", 1, 1, 4)); +} +TEST(TEST_NYXUS, TEST_RAW_OMETIFF_3D_ZYX) { + ASSERT_NO_THROW (test_raw_ometiff_addressing("dim3_zyx.ome.tif", 1, 1, 4)); +} +TEST(TEST_NYXUS, TEST_OMETIFF_2D_YX) { + ASSERT_NO_THROW (test_ometiff_addressing("dim2_yx.ome.tif", 1, 1, 1)); +} + +// Plain multi-page TIFF (no OME-XML): the legacy page=Z fallback must still work. +TEST(TEST_NYXUS, TEST_OMETIFF_PLAIN_MULTIPAGE_FALLBACK) { + ASSERT_NO_THROW (test_ometiff_addressing("dim3_plain.tif", 1, 1, 4)); +} +TEST(TEST_NYXUS, TEST_RAW_OMETIFF_PLAIN_MULTIPAGE_FALLBACK) { + ASSERT_NO_THROW (test_raw_ometiff_addressing("dim3_plain.tif", 1, 1, 4)); +} + +// Negative: out-of-range channel/timeframe through the whole-volume facade must throw. +TEST(TEST_NYXUS, TEST_OMETIFF_LOAD_VOLUME_OUT_OF_RANGE) { + ASSERT_NO_THROW (test_ometiff_load_volume_out_of_range()); +} + +// Regression: a single-channel mask is reused across all intensity channels (not read OOB). +TEST(TEST_NYXUS, TEST_OMETIFF_MULTICHANNEL_MASK_PAIRING) { + ASSERT_NO_THROW (test_ometiff_multichannel_mask_pairing()); +} + +// Strip loaders advertise the OME C/T extents; the plain (non-OME) multi-page TIFF +// keeps C=T=1 (its pages are Z-slices, not channels/timeframes). +TEST(TEST_NYXUS, TEST_OMETIFF_CT_COUNTS) { + ASSERT_NO_THROW (test_ometiff_ct_counts("dim5.ome.tif", 2, 3, 4)); + ASSERT_NO_THROW (test_ometiff_ct_counts("dim4_tzyx.ome.tif", 2, 1, 4)); + ASSERT_NO_THROW (test_ometiff_ct_counts("dim4_czyx.ome.tif", 1, 3, 4)); + ASSERT_NO_THROW (test_ometiff_ct_counts("dim3_zyx.ome.tif", 1, 1, 4)); + ASSERT_NO_THROW (test_ometiff_ct_counts("dim3_plain.tif", 1, 1, 4)); +} + +// Negative: out-of-range Z/C/T plane index must throw. +TEST(TEST_NYXUS, TEST_OMETIFF_OUT_OF_RANGE_THROWS) { + ASSERT_NO_THROW (test_ometiff_out_of_range_throws()); +} + +// Illegal / adversarial: RGB / corrupt / missing files must be rejected cleanly. +TEST(TEST_NYXUS, TEST_OMETIFF_MALFORMED_THROWS) { + ASSERT_NO_THROW (test_ometiff_malformed_throws()); +} + + +// Regression: the 3D prescan must scan the WHOLE volume of EVERY (channel, timeframe), +// not one Z-plane of (c0,t0). dim5.ome.tif is C=3,T=2,Z=4,Y=6,X=8 encoding values 1..1152, +// so the slide intensity range must be exactly [1, 1152]. Before the fix the prescan +// (a) did a single load_tile(0,0) and then indexed W*H*D voxels off that ONE-plane buffer, +// reading out of bounds (observed range 0..44,465 of garbage), and (b) covered only +// (c0,t0) -> a range of 1..192, which under-sized intensity-indexed buffers for c>0 and +// segfaulted. The area check guards that ROI geometry is taken from the first pass only +// (otherwise it would be multiplied by n_channels*n_timeframes). +TEST(TEST_NYXUS, TEST_3D_PRESCAN_SLIDE_RANGE) { + fs::path ip = ometiff_data_path("dim5.ome.tif"); + ASSERT_TRUE(fs::exists(ip)) << ip.string(); + + Environment e; + SlideProps p (ip.string(), ""); // whole-slide: no mask + ASSERT_TRUE(Nyxus::scan_slide_props(p, 3, e.anisoOptions, e.resultOptions.need_annotation())); + + EXPECT_EQ(p.inten_channels, (size_t)3); + EXPECT_EQ(p.inten_time, (size_t)2); + EXPECT_EQ(p.volume_d, (size_t)4); + // the full encoded range across ALL (c,t) -- not garbage, and not just (c0,t0)'s 1..192 + EXPECT_DOUBLE_EQ(p.min_preroi_inten, 1.0); + EXPECT_DOUBLE_EQ(p.max_preroi_inten, 1152.0); + // geometry from the first pass only: one whole-slide ROI of exactly X*Y*Z voxels + EXPECT_EQ(p.max_roi_area, (size_t)(8 * 6 * 4)); +} + +// Regression: the 3D WHOLE-VOLUME reduce path. reduce_trivial_3d_wholevolume calls +// D3_VoxelIntensityFeatures::extract(), which used to invoke the 2-arg calculate() -- a stub +// that throws "illegal call" -- so EVERY 3D whole-volume featurization died before writing a +// row. The segmented path reduces via reduce(), which passes the Dataset, so nothing covered +// this. Mirrors featurize_wholevolume()'s vROI setup, then reduces. +TEST(TEST_NYXUS, TEST_3D_WHOLEVOLUME_REDUCE) { + fs::path ds = ometiff_data_path("dim3_zyx.ome.tif"); // 3D X8 Y6 Z4 + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + Environment e; + // enable the 3D intensity features so the reduce actually runs them + e.theFeatureSet.enableAll(false); + e.theFeatureSet.enableFeatures(D3_VoxelIntensityFeatures::featureset); + + // prescan the slide (whole-volume => no mask) + e.dataset.dataset_props.reserve(1); + SlideProps& sp = e.dataset.dataset_props.emplace_back(ds.string(), ""); + ASSERT_TRUE(Nyxus::scan_slide_props(sp, 3, e.anisoOptions, e.resultOptions.need_annotation())); + e.dataset.update_dataset_props_extrema(); + + // build the vROI exactly as featurize_wholevolume() does + FpImageOptions fp; + ImageLoader ilo; + ASSERT_TRUE(ilo.open(sp, fp)) << ds.string(); + LR vroi(1); + vroi.slide_idx = 0; + vroi.aux_area = sp.max_roi_area; + vroi.aabb.init_from_whd(sp.max_roi_w, sp.max_roi_h, sp.max_roi_d); + vroi.aux_min = (PixIntens)0; + vroi.aux_max = (PixIntens)(sp.max_preroi_inten - sp.min_preroi_inten); + ASSERT_NO_THROW(vroi.initialize_fvals()); + ASSERT_TRUE(Nyxus::scan_trivial_wholevolume(vroi, ds.string(), ilo, 0/*channel*/, 0/*timeframe*/)); + ASSERT_GT(vroi.raw_pixels_3D.size(), 0u); + vroi.aux_image_cube.allocate(vroi.aabb.get_width(), vroi.aabb.get_height(), vroi.aabb.get_z_depth()); + vroi.aux_image_cube.calculate_from_pixelcloud(vroi.raw_pixels_3D, vroi.aabb); + + // THE REGRESSION: this used to throw "illegal call of D3_VoxelIntensityFeatures::calculate" + ASSERT_NO_THROW(Nyxus::reduce_trivial_3d_wholevolume(e, vroi)); + + // and it must actually produce values (MAX >= MIN, both finite) + double vmin = vroi.get_fvals((int)Nyxus::Feature3D::MIN)[0]; + double vmax = vroi.get_fvals((int)Nyxus::Feature3D::MAX)[0]; + EXPECT_GE(vmax, vmin); + EXPECT_GT(vmax, 0.0); + ilo.close(); +} + +// Regression: an OVERSIZED whole volume of a format that delivers plane-by-plane (e.g. OME-TIFF) +// must now featurize SUCCESSFULLY out-of-core, producing the SAME feature values as the in-RAM +// (fitting) run of the identical file -- not fail, and not emit a zero row. This supersedes the +// old TEST_3D_WHOLEVOLUME_OVERSIZED_FAILS_LOUDLY premise: before workflow_3d_whole.cpp's oversized +// branch streamed via populate_3d_voxel_cloud/run_3d_ooc_features, NO whole-volume streaming path +// existed at all, so oversized-but-streamable volumes failed loudly (a deliberate, validated +// fallback at the time). Now the only remaining loud-fail case is a genuinely unstreamable format +// (whole-4D-in-one-read, e.g. NIfTI) -- see TEST_3D_WHOLEVOLUME_UNSTREAMABLE_FORMAT_FAILS_LOUDLY. +TEST(TEST_NYXUS, TEST_3D_WHOLEVOLUME_OVERSIZED_STREAMS_OOC) { + fs::path ds = ometiff_data_path("dim3_zyx.ome.tif"); // 3D X8 Y6 Z4 + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + auto run_and_read_row = [&](bool oversized, const fs::path& outdir) -> std::string + { + fs::remove_all(outdir); fs::create_directories(outdir); + Environment e; + e.theFeatureSet.enableAll(false); + e.theFeatureSet.enableFeatures(D3_VoxelIntensityFeatures::featureset); + e.theFeatureSet.enableFeatures(D3_SurfaceFeature::featureset); + e.theFeatureSet.enableFeatures(D3_GLCM_feature::featureset); + // Cover every streaming 3D texture family so the out-of-core paths are held byte-exact + // against the in-RAM path (not just intensity/surface/GLCM) + e.theFeatureSet.enableFeatures(D3_GLDM_feature::featureset); + e.theFeatureSet.enableFeatures(D3_GLRLM_feature::featureset); + e.theFeatureSet.enableFeatures(D3_GLSZM_feature::featureset); + e.theFeatureSet.enableFeatures(D3_GLDZM_feature::featureset); + e.theFeatureSet.enableFeatures(D3_NGLDM_feature::featureset); + e.theFeatureSet.enableFeatures(D3_NGTDM_feature::featureset); + // The full sequence main_nyxus.cpp runs before any workflow (theFeatureMgr.compile() -> + // apply_user_selection() -> init_feature_classes() -> compile_feature_settings()). Skipping + // theFeatureMgr setup leaves get_num_requested_features()==0, so run_3d_ooc_features's loop + // never executes and every feature stays at its zero-initialized default (first symptom: an + // all-zero OOC row). Skipping compile_feature_settings() leaves fsett_D3_* at size 0, so any + // STNGS_*(s) macro access (e.g. surface's STNGS_SINGLEROI) reads out of bounds -- crashed + // (SEH 0xc0000005) before this was added. + EXPECT_TRUE(e.theFeatureMgr.compile()); + e.theFeatureMgr.apply_user_selection (e.theFeatureSet); + EXPECT_TRUE(e.theFeatureMgr.init_feature_classes()); + e.compile_feature_settings(); + EXPECT_TRUE(e.set_ram_limit(oversized ? 0 : 64)); // 0 -> force oversized; 64 MB comfortably fits this tiny (X8 Y6 Z4) volume + e.output_dir = outdir.string(); + + std::vector ifiles{ ds.string() }; + auto [ok, erm] = Nyxus::processDataset_3D_wholevolume(e, ifiles, 1, Nyxus::SaveOption::saveCSV, outdir.string()); + EXPECT_TRUE(ok) << (erm ? *erm : std::string("(no error message)")); + + std::string row; + size_t datarows = 0; + for (auto& de : fs::directory_iterator(outdir)) + if (de.path().extension() == ".csv") + { + std::ifstream f(de.path()); std::string header, ln; + std::getline(f, header); + while (std::getline(f, ln)) if (!ln.empty()) { row = ln; ++datarows; } + } + EXPECT_EQ(datarows, (size_t)1) << "expected exactly one feature row"; + fs::remove_all(outdir); + return row; + }; + + std::string ooc_row = run_and_read_row(true, fs::temp_directory_path() / "nyxus_wv_ooc_test"); + std::string ram_row = run_and_read_row(false, fs::temp_directory_path() / "nyxus_wv_ram_test"); + + ASSERT_FALSE(ooc_row.empty()); + ASSERT_FALSE(ram_row.empty()); + EXPECT_EQ(ooc_row, ram_row) << "the out-of-core whole-volume row must match the in-RAM row exactly"; +} + +// Regression: a whole volume whose loader delivers the ENTIRE X*Y*Z*T blob in one read (e.g. +// NIfTI -- see ImageLoader::stream_volume_planes) cannot stream plane-by-plane, so an oversized +// NIfTI whole volume must still fail loudly (no streaming path exists for it) rather than crash or +// emit a silent zero row. ram_limit=0 forces oversized regardless of the fixture's actual size. +TEST(TEST_NYXUS, TEST_3D_WHOLEVOLUME_UNSTREAMABLE_FORMAT_FAILS_LOUDLY) { + fs::path p(__FILE__); + fs::path ds(p.parent_path().string() + fs::path("/data/hounsfield/ct3d_int16.nii").make_preferred().string()); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + fs::path outdir = fs::temp_directory_path() / "nyxus_wv_nifti_ooc_test"; + fs::remove_all(outdir); fs::create_directories(outdir); + + Environment e; + e.theFeatureSet.enableAll(false); + e.theFeatureSet.enableFeatures(D3_VoxelIntensityFeatures::featureset); + ASSERT_TRUE(e.theFeatureMgr.compile()); + e.theFeatureMgr.apply_user_selection (e.theFeatureSet); + ASSERT_TRUE(e.theFeatureMgr.init_feature_classes()); + e.compile_feature_settings(); + ASSERT_TRUE(e.set_ram_limit(0)); // force oversized regardless of this small fixture's real size + + std::vector ifiles{ ds.string() }; + auto [ok, erm] = Nyxus::processDataset_3D_wholevolume(e, ifiles, 1, Nyxus::SaveOption::saveCSV, outdir.string()); + + EXPECT_FALSE(ok) << "an oversized NIfTI whole volume has no streaming path and must fail loudly"; + + size_t datarows = 0; + for (auto& de : fs::directory_iterator(outdir)) + if (de.path().extension() == ".csv") + { + std::ifstream f(de.path()); std::string ln; size_t n = 0; + while (std::getline(f, ln)) if (!ln.empty()) ++n; + if (n) datarows += n - 1; // minus header + } + EXPECT_EQ(datarows, (size_t)0) << "no feature row should be written for an unstreamable oversized volume"; + fs::remove_all(outdir); +} + +// Regression (found by running nyxus under a hard memory cap): the whole-volume oversized check +// must use the 3D footprint estimator (W*H*D for the image cube), not the 2D one (W*H). The 2D +// estimator ignores depth, so it under-counted a volume's memory by ~depth x, let oversized +// volumes slip through the "trivial" path, and they OOM-crashed under a real memory limit even +// with a matching --ramLimit. This pins that the 3D estimator accounts for depth (the 2D one +// does not) so featurize_wholevolume's switch to get_ram_footprint_estimate_3D stays correct. +TEST(TEST_NYXUS, TEST_3D_RAM_FOOTPRINT_COUNTS_DEPTH) { + // Two ROIs with identical W/H and voxel count, differing ONLY in bounding-box depth. This + // isolates depth's effect on each estimator. + LR flat(1); + flat.aabb.init_from_whd(64, 64, 1); + flat.aux_area = 4096; + LR tall(1); + tall.aabb.init_from_whd(64, 64, 64); + tall.aux_area = 4096; + + // the 2D estimator's image-matrix term is W*H -> it IGNORES depth: identical for both + EXPECT_EQ(flat.get_ram_footprint_estimate(1), tall.get_ram_footprint_estimate(1)); + + // the 3D estimator's image-cube term is W*H*D -> the 64x-deeper bbox is far larger. This is + // the term the 2D estimator missed, which under-counted whole volumes and let them OOM. + EXPECT_GT(tall.get_ram_footprint_estimate_3D(1), flat.get_ram_footprint_estimate_3D(1) * 10) + << "3D footprint estimator is not counting depth"; +} + +// Regression (found while chasing the anisotropic-resampling hang, TEST_3D_SEGMENTED_ANISOTROPIC_* +// above): both footprint estimators computed (n_rois - 1) * sizeof(int) for the "neighbors" term. +// processTrivialRois_3D (and the 2D/2.5D siblings) call this with an in-progress BATCH count +// (Pending.size()), which is 0 on every batch's first item -- size_t(0-1) underflows to SIZE_MAX, +// and the subsequent multiply overflows to another huge wrapped value, silently misrouting even a +// tiny single-ROI batch through the "oversized, scan immediately" path instead of genuinely +// batching. n_rois==0 must mean "zero other ROIs, so 0 bytes for the neighbors term", not garbage. +TEST(TEST_NYXUS, TEST_RAM_FOOTPRINT_ESTIMATE_ZERO_ROIS_DOES_NOT_UNDERFLOW) { + LR r(1); + r.aabb.init_from_whd(8, 8, 4); + r.aux_area = 64; + + size_t with_zero = r.get_ram_footprint_estimate(0); + size_t with_one = r.get_ram_footprint_estimate(1); // (1-1)=0 neighbors bytes too -- same base cost + EXPECT_EQ(with_zero, with_one) << "n_rois=0 and n_rois=1 both contribute 0 neighbor bytes"; + // sanity ceiling: a real (non-underflowed) footprint for an 8x8x4 ROI is a few KB, not + // anywhere near what (size_t)(0-1)*sizeof(int) would produce (~16 exabytes) + EXPECT_LT(with_zero, (size_t)1'000'000) << "n_rois=0 must not underflow into an astronomical value"; + + size_t with_zero_3d = r.get_ram_footprint_estimate_3D(0); + size_t with_one_3d = r.get_ram_footprint_estimate_3D(1); + EXPECT_EQ(with_zero_3d, with_one_3d); + EXPECT_LT(with_zero_3d, (size_t)1'000'000); +} + +// Regression-guard: processNontrivialRois_3D's per-feature out-of-core dispatch +// (phase3_3d.cpp) must throw for any 3D FeatureMethod NOT covered by is_3d_ooc_supported() -- +// otherwise a future feature added without a streaming osized_calculate would silently read +// raw_voxels_NT (which OOC never populates for it) via the base FeatureMethod::osized_scan_whole_image +// default, producing a wrong/zero row instead of an actionable error. Every CURRENT 3D feature class +// is supported, so there is no live "unsupported" feature to exercise this through the normal +// featurize path; this pins the ALLOW-LIST FUNCTION ITSELF directly, using a minimal stand-in +// FeatureMethod that is deliberately never added to the allow-list, alongside real supported classes. +class DummyUnsupported3DFeature : public FeatureMethod +{ +public: + DummyUnsupported3DFeature() : FeatureMethod("DummyUnsupported3DFeature") {} + void calculate (LR&, const Fsettings&) override {} + void osized_add_online_pixel (size_t, size_t, uint32_t) override {} + void osized_calculate (LR&, const Fsettings&, ImageLoader&) override {} + void save_value (std::vector>&) override {} +}; + +TEST(TEST_NYXUS, TEST_3D_OOC_GUARD_REJECTS_UNSUPPORTED_FEATURE) { + DummyUnsupported3DFeature unsupported; + EXPECT_FALSE(Nyxus::is_3d_ooc_supported(&unsupported)) + << "a 3D feature class outside the allow-list must be rejected by the OOC guard"; + + D3_VoxelIntensityFeatures intensityFeature; + EXPECT_TRUE(Nyxus::is_3d_ooc_supported(&intensityFeature)) + << "a real streaming-supported 3D feature (intensity) must be accepted"; + + D3_GLCM_feature glcmFeature; + EXPECT_TRUE(Nyxus::is_3d_ooc_supported(&glcmFeature)) + << "a real streaming-supported 3D texture feature (GLCM) must be accepted"; +} + +// Regression: separatecsv derives ONE output path per slide, but the CSV sinks are invoked +// once per (channel, timeframe) plane and used to open that path with mode "w" every time -- +// so each plane truncated the one before it and the file ended up holding only the LAST +// channel. Confirmed against the pre-fix build, where this file had 1 data row (c_index=1) +// instead of 2. The t_index/c_index columns exist precisely so the planes can coexist as rows. +TEST(TEST_NYXUS, TEST_CSV_MULTICHANNEL_NO_OVERWRITE) { + fs::path ds = ometiff_data_path("dim3_zyx.ome.tif"); // 3D X8 Y6 Z4 + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + fs::path outdir = fs::temp_directory_path() / "nyxus_csv_ct_test"; + fs::remove_all(outdir); + fs::create_directories(outdir); + + Environment e; + e.separateCsv = true; // the mode that overwrote (and the default) + e.output_dir = outdir.string(); + e.theFeatureSet.enableAll(false); + e.theFeatureSet.enableFeatures(D3_VoxelIntensityFeatures::featureset); + + e.dataset.dataset_props.reserve(1); + SlideProps& sp = e.dataset.dataset_props.emplace_back(ds.string(), ""); + ASSERT_TRUE(Nyxus::scan_slide_props(sp, 3, e.anisoOptions, e.resultOptions.need_annotation())); + e.dataset.update_dataset_props_extrema(); + + LR vroi(1); + vroi.slide_idx = 0; + vroi.aux_area = sp.max_roi_area; + vroi.aabb.init_from_whd(sp.max_roi_w, sp.max_roi_h, sp.max_roi_d); + ASSERT_NO_THROW(vroi.initialize_fvals()); + + // Two channel planes of the SAME slide, exactly as the whole-volume workflow emits them + ASSERT_TRUE(Nyxus::save_features_2_csv_wholeslide (e, vroi, ds.string(), "", outdir.string(), 0, 0)); + ASSERT_TRUE(Nyxus::save_features_2_csv_wholeslide (e, vroi, ds.string(), "", outdir.string(), 0, 1)); + + // Read the single file back + std::vector lines; + { + std::ifstream f (Nyxus::get_feature_output_fname (e, ds.string(), "")); + ASSERT_TRUE(f.good()); + std::string ln; + while (std::getline(f, ln)) + if (!ln.empty()) + lines.push_back(ln); + } + + ASSERT_EQ(lines.size(), (size_t)3) << "expected 1 header + one row per channel plane"; + EXPECT_NE(lines[0].find("\"c_index\""), std::string::npos) << "line 0 must be the header"; + // ...and the header must appear exactly once, not once per plane + EXPECT_EQ(lines[1].find("\"c_index\""), std::string::npos); + EXPECT_EQ(lines[2].find("\"c_index\""), std::string::npos); + + fs::remove_all(outdir); +} + +// Physical-calibration logic (negative + positive). resolve_slide_anisotropy +// must NOT engage the anisotropic (resampling) path unless it's genuinely warranted: +// - flag off -> false, (1,1,1) even with anisotropic spacing +// - degenerate spacing (a 0 axis) -> false, (1,1,1) (guarded, no div-by-zero) +// - isotropic spacing (all equal) -> false, (1,1,1) (nothing to correct) +// - out-of-range slide index -> false, (1,1,1) (no OOB read) +// - real anisotropic spacing -> true, ratios normalized so min == 1 +// - explicit --aniso* -> true, the CLI values (win over physical) +TEST(TEST_NYXUS, TEST_RESOLVE_SLIDE_ANISOTROPY) { + Environment e; + e.use_physical_spacing_ = true; // opt-in on; anisoOptions stays un-customized + e.dataset.dataset_props.clear(); + SlideProps p; // ctor sets phys_x/y/z = 1.0 + e.dataset.dataset_props.push_back(p); + double ax = -1, ay = -1, az = -1; + + // degenerate: a zero-length axis must not divide-by-zero -> isotropic fallback + e.dataset.dataset_props[0].phys_x = 1.0; e.dataset.dataset_props[0].phys_y = 1.0; e.dataset.dataset_props[0].phys_z = 0.0; + EXPECT_FALSE(Nyxus::resolve_slide_anisotropy(e, 0, ax, ay, az)); + EXPECT_DOUBLE_EQ(ax, 1.0); EXPECT_DOUBLE_EQ(ay, 1.0); EXPECT_DOUBLE_EQ(az, 1.0); + + // isotropic but non-unit spacing -> normalized to (1,1,1) -> no anisotropic path + e.dataset.dataset_props[0].phys_x = 2.0; e.dataset.dataset_props[0].phys_y = 2.0; e.dataset.dataset_props[0].phys_z = 2.0; + EXPECT_FALSE(Nyxus::resolve_slide_anisotropy(e, 0, ax, ay, az)); + EXPECT_DOUBLE_EQ(az, 1.0); + + // out-of-range slide index -> safe (no dataset_props[99] read) + EXPECT_FALSE(Nyxus::resolve_slide_anisotropy(e, 99, ax, ay, az)); + EXPECT_DOUBLE_EQ(ax, 1.0); + + // genuinely anisotropic voxels (z 4x thicker) -> engage, ratio-normalized min == 1 + e.dataset.dataset_props[0].phys_x = 0.5; e.dataset.dataset_props[0].phys_y = 0.5; e.dataset.dataset_props[0].phys_z = 2.0; + EXPECT_TRUE(Nyxus::resolve_slide_anisotropy(e, 0, ax, ay, az)); + EXPECT_DOUBLE_EQ(ax, 1.0); EXPECT_DOUBLE_EQ(ay, 1.0); EXPECT_DOUBLE_EQ(az, 4.0); + + // flag OFF -> never engage, even with anisotropic spacing present + e.use_physical_spacing_ = false; + EXPECT_FALSE(Nyxus::resolve_slide_anisotropy(e, 0, ax, ay, az)); + EXPECT_DOUBLE_EQ(az, 1.0); +} + +// TEST_RESOLVE_SLIDE_ANISOTROPY covers the DECISION (physical spacing -> ratios). This covers +// that the resolved ratios actually RESCALE ROI geometry end-to-end: the 3D prescan's +// anisotropic branch (make_anisotropic_aabb 3-arg -> AABB::apply_anisotropy) was never +// exercised -- every other test uses make_nonanisotropic_aabb. A customized az=4 must scale the +// ROI's z-depth ~4x while leaving x/y (ax=ay=1) unchanged; without applying anisotropy the +// depth would be identical to the isotropic run. dim3_mask's ROI spans all Z (depth 4). +TEST(TEST_NYXUS, TEST_3D_ANISOTROPY_RESCALES_ROI_DEPTH) { + fs::path ip = ometiff_data_path("dim5.ome.tif"); + fs::path mp = ometiff_data_path("dim3_mask.ome.tif"); + ASSERT_TRUE(fs::exists(ip) && fs::exists(mp)); + Environment e; + + SlideProps iso (ip.string(), mp.string()); + AnisotropyOptions aniso_off; // un-customized -> isotropic AABB + ASSERT_FALSE(aniso_off.customized()); + ASSERT_TRUE(Nyxus::scan_slide_props(iso, 3, aniso_off, e.resultOptions.need_annotation())); + + SlideProps ani (ip.string(), mp.string()); + AnisotropyOptions aniso_z4; + aniso_z4.set_aniso_z(4.0); // z 4x thicker + ASSERT_TRUE(aniso_z4.customized()); + ASSERT_TRUE(Nyxus::scan_slide_props(ani, 3, aniso_z4, e.resultOptions.need_annotation())); + + EXPECT_GT(ani.max_roi_d, iso.max_roi_d) << "z-anisotropy did not rescale ROI depth"; + EXPECT_GE(ani.max_roi_d, iso.max_roi_d * 3) << "z-depth not scaled ~4x"; + EXPECT_EQ(ani.max_roi_w, iso.max_roi_w) << "x (ax=1) must be unchanged"; + EXPECT_EQ(ani.max_roi_h, iso.max_roi_h) << "y (ay=1) must be unchanged"; +} + +// Regression: the whole-volume anisotropic scan (scan_trivial_wholevolume_anisotropic) +// previously (a) clobbered its own for-loop counter with the physical voxel index, which +// could run far past nVox iterations before the loop's own exit condition was ever +// satisfied again -- a hang, not just a slowdown; (b) indexed rows by fullH (height) instead +// of fullW (width), reading wrong voxels; (c) left vroi.aabb/aux_area at the PRE-resample +// physical values, so aux_image_cube (sized from the stale aabb) was allocated too small for +// the resampled cloud -- an out-of-bounds write -- and MEAN (which divides by aux_area) came +// out exactly 4x too large. Nothing exercised this combination before (every other 3D +// anisotropy test only covers the prescan's aabb, not the actual featurize+reduce). MIN/MAX +// are structurally invariant to nearest-neighbor upsampling, but so is MEAN under this scan's +// truncation-based mapping (every physical voxel is duplicated the SAME number of times) -- +// so it must match the isotropic run exactly, not just "look plausible". +TEST(TEST_NYXUS, TEST_3D_WHOLEVOLUME_ANISOTROPIC_REDUCE_MATCHES_ISOTROPIC) { + fs::path ds = ometiff_data_path("dim3_zyx.ome.tif"); // 3D X8 Y6 Z4 + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + Environment e; + e.theFeatureSet.enableAll(false); + e.theFeatureSet.enableFeatures(D3_VoxelIntensityFeatures::featureset); + + e.dataset.dataset_props.reserve(1); + SlideProps& sp = e.dataset.dataset_props.emplace_back(ds.string(), ""); + ASSERT_TRUE(Nyxus::scan_slide_props(sp, 3, e.anisoOptions, e.resultOptions.need_annotation())); + e.dataset.update_dataset_props_extrema(); + // force anisotropic calibration (z 4x thicker than x/y) regardless of the fixture's own metadata + sp.phys_x = 0.5; sp.phys_y = 0.5; sp.phys_z = 2.0; + e.use_physical_spacing_ = true; + + double ax, ay, az; + ASSERT_TRUE(Nyxus::resolve_slide_anisotropy(e, 0, ax, ay, az)); + ASSERT_DOUBLE_EQ(ax, 1.0); ASSERT_DOUBLE_EQ(ay, 1.0); ASSERT_DOUBLE_EQ(az, 4.0); + + FpImageOptions fp; + + // anisotropic run + ImageLoader ilo_a; + ASSERT_TRUE(ilo_a.open(sp, fp)) << ds.string(); + LR vroi_a(1); + vroi_a.slide_idx = 0; + vroi_a.aux_area = sp.max_roi_area; + vroi_a.aabb.init_from_whd(sp.max_roi_w, sp.max_roi_h, sp.max_roi_d); + vroi_a.aux_min = (PixIntens)0; + vroi_a.aux_max = (PixIntens)(sp.max_preroi_inten - sp.min_preroi_inten); + ASSERT_NO_THROW(vroi_a.initialize_fvals()); + ASSERT_TRUE(Nyxus::scan_trivial_wholevolume_anisotropic(vroi_a, ds.string(), ilo_a, ax, ay, az, 0, 0)); + // the fix under test (mirrors workflow_3d_whole.cpp's featurize_triv_wholevolume): + vroi_a.aabb.update_from_voxelcloud(vroi_a.raw_pixels_3D); + vroi_a.aux_area = (unsigned int) vroi_a.raw_pixels_3D.size(); + vroi_a.aux_image_cube.allocate(vroi_a.aabb.get_width(), vroi_a.aabb.get_height(), vroi_a.aabb.get_z_depth()); + ASSERT_NO_THROW(vroi_a.aux_image_cube.calculate_from_pixelcloud(vroi_a.raw_pixels_3D, vroi_a.aabb)); + ASSERT_NO_THROW(Nyxus::reduce_trivial_3d_wholevolume(e, vroi_a)); + ilo_a.close(); + + // isotropic baseline, same fixture + ImageLoader ilo_i; + ASSERT_TRUE(ilo_i.open(sp, fp)) << ds.string(); + LR vroi_i(1); + vroi_i.slide_idx = 0; + vroi_i.aux_area = sp.max_roi_area; + vroi_i.aabb.init_from_whd(sp.max_roi_w, sp.max_roi_h, sp.max_roi_d); + vroi_i.aux_min = (PixIntens)0; + vroi_i.aux_max = (PixIntens)(sp.max_preroi_inten - sp.min_preroi_inten); + ASSERT_NO_THROW(vroi_i.initialize_fvals()); + ASSERT_TRUE(Nyxus::scan_trivial_wholevolume(vroi_i, ds.string(), ilo_i, 0, 0)); + vroi_i.aux_image_cube.allocate(vroi_i.aabb.get_width(), vroi_i.aabb.get_height(), vroi_i.aabb.get_z_depth()); + ASSERT_NO_THROW(vroi_i.aux_image_cube.calculate_from_pixelcloud(vroi_i.raw_pixels_3D, vroi_i.aabb)); + ASSERT_NO_THROW(Nyxus::reduce_trivial_3d_wholevolume(e, vroi_i)); + ilo_i.close(); + + // the resampled cloud really is ~4x bigger (z upsampled), not stuck at the physical count + EXPECT_GE(vroi_a.raw_pixels_3D.size(), vroi_i.raw_pixels_3D.size() * 3); + + double mean_a = vroi_a.get_fvals((int)Nyxus::Feature3D::MEAN)[0]; + double mean_i = vroi_i.get_fvals((int)Nyxus::Feature3D::MEAN)[0]; + EXPECT_DOUBLE_EQ(mean_a, mean_i) << "MEAN must be resampling-invariant under uniform duplication"; + EXPECT_DOUBLE_EQ(vroi_a.get_fvals((int)Nyxus::Feature3D::MIN)[0], vroi_i.get_fvals((int)Nyxus::Feature3D::MIN)[0]); + EXPECT_DOUBLE_EQ(vroi_a.get_fvals((int)Nyxus::Feature3D::MAX)[0], vroi_i.get_fvals((int)Nyxus::Feature3D::MAX)[0]); +} + +// Regression (segmented counterpart): processTrivialRois_3D's anisotropic branch +// (scanTrivialRois_3D_anisotropic) populates raw_pixels_3D with the RESAMPLED voxel cloud, +// but aux_area (set during Phase 1 from the PHYSICAL, pre-resample voxel count) was never +// updated to match -- caught in two places (the main batch loop AND the "remaining pending" +// cleanup block are near-identical but NOT textually identical, so fixing one via a +// find-and-replace silently missed the other). MEAN (and anything else that divides by +// aux_area) was off by the resampling factor. aux_area must always equal the actual cloud size. +TEST(TEST_NYXUS, TEST_3D_SEGMENTED_ANISOTROPIC_AUX_AREA_MATCHES_VOXELCLOUD) { + fs::path ip = ometiff_data_path("dim5.ome.tif"); + fs::path mp = ometiff_data_path("dim3_mask.ome.tif"); + ASSERT_TRUE(fs::exists(ip) && fs::exists(mp)); + + Environment e; + e.theFeatureSet.enableAll(false); + e.theFeatureSet.enableFeatures(D3_VoxelIntensityFeatures::featureset); + + e.dataset.dataset_props.reserve(1); + SlideProps& sp = e.dataset.dataset_props.emplace_back(ip.string(), mp.string()); + ASSERT_TRUE(Nyxus::scan_slide_props(sp, 3, e.anisoOptions, e.resultOptions.need_annotation())); + e.dataset.update_dataset_props_extrema(); + sp.phys_x = 0.5; sp.phys_y = 0.5; sp.phys_z = 2.0; // force anisotropic (z 4x) + e.use_physical_spacing_ = true; + + clear_slide_rois (e.uniqueLabels, e.roiData); + ASSERT_TRUE(gatherRoisMetrics_3D(e, 0, ip.string(), mp.string(), 0, 0)); + ASSERT_GT(e.uniqueLabels.size(), 0u); + std::vector labels (e.uniqueLabels.begin(), e.uniqueLabels.end()); + std::unordered_map physical_area; // Phase 1's PRE-resample count, per label + for (auto lab : labels) + { + e.roiData[lab].initialize_fvals(); + physical_area[lab] = e.roiData[lab].aux_area; + } + + double ax, ay, az; + ASSERT_TRUE(Nyxus::resolve_slide_anisotropy(e, 0, ax, ay, az)); + ASSERT_DOUBLE_EQ(az, 4.0); + + // Call the scan directly (bypassing processTrivialRois_3D's batching, which has its own + // unrelated, pre-existing bug: get_ram_footprint_estimate(Pending.size()) underflows when + // Pending.size()==0 on the very first loop iteration, size_t(0-1)*sizeof(int) wrapping to + // an astronomical value that can route even a tiny single-ROI batch through the "oversized" + // immediate-scan branch unpredictably -- a separate footprint-estimation bug, not what this + // test targets) -- exercises the exact fix under test (see the identical logic and its + // rationale at both of processTrivialRois_3D's call sites in phase2_3d.cpp). + ASSERT_TRUE(Nyxus::scanTrivialRois_3D_anisotropic(e, labels, ip.string(), mp.string(), 0, 0, ax, ay, az)); + for (auto lab : labels) + { + LR& r = e.roiData[lab]; + r.aabb.update_from_voxelcloud(r.raw_pixels_3D); + r.aux_area = (unsigned int) r.raw_pixels_3D.size(); + } + + for (auto lab : labels) + { + LR& r = e.roiData[lab]; + EXPECT_GT(r.raw_pixels_3D.size(), 0u) << "label " << lab; + EXPECT_EQ(r.aux_area, r.raw_pixels_3D.size()) + << "label " << lab << ": aux_area must track the RESAMPLED cloud, not the stale physical count"; + // resampling z 4x must have grown the cloud past the PRE-resample physical count (not + // an exact 4x -- the rounding-based nearest-neighbor mapping under- or over-represents + // the boundary slice by up to one duplication step, so the growth factor isn't clean) + EXPECT_GT(r.raw_pixels_3D.size(), physical_area[lab]) + << "label " << lab << ": resampling did not grow the cloud past its physical count of " << physical_area[lab]; + } +} + +int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); int ret = RUN_ALL_TESTS(); diff --git a/tests/test_arrow_mechanics.h b/tests/test_arrow_mechanics.h index 9c5dc0f5..e5abacec 100644 --- a/tests/test_arrow_mechanics.h +++ b/tests/test_arrow_mechanics.h @@ -64,42 +64,34 @@ std::shared_ptr create_features_table(const std::vector> fields; + // FIX (IO): 3 leading string columns (intensity, mask, phys_unit) + ROI_label int32 + float64 rest + const int num_str_cols = 3; fields.push_back(arrow::field(header[0], arrow::utf8())); fields.push_back(arrow::field(header[1], arrow::utf8())); - fields.push_back(arrow::field(header[2], arrow::int32())); + fields.push_back(arrow::field(header[2], arrow::utf8())); + fields.push_back(arrow::field(header[3], arrow::int32())); - for (int i = 3; i < header.size(); ++i) + for (int i = 4; i < header.size(); ++i) { fields.push_back(arrow::field(header[i], arrow::float64())); } auto schema = arrow::schema(fields); - arrow::StringBuilder string_builder_0; - - std::vector temp_string_vec1(string_columns.size()/2); - std::vector temp_string_vec2(string_columns.size()/2); - - for (int i = 0; i < string_columns.size(); i+=2) { - temp_string_vec1[i/2] = string_columns[i]; - temp_string_vec2[i/2] = string_columns[i+1]; - } - - PARQUET_THROW_NOT_OK(string_builder_0.AppendValues(temp_string_vec1)); - - arrow::StringBuilder string_builder_1; - - PARQUET_THROW_NOT_OK(string_builder_1.AppendValues(temp_string_vec2)); - - std::shared_ptr array_0, array_1; - - PARQUET_THROW_NOT_OK(string_builder_0.Finish(&array_0)); - PARQUET_THROW_NOT_OK(string_builder_1.Finish(&array_1)); - std::vector> arrays; - arrays.push_back(array_0); - arrays.push_back(array_1); + // de-interleave the flat string buffer (row-major, num_str_cols per row) into one array per column + for (int c = 0; c < num_str_cols; ++c) + { + std::vector col(number_of_rows); + for (int r = 0; r < number_of_rows; ++r) + col[r] = string_columns[r * num_str_cols + c]; + arrow::StringBuilder sb; + PARQUET_THROW_NOT_OK(sb.AppendValues(col)); + std::shared_ptr arr; + PARQUET_THROW_NOT_OK(sb.Finish(&arr)); + arrays.push_back(arr); + } // add labels arrow::Int32Builder labels_builder; @@ -224,8 +216,13 @@ void test_arrow() for (const auto& row : row_data) { string_columns.push_back(std::get<0>(row)[0]); string_columns.push_back(std::get<0>(row)[1]); + string_columns.push_back(std::get<0>(row)[2]); // phys_unit (FIX (IO): 3rd leading string) numeric_columns.push_back(std::get<1>(row)); // ROI label - numeric_columns.push_back(std::get<2>(row)); // time + numeric_columns.push_back(std::get(row)); // time + numeric_columns.push_back(std::get(row)); // channel (FIX (IO): c_index column) + numeric_columns.push_back(std::get(row)); // phys_x (FIX (IO)) + numeric_columns.push_back(std::get(row)); // phys_y + numeric_columns.push_back(std::get(row)); // phys_z for (const auto& data : std::get(row)) { numeric_columns.push_back(data); } @@ -295,8 +292,13 @@ void test_parquet() { for (const auto& row : row_data) { string_columns.push_back(std::get<0>(row)[0]); string_columns.push_back(std::get<0>(row)[1]); + string_columns.push_back(std::get<0>(row)[2]); // phys_unit (FIX (IO): 3rd leading string) numeric_columns.push_back(std::get<1>(row)); // ROI label - numeric_columns.push_back(std::get<2>(row)); // time + numeric_columns.push_back(std::get(row)); // time + numeric_columns.push_back(std::get(row)); // channel (FIX (IO): c_index column) + numeric_columns.push_back(std::get(row)); // phys_x (FIX (IO)) + numeric_columns.push_back(std::get(row)); // phys_y + numeric_columns.push_back(std::get(row)); // phys_z for (const auto& data : std::get(row)) { numeric_columns.push_back(data); } diff --git a/tests/test_data.h b/tests/test_data.h index cfaf3526..3954e3be 100644 --- a/tests/test_data.h +++ b/tests/test_data.h @@ -689,7 +689,7 @@ const static NyxusPixel roiDataForPerimeterTest[] = { }; const static std::tuple, std::vector> features888{ - std::make_tuple(std::vector{"intensity_image", "mask_image", "ROI_label", "time", "COV", "COVERED_IMAGE_INTENSITY_RANGE", "ENERGY", + std::make_tuple(std::vector{"intensity_image", "mask_image", "phys_unit", "ROI_label", "time", "c_index", "phys_x", "phys_y", "phys_z", "COV", "COVERED_IMAGE_INTENSITY_RANGE", "ENERGY", "ENTROPY", "EXCESS_KURTOSIS", "HYPERFLATNESS", "HYPERSKEWNESS", "INTEGRATED_INTENSITY", "INTERQUARTILE_RANGE", "KURTOSIS", "MAX", "MEAN", "MEAN_ABSOLUTE_DEVIATION", "MEDIAN", "MEDIAN_ABSOLUTE_DEVIATION", "MIN", "MODE", "P01", "P10", "P25", "P75", "P90", "P99", "QCOD", "RANGE", "ROBUST_MEAN", "ROBUST_MEAN_ABSOLUTE_DEVIATION", "ROOT_MEAN_SQUARED", "SKEWNESS", "STANDARD_DEVIATION", @@ -697,36 +697,44 @@ const static NyxusPixel roiDataForPerimeterTest[] = { std::vector{ FTABLE_RECORD { std::make_tuple( - std::vector{"Segmentation0", "Intensity0"}, + std::vector{"Segmentation0", "Intensity0", ""}, 1, DEFAULT_T_INDEX, + DEFAULT_C_INDEX, + 1.0, 1.0, 1.0, std::vector {0.6550544230766455, 0.8333333333333334, 245.0, 1.4577174691301484, -1.4129028124799794, 2.8044001823929605, 1.0173315649829293, 59.0, 3.009722222222222, 1.5870971875200206, 6.0, 2.95, 1.7550000000000001, 4.0, 1.65, 1.0, 1.0, 1.001111111111111, 1.011111111111111, 1.0277777777777777, 4.0375, 5.966666666666667, 5.996666666666667, 0.5941870030161777, 5.0, 0.0, 1.2, 3.5, 0.20687260936809726, 1.9324105480761042, 1.8834808201837363, 0.43210013459357943, 3.7342105263157896, 3.5475000000000003, 154.0, 28.57142857142857} ) }, FTABLE_RECORD { std::make_tuple( - std::vector{"Segmentation1", "Intensity1"}, + std::vector{"Segmentation1", "Intensity1", ""}, 2, DEFAULT_T_INDEX, + DEFAULT_C_INDEX, + 1.0, 1.0, 1.0, std::vector {0.8045973903258694, 0.8333333333333334, 157.0, 1.457424881409171, -0.5612338213749162, 8.091332781325145, 3.896507376634866, 43.0, 2.9958333333333336, 2.438766178625084, 6.0, 2.263157894736842, 1.5955678670360112, 1.0, 1.263157894736842, 1.0, 1.0, 1.0007916666666667, 1.0079166666666666, 1.0197916666666667, 4.015625, 5.952500000000001, 5.99525, 0.5949524203558131, 5.0, 0.0, 1.2, 2.8745709061939735, 0.9598465178383306, 1.8209309360006518, 1.772364025313972, 0.41775020701019844, 3.3157894736842097, 3.141274238227146, 165.0, 28.57142857142857} ) }, FTABLE_RECORD { std::make_tuple( - std::vector{"Segmentation2", "Intensity2"}, + std::vector{"Segmentation2", "Intensity2", ""}, 3, DEFAULT_T_INDEX, + DEFAULT_C_INDEX, + 1.0, 1.0, 1.0, std::vector {0.9067141318473291, 0.8333333333333334, 83.0, 0.8030717273241024, 2.994153500512417, 42.98771025542074, 14.88820850639021, 29.0, 0.030000000000000027, 5.994153500512417, 6.0, 1.6111111111111112, 1.0185185185185182, 1.0, 0.6111111111111112, 1.0, 1.0, 1.0006, 1.006, 1.015, 1.045, 4.03, 5.991, 0.014563106796116521, 5.0, 0.0, 1.2, 2.1473497877875207, 2.10443379958053, 1.4608172124206968, 1.4196591487978487, 0.3443179189922347, 2.1339869281045756, 2.0154320987654324, 230.0, 28.57142857142857} ) }, FTABLE_RECORD { std::make_tuple( - std::vector{"Segmentation0", "Intensity0"}, + std::vector{"Segmentation0", "Intensity0", ""}, 4, DEFAULT_T_INDEX, + DEFAULT_C_INDEX, + 1.0, 1.0, 1.0, std::vector {.9067141318473291, 0.8333333333333334, 83.0, 0.8030717273241024, 2.994153500512417, 42.98771025542074, 14.88820850639021, 29.0, 0.030000000000000027, 5.994153500512417, 6.0, 1.6111111111111112, 1.0185185185185182, 1.0, 0.6111111111111112, 1.0, 1.0, 1.0006, 1.006, 1.015, 1.045, 4.03, 5.991, 0.014563106796116521, 5.0, 0.0, 1.2, 2.1473497877875207, 2.10443379958053, 1.4608172124206968, 1.4196591487978487, 0.3443179189922347, 2.1339869281045756, 2.0154320987654324, 230.0, 28.57142857142857} ) } diff --git a/tests/test_hu_mechanics.h b/tests/test_hu_mechanics.h index 33876a03..afcf76f7 100644 --- a/tests/test_hu_mechanics.h +++ b/tests/test_hu_mechanics.h @@ -75,7 +75,7 @@ static std::vector hu_load_tile0(const char* fixture, bool preserve_hu size_t th = ldr.tileHeight(0), tw = ldr.tileWidth(0); auto tile = std::make_shared>(th * tw, 0u); - EXPECT_NO_THROW(ldr.loadTileFromFile(tile, 0, 0, 0, 0)); + EXPECT_NO_THROW(ldr.loadTileFromFile(tile, 0, 0, 0, 0/*channel*/, 0/*timeframe*/, 0)); return *tile; } @@ -124,7 +124,7 @@ static std::vector hu_load_dicom_tile0(const char* fixture, bool prese NyxusGrayscaleDicomLoader ldr(1, ds.string(), fpmin, preserve_hu); size_t th = ldr.tileHeight(0), tw = ldr.tileWidth(0); auto tile = std::make_shared>(th * tw, 0u); - EXPECT_NO_THROW(ldr.loadTileFromFile(tile, 0, 0, 0, 0)); + EXPECT_NO_THROW(ldr.loadTileFromFile(tile, 0, 0, 0, 0/*channel*/, 0/*timeframe*/, 0)); return *tile; } diff --git a/tests/test_morphology_common.h b/tests/test_morphology_common.h index a7221203..a8b2bb47 100644 --- a/tests/test_morphology_common.h +++ b/tests/test_morphology_common.h @@ -238,7 +238,7 @@ static void calculate_fractal_blob512_feature_values(std::vector>(tw * th * td); - loader.loadTileFromFile(tile, 0, 0, 0, 0); // 512x512 <= 1024 strip tile: one read + loader.loadTileFromFile(tile, 0, 0, 0, 0/*channel*/, 0/*timeframe*/, 0); // 512x512 <= 1024 strip tile: one read std::vector px; for (size_t y = 0; y < H; y++) diff --git a/tests/test_ome_meta.h b/tests/test_ome_meta.h new file mode 100644 index 00000000..ceb86720 --- /dev/null +++ b/tests/test_ome_meta.h @@ -0,0 +1,595 @@ +#pragma once + +// Unit tests for the native-OME metadata parsers / OmeAxes descriptor. +// The OME-XML parser and OmeAxes helpers are exercised here with embedded +// fixtures (no external files). The OME-Zarr parser is guarded by +// OMEZARR_SUPPORT (compiled only in USE_Z5 builds); it is additionally +// validated end-to-end against the reference-data matrix by +// io_oracle/ome_meta_selftest.cpp. + +#include +#include "../src/nyx/ome/ome_axes.h" +#include "../src/nyx/ome/ome_tiff_meta.h" + +// 5D, non-default DimensionOrder XYZTC (C and T swapped) — the case that catches +// a reader assuming TCZYX. Taken verbatim from io_oracle/fixtures/xml/5d_ctzyx.xml. +static const char* kOmeXml_5d_ctzyx = + "" + "" + "" + "" + ""; + +static const char* kOmeXml_2d = + "" + ""; + +TEST(OmeTiffMeta, ParsesNonDefault5D) +{ + Nyxus::OmeAxes a = Nyxus::parse_ome_xml(kOmeXml_5d_ctzyx); + ASSERT_TRUE(a.valid); + EXPECT_EQ(a.sizeX, 8u); + EXPECT_EQ(a.sizeY, 6u); + EXPECT_EQ(a.sizeZ, 4u); + EXPECT_EQ(a.sizeC, 3u); + EXPECT_EQ(a.sizeT, 2u); + // XYZTC means C slowest, T next -> on-disk (slowest-first) is C,T,Z,Y,X. + EXPECT_EQ(a.storageOrder, "CTZYX"); + EXPECT_EQ(a.omeDimensionOrder, "XYZTC"); + EXPECT_EQ(a.dtype, Nyxus::PixelType::UInt16); + EXPECT_EQ(a.bitsPerSample, 16u); + EXPECT_DOUBLE_EQ(a.physX, 0.5); + EXPECT_DOUBLE_EQ(a.physZ, 2.0); + EXPECT_DOUBLE_EQ(a.timeIncrement, 1.0); + EXPECT_EQ(a.unitZ, "micrometer"); + EXPECT_EQ(a.unitT, "second"); + EXPECT_TRUE(a.isVolumetric()); + EXPECT_TRUE(a.isMultiChannel()); + EXPECT_TRUE(a.isTimeSeries()); + ASSERT_EQ(a.channelNames.size(), 2u); + EXPECT_EQ(a.channelNames[0], "DAPI"); + EXPECT_EQ(a.channelNames[1], "GFP"); + // axis-role lookup independent of on-disk order + EXPECT_EQ(a.storageIndexOf('C'), 0); // C is first (slowest) in CTZYX + EXPECT_EQ(a.storageIndexOf('X'), 4); +} + +// plane->IFD mapping. Base XML: XYZCT, Z=2 C=2 T=1 -> 4 planes, canonical +// ordinal ord = z + c*2. +static std::string tiffdata_xml(const std::string& blocks) +{ + return "" + blocks + + ""; +} + +// A single canonical block (tifffile's form) means contiguous-from-IFD-0 == the default, so +// the map is left empty and ifdForPlane stays canonical. +TEST(OmeTiffMetaTiffData, CanonicalSingleBlockStaysIdentity) +{ + Nyxus::OmeAxes a = Nyxus::parse_ome_xml(tiffdata_xml("")); + ASSERT_TRUE(a.valid); + EXPECT_TRUE(a.planeToIfd.empty()); + EXPECT_FALSE(a.multiFileTiff); + EXPECT_EQ(a.ifdForPlane(0, 0, 0), 0u); + EXPECT_EQ(a.ifdForPlane(1, 1, 0), 3u); // ord = 1 + 1*2 +} + +// Per-plane blocks that reverse the IFDs: ifdForPlane must return the declared IFD. +TEST(OmeTiffMetaTiffData, ReversedPerPlaneMapping) +{ + Nyxus::OmeAxes a = Nyxus::parse_ome_xml(tiffdata_xml( + "" + "" + "" + "")); + ASSERT_TRUE(a.valid); + ASSERT_EQ(a.planeToIfd.size(), 4u); + EXPECT_EQ(a.ifdForPlane(0, 0, 0), 3u); + EXPECT_EQ(a.ifdForPlane(1, 0, 0), 2u); + EXPECT_EQ(a.ifdForPlane(0, 1, 0), 1u); + EXPECT_EQ(a.ifdForPlane(1, 1, 0), 0u); +} + +// P5: PlaneCount omitted -> defaults to "the remaining planes from the start plane" (OME +// spec). A single block with only IFD given must map every plane from that offset. +TEST(OmeTiffMetaTiffData, OmittedPlaneCountCoversRemaining) +{ + Nyxus::OmeAxes a = Nyxus::parse_ome_xml(tiffdata_xml("")); + ASSERT_TRUE(a.valid); + ASSERT_EQ(a.planeToIfd.size(), 4u); // 4 planes (Z=2,C=2) + EXPECT_EQ(a.ifdForPlane(0, 0, 0), 5u); // start at IFD 5 + EXPECT_EQ(a.ifdForPlane(1, 0, 0), 6u); + EXPECT_EQ(a.ifdForPlane(0, 1, 0), 7u); + EXPECT_EQ(a.ifdForPlane(1, 1, 0), 8u); // all four planes mapped, consecutively +} + +// A non-zero starting IFD (planes contiguous but offset, e.g. a multi-image container). +TEST(OmeTiffMetaTiffData, NonZeroStartOffsetsAllPlanes) +{ + Nyxus::OmeAxes a = Nyxus::parse_ome_xml(tiffdata_xml("")); + ASSERT_TRUE(a.valid); + ASSERT_EQ(a.planeToIfd.size(), 4u); + EXPECT_EQ(a.ifdForPlane(0, 0, 0), 10u); + EXPECT_EQ(a.ifdForPlane(1, 1, 0), 13u); +} + +// A naming another file ( child) is multi-file: unsupported, flagged, and +// that plane is left at the canonical fallback rather than pointed at a wrong local IFD. +TEST(OmeTiffMetaTiffData, MultiFileUuidBlockFlaggedAndSkipped) +{ + Nyxus::OmeAxes a = Nyxus::parse_ome_xml(tiffdata_xml( + "" + "urn:uuid:abc")); + ASSERT_TRUE(a.valid); + EXPECT_TRUE(a.multiFileTiff); + EXPECT_EQ(a.ifdForPlane(0, 0, 0), 0u); // canonical fallback (not mapped) +} + +TEST(OmeTiffMeta, Parses2DSingletonAxesCollapse) +{ + Nyxus::OmeAxes a = Nyxus::parse_ome_xml(kOmeXml_2d); + ASSERT_TRUE(a.valid); + EXPECT_EQ(a.sizeZ, 1u); + EXPECT_EQ(a.sizeC, 1u); + EXPECT_EQ(a.sizeT, 1u); + EXPECT_EQ(a.storageOrder, "YX"); // singleton Z/C/T dropped; planes stay YX + EXPECT_FALSE(a.isVolumetric()); + EXPECT_FALSE(a.isMultiChannel()); + EXPECT_FALSE(a.isTimeSeries()); +} + +TEST(OmeTiffMeta, InvalidWhenNoPixels) +{ + Nyxus::OmeAxes a = Nyxus::parse_ome_xml(""); + EXPECT_FALSE(a.valid); +} + +// --------------------------------------------------------------------------- +// Negative / robustness: bad or missing metadata for every field we parse. +// The parser must never crash and must degrade to safe defaults. +// --------------------------------------------------------------------------- + +TEST(OmeTiffMetaBad, EmptyAndGarbageInput) +{ + EXPECT_FALSE(Nyxus::parse_ome_xml("").valid); + EXPECT_FALSE(Nyxus::parse_ome_xml("not xml at all").valid); + EXPECT_FALSE(Nyxus::parse_ome_xml("").valid); +} + +TEST(OmeTiffMetaBad, UnterminatedPixelsTagIsInvalid) +{ + // no '>' closing the Pixels start-tag + Nyxus::OmeAxes a = Nyxus::parse_ome_xml(""); + EXPECT_FALSE(a.valid); +} + +TEST(OmeTiffMetaBad, AllSizesMissingDefaultToOne) +{ + Nyxus::OmeAxes a = Nyxus::parse_ome_xml(""); + ASSERT_TRUE(a.valid); + EXPECT_EQ(a.sizeX, 1u); EXPECT_EQ(a.sizeY, 1u); EXPECT_EQ(a.sizeZ, 1u); + EXPECT_EQ(a.sizeC, 1u); EXPECT_EQ(a.sizeT, 1u); + EXPECT_EQ(a.storageOrder, "YX"); + EXPECT_EQ(a.omeDimensionOrder, "XYZCT"); // missing -> default + EXPECT_EQ(a.dtype, Nyxus::PixelType::UInt16); // missing Type -> default + EXPECT_DOUBLE_EQ(a.physX, 1.0); // missing PhysicalSize -> 1.0 + EXPECT_DOUBLE_EQ(a.physZ, 1.0); + EXPECT_DOUBLE_EQ(a.timeIncrement, 1.0); + EXPECT_TRUE(a.unitXY.empty()); + EXPECT_TRUE(a.unitZ.empty()); + EXPECT_TRUE(a.unitT.empty()); +} + +TEST(OmeTiffMetaBad, NonNumericSizeDefaultsToOne) +{ + Nyxus::OmeAxes a = Nyxus::parse_ome_xml( + ""); + ASSERT_TRUE(a.valid); + EXPECT_EQ(a.sizeX, 1u); // "abc" -> 1, not 0 + EXPECT_EQ(a.sizeY, 6u); + EXPECT_EQ(a.sizeZ, 1u); // "" -> 1 +} + +TEST(OmeTiffMetaBad, ZeroSizeClampedToOne) +{ + Nyxus::OmeAxes a = Nyxus::parse_ome_xml( + ""); + ASSERT_TRUE(a.valid); + EXPECT_EQ(a.sizeX, 1u); // 0 would divide-by-zero the tile grid + EXPECT_EQ(a.sizeY, 1u); + EXPECT_EQ(a.sizeZ, 4u); +} + +TEST(OmeTiffMetaBad, NegativeSizeRejectedToDefault) +{ + // strtoull wraps a leading '-' into a huge extent; parser must reject it + Nyxus::OmeAxes a = Nyxus::parse_ome_xml(""); + ASSERT_TRUE(a.valid); + EXPECT_EQ(a.sizeX, 1u); // negative -> default 1, not a giant number + EXPECT_EQ(a.sizeY, 6u); +} + +TEST(OmeTiffMetaBad, MissingDimensionOrderDefaults) +{ + Nyxus::OmeAxes a = Nyxus::parse_ome_xml(""); + EXPECT_EQ(a.omeDimensionOrder, "XYZCT"); + EXPECT_EQ(a.storageOrder, "ZYX"); +} + +TEST(OmeTiffMetaBad, MalformedDimensionOrderFallsBack) +{ + for (const char* bad : { "FOOBAR", "XYZC", "XYZCC", "XXYZCT", "xyzct", "" }) + { + std::string xml = std::string(""; + Nyxus::OmeAxes a = Nyxus::parse_ome_xml(xml); + ASSERT_TRUE(a.valid) << bad; + EXPECT_EQ(a.omeDimensionOrder, "XYZCT") << "bad input: " << bad; + } +} + +TEST(OmeTiffMetaBad, SingleQuotedAttributes) +{ + Nyxus::OmeAxes a = Nyxus::parse_ome_xml( + ""); + ASSERT_TRUE(a.valid); + EXPECT_EQ(a.sizeX, 8u); + EXPECT_EQ(a.sizeZ, 4u); + EXPECT_EQ(a.dtype, Nyxus::PixelType::UInt8); + EXPECT_DOUBLE_EQ(a.physX, 0.25); +} + +TEST(OmeTiffMetaBad, UnknownAndMissingTypeFallBackToUint16) +{ + EXPECT_EQ(Nyxus::parse_ome_xml("").dtype, + Nyxus::PixelType::UInt16); + EXPECT_EQ(Nyxus::parse_ome_xml("").dtype, + Nyxus::PixelType::UInt16); +} + +TEST(OmeTiffMetaBad, AllPixelTypesResolve) +{ + auto ty = [](const char* t) { + std::string xml = std::string(""; + return Nyxus::parse_ome_xml(xml).dtype; + }; + EXPECT_EQ(ty("uint8"), Nyxus::PixelType::UInt8); + EXPECT_EQ(ty("int16"), Nyxus::PixelType::Int16); + EXPECT_EQ(ty("int32"), Nyxus::PixelType::Int32); + EXPECT_EQ(ty("float"), Nyxus::PixelType::Float32); + EXPECT_EQ(ty("double"), Nyxus::PixelType::Float64); + EXPECT_EQ(Nyxus::parse_ome_xml("").bitsPerSample, 64u); +} + +TEST(OmeTiffMetaBad, NonNumericPhysicalSizesDefaultToOne) +{ + Nyxus::OmeAxes a = Nyxus::parse_ome_xml( + ""); + ASSERT_TRUE(a.valid); + EXPECT_DOUBLE_EQ(a.physX, 1.0); // failed parse -> default, never 0 + EXPECT_DOUBLE_EQ(a.physZ, 1.0); + EXPECT_DOUBLE_EQ(a.timeIncrement, 1.0); +} + +TEST(OmeTiffMetaBad, NanInfPhysicalSizesRejected) +{ + // strtod parses "NaN"/"inf" prefixes; such values must not leak into calibration + for (const char* bad : { "NaN", "NaNsense", "inf", "-inf", "infinity" }) + { + std::string xml = std::string(""; + Nyxus::OmeAxes a = Nyxus::parse_ome_xml(xml); + ASSERT_TRUE(a.valid) << bad; + EXPECT_DOUBLE_EQ(a.physZ, 1.0) << "PhysicalSizeZ from " << bad; + EXPECT_DOUBLE_EQ(a.timeIncrement, 1.0) << "TimeIncrement from " << bad; + } +} + +TEST(OmeTiffMetaBad, SizeXNotConfusedWithPhysicalSizeX) +{ + // attribute-name boundary: SizeX must not read from PhysicalSizeX + Nyxus::OmeAxes a = Nyxus::parse_ome_xml( + ""); + ASSERT_TRUE(a.valid); + EXPECT_EQ(a.sizeX, 8u); + // physX/unitXY are canonicalized to micrometer (9.9 nm = 0.0099 um); see + // UnitCanonicalization.NanometerConvertedToMicrometer for a test dedicated to that. + EXPECT_DOUBLE_EQ(a.physX, 0.0099); + EXPECT_EQ(a.unitXY, "micrometer"); +} + +// canonicalize_to_micrometer / unit_scale_to_micrometer directly -- no XML/JSON parsing +// involved, so these pin the conversion table itself independent of either format parser. +TEST(UnitCanonicalization, KnownUnitsConvertedToMicrometer) +{ + struct Case { const char* unit; double input; double expected_um; }; + const Case cases[] = { + { "nanometer", 500.0, 0.5 }, + { "nm", 500.0, 0.5 }, + { "micrometer", 2.0, 2.0 }, + { "um", 2.0, 2.0 }, + { "millimeter", 0.01, 10.0 }, + { "mm", 0.01, 10.0 }, + { "centimeter", 0.001, 10.0 }, + { "meter", 0.000002, 2.0 }, + { "angstrom", 50000.0, 5.0 }, + }; + for (const auto& c : cases) + { + double v = c.input; + std::string u = c.unit; + Nyxus::canonicalize_to_micrometer(v, u); + EXPECT_NEAR(v, c.expected_um, 1e-9) << "unit=" << c.unit; + EXPECT_EQ(u, "micrometer") << "unit=" << c.unit; + } +} + +TEST(UnitCanonicalization, UnrecognizedOrEmptyUnitLeftAsIs) +{ + for (const char* unit : { "", "furlong", "pixel" }) + { + double v = 42.0; + std::string u = unit; + Nyxus::canonicalize_to_micrometer(v, u); + EXPECT_DOUBLE_EQ(v, 42.0) << "unit=" << unit; + EXPECT_EQ(u, unit) << "unit=" << unit; + } +} + +TEST(UnitCanonicalization, OmeTiffNanometerXAndMillimeterZBothConvert) +{ + // X/Y declared in nanometer, Z in a DIFFERENT unit (millimeter) -- each axis must + // convert using its OWN declared unit, not X/Y's unit applied to Z (or vice versa). + Nyxus::OmeAxes a = Nyxus::parse_ome_xml( + ""); + ASSERT_TRUE(a.valid); + EXPECT_NEAR(a.physX, 0.5, 1e-9); + EXPECT_NEAR(a.physY, 0.5, 1e-9); + EXPECT_NEAR(a.physZ, 3.0, 1e-9); + EXPECT_EQ(a.unitXY, "micrometer"); + EXPECT_EQ(a.unitZ, "micrometer"); + // storageAxes mirror the canonicalized per-axis values too + int ix = a.storageIndexOf('X'), iy = a.storageIndexOf('Y'), iz = a.storageIndexOf('Z'); + ASSERT_GE(ix, 0); ASSERT_GE(iy, 0); ASSERT_GE(iz, 0); + EXPECT_NEAR(a.storageAxes[ix].physical, 0.5, 1e-9); + EXPECT_NEAR(a.storageAxes[iy].physical, 0.5, 1e-9); + EXPECT_NEAR(a.storageAxes[iz].physical, 3.0, 1e-9); +} + +TEST(OmeTiffMetaBad, WhitespaceAndNewlinesInTag) +{ + Nyxus::OmeAxes a = Nyxus::parse_ome_xml( + ""); + ASSERT_TRUE(a.valid); + EXPECT_EQ(a.sizeX, 8u); + EXPECT_EQ(a.sizeZ, 4u); + EXPECT_EQ(a.omeDimensionOrder, "XYZCT"); +} + +TEST(OmeTiffMetaBad, ChannelsWithoutNameYieldNoNames) +{ + Nyxus::OmeAxes a = Nyxus::parse_ome_xml( + "" + ""); + ASSERT_TRUE(a.valid); + EXPECT_EQ(a.sizeC, 2u); + EXPECT_TRUE(a.channelNames.empty()); // no Name attr -> nothing collected +} + +TEST(OmeTiffMetaBad, FirstPixelsWinsAcrossMultipleImages) +{ + Nyxus::OmeAxes a = Nyxus::parse_ome_xml( + "" + ""); + ASSERT_TRUE(a.valid); + EXPECT_EQ(a.sizeX, 8u); + EXPECT_EQ(a.sizeY, 6u); +} + +TEST(OmeAxes, PixelTypeMappings) +{ + using namespace Nyxus; + EXPECT_EQ(pixel_type_from_ome_type("uint16"), PixelType::UInt16); + EXPECT_EQ(pixel_type_from_ome_type("float"), PixelType::Float32); + EXPECT_EQ(pixel_type_from_ome_type("double"), PixelType::Float64); + // zarr v2 numpy strings and v3 names both resolve + EXPECT_EQ(pixel_type_from_zarr_dtype(" shape = {3, 2, 4, 6, 8}; // C,T,Z,Y,X + Nyxus::OmeAxes a = Nyxus::parse_ome_zarr(attrs, shape, " shape, const char* dt = " all default + EXPECT_DOUBLE_EQ(a.physY, 1.0); +} + +TEST(OmeZarrMetaBad, NonNumericScaleElementDefaultsThatAxis) +{ + Nyxus::OmeAxes a = zarr_parse( + R"({"multiscales":[{"axes":[{"name":"y","type":"space"},{"name":"x","type":"space"}], + "datasets":[{"path":"0","coordinateTransformations":[{"type":"scale","scale":["oops",0.5]}]}]}]})", + {6, 8}); + ASSERT_TRUE(a.valid); + EXPECT_DOUBLE_EQ(a.physY, 1.0); // "oops" -> 1.0 + EXPECT_DOUBLE_EQ(a.physX, 0.5); +} + +TEST(OmeZarrMetaBad, MalformedAxisEntrySkipped) +{ + // a non-object axis entry must be skipped, not crash + Nyxus::OmeAxes a = zarr_parse( + R"({"multiscales":[{"axes":["notanobject",{"name":"y","type":"space"},{"name":"x","type":"space"}], + "datasets":[{"path":"0","coordinateTransformations":[{"type":"scale","scale":[1,0.5,0.5]}]}]}]})", + {1, 6, 8}); + ASSERT_TRUE(a.valid); + EXPECT_EQ(a.sizeX, 8u); + EXPECT_EQ(a.sizeY, 6u); +} + +TEST(OmeZarrMetaBad, UnknownDtypeFallsBackUint16) +{ + Nyxus::OmeAxes a = zarr_parse( + R"({"multiscales":[{"axes":[{"name":"y","type":"space"},{"name":"x","type":"space"}], + "datasets":[{"path":"0","coordinateTransformations":[{"type":"scale","scale":[1,1]}]}]}]})", + {6, 8}, "weird64"); + ASSERT_TRUE(a.valid); + EXPECT_EQ(a.dtype, Nyxus::PixelType::UInt16); +} + +TEST(OmeZarrMetaBad, V3OmeKeyPathParses) +{ + // 0.5 nests the model under "ome" + Nyxus::OmeAxes a = zarr_parse( + R"({"ome":{"version":"0.5","multiscales":[{ + "axes":[{"name":"z","type":"space"},{"name":"y","type":"space"},{"name":"x","type":"space"}], + "datasets":[{"path":"0","coordinateTransformations":[{"type":"scale","scale":[2.0,0.5,0.5]}]}]}]}})", + {4, 6, 8}, "uint16"); + ASSERT_TRUE(a.valid); + EXPECT_EQ(a.sizeZ, 4u); + EXPECT_EQ(a.storageOrder, "ZYX"); + EXPECT_DOUBLE_EQ(a.physZ, 2.0); +} + +TEST(OmeZarrMetaBad, ShorterShapeThanAxesDefaultsMissingSizes) +{ + // level0Shape shorter than axes list -> missing dims default to size 1, no OOB + Nyxus::OmeAxes a = zarr_parse( + R"({"multiscales":[{"axes":[{"name":"c","type":"channel"},{"name":"y","type":"space"},{"name":"x","type":"space"}], + "datasets":[{"path":"0","coordinateTransformations":[{"type":"scale","scale":[1,0.5,0.5]}]}]}]})", + {3}); // only C provided + ASSERT_TRUE(a.valid); + EXPECT_EQ(a.sizeC, 3u); + EXPECT_EQ(a.sizeY, 1u); + EXPECT_EQ(a.sizeX, 1u); +} + +TEST(UnitCanonicalization, OmeZarrNanometerXYAndAngstromZBothConvert) +{ + // X/Y declared in nanometer, Z in a DIFFERENT unit (angstrom) -- each axis must + // convert using its OWN declared unit. + Nyxus::OmeAxes a = zarr_parse( + R"({"multiscales":[{ + "axes":[{"name":"z","type":"space","unit":"angstrom"}, + {"name":"y","type":"space","unit":"nanometer"}, + {"name":"x","type":"space","unit":"nanometer"}], + "datasets":[{"path":"0","coordinateTransformations":[{"type":"scale","scale":[30000.0,500.0,500.0]}]}]}]})", + {4, 6, 8}, "uint16"); + ASSERT_TRUE(a.valid); + EXPECT_NEAR(a.physX, 0.5, 1e-9); + EXPECT_NEAR(a.physY, 0.5, 1e-9); + EXPECT_NEAR(a.physZ, 3.0, 1e-9); + EXPECT_EQ(a.unitXY, "micrometer"); + EXPECT_EQ(a.unitZ, "micrometer"); +} + +TEST(UnitCanonicalization, OmeZarrAlreadyMicrometerUnaffected) +{ + // An already-canonical store must come through byte-for-byte unchanged (no accidental + // double-conversion). + Nyxus::OmeAxes a = zarr_parse( + R"({"multiscales":[{ + "axes":[{"name":"z","type":"space","unit":"micrometer"}, + {"name":"y","type":"space","unit":"micrometer"}, + {"name":"x","type":"space","unit":"micrometer"}], + "datasets":[{"path":"0","coordinateTransformations":[{"type":"scale","scale":[2.0,0.5,0.5]}]}]}]})", + {4, 6, 8}, "uint16"); + ASSERT_TRUE(a.valid); + EXPECT_DOUBLE_EQ(a.physX, 0.5); + EXPECT_DOUBLE_EQ(a.physY, 0.5); + EXPECT_DOUBLE_EQ(a.physZ, 2.0); + EXPECT_EQ(a.unitXY, "micrometer"); + EXPECT_EQ(a.unitZ, "micrometer"); +} +#endif diff --git a/tests/test_ometiff_mechanics.h b/tests/test_ometiff_mechanics.h new file mode 100644 index 00000000..b25c1557 --- /dev/null +++ b/tests/test_ometiff_mechanics.h @@ -0,0 +1,395 @@ +#pragma once + +// OME-TIFF native read: the multi-page loaders map (z,c,t) to the correct IFD via +// the OME-XML DimensionOrder, instead of assuming every page is a Z-slice. +// +// Fixtures (tests/data/ometiff, see gen_ome_tiff.py) encode every voxel as +// value(x,y,z,c,t) = 1 + ((((t*C + c)*Z + z)*Y + y)*X + x), C=3,Z=4,Y=6,X=8 +// so reading plane (z,c,t) must return that plane's values; a loader that mapped +// (z,c,t) to the wrong page returns provably wrong data. Unlike OME-Zarr this needs +// no USE_Z5 -- TIFF is a core dependency, so these run in every build. + +#include +#include +#include +#include "../src/nyx/grayscale_tiff.h" +#include "../src/nyx/raw_tiff.h" +#include "../src/nyx/image_loader.h" +#include "../src/nyx/globals.h" // scan_trivial_wholevolume, LR +#include "../src/nyx/helpers/fsystem.h" + +static inline fs::path ometiff_data_path(const char* name) +{ + fs::path p(__FILE__); + fs::path rel(std::string("/data/ometiff/") + name); + return fs::path(p.parent_path().string() + rel.make_preferred().string()); +} + +static inline uint32_t ometiff_enc(int x, int y, int z, int c, int t) +{ + const int C = 3, Z = 4, Y = 6, X = 8; + return static_cast(1 + ((((t * C + c) * Z + z) * Y + y) * X + x)); +} + +// AbstractTileLoader stack (NyxusGrayscaleTiffStripLoader). (T,C,Z) are the store's +// extents (X=8,Y=6 fixed). One body covers TCZYX, non-default CTZYX, 3D, 2D, and +// the plain non-OME multi-page store -- all must return the same encoded values. +void test_ometiff_addressing(const char* store, int T, int C, int Z) +{ + const int Y = 6, X = 8; + fs::path ds = ometiff_data_path(store); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + auto ldr = NyxusGrayscaleTiffStripLoader(1, ds.string()); + ASSERT_EQ(ldr.fullWidth(0), (size_t)X); + ASSERT_EQ(ldr.fullHeight(0), (size_t)Y); + ASSERT_EQ(ldr.fullDepth(0), (size_t)Z); // 5D OME-TIFF must report SizeZ, not the total page count + const size_t tw = ldr.tileWidth(0); + auto tile = std::make_shared>(ldr.tileHeight(0) * tw, 0u); + + for (int t = 0; t < T; ++t) + for (int c = 0; c < C; ++c) + for (int z = 0; z < Z; ++z) + { + std::fill(tile->begin(), tile->end(), 0u); + ASSERT_NO_THROW(ldr.loadTileFromFile(tile, 0, 0, z, c, t, 0)); + const std::vector& buf = *tile; + for (int y = 0; y < Y; ++y) + for (int x = 0; x < X; ++x) + ASSERT_EQ(buf[y * tw + x], ometiff_enc(x, y, z, c, t)) + << store << " plane (z" << z << " c" << c << " t" << t << ") at (" << x << "," << y << ")"; + } +} + +// RawFormatLoader stack (RawTiffStripLoader). +void test_raw_ometiff_addressing(const char* store, int T, int C, int Z) +{ + const int Y = 6, X = 8; + fs::path ds = ometiff_data_path(store); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + auto ldr = RawTiffStripLoader(1, ds.string()); + ASSERT_EQ(ldr.fullDepth(0), (size_t)Z); + const size_t w = ldr.fullWidth(0); + + for (int t = 0; t < T; ++t) + for (int c = 0; c < C; ++c) + for (int z = 0; z < Z; ++z) + { + ASSERT_NO_THROW(ldr.loadTileFromFile(0, 0, z, c, t, 0)); + for (int y = 0; y < Y; ++y) + for (int x = 0; x < X; ++x) + ASSERT_EQ(ldr.get_uint32_pixel(y * w + x), ometiff_enc(x, y, z, c, t)) + << store << " plane (z" << z << " c" << c << " t" << t << ") at (" << x << "," << y << ")"; + } + ldr.free_tile(); +} + +// End-to-end through the WIRED volumetric consumer: scan_trivial_wholevolume must +// feed the whole X*Y*Z volume (all Z planes) into the ROI's voxel cloud, with the +// correct encoded intensity. Before the wiring it read only plane z=0, so this +// asserts the consumer fix, not just the facade. +void test_ometiff_wholevolume_consumer(const char* store, int Z) +{ + const int Y = 6, X = 8; + fs::path ds = ometiff_data_path(store); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + SlideProps p; + p.fname_int = ds.string(); + p.fname_seg = ""; + FpImageOptions fp; + ImageLoader ilo; + ASSERT_TRUE(ilo.open(p, fp)) << ds.string(); + + LR vroi; + ASSERT_TRUE(Nyxus::scan_trivial_wholevolume(vroi, ds.string(), ilo, 0/*channel*/, 0/*timeframe*/)); + + // every voxel of the X*Y*Z volume must be captured (not just plane z=0 -> 48) + ASSERT_EQ(vroi.raw_pixels_3D.size(), (size_t)X * Y * Z); + for (const Pixel3& px : vroi.raw_pixels_3D) + ASSERT_EQ((uint32_t)px.inten, ometiff_enc((int)px.x, (int)px.y, (int)px.z, 0, 0)) + << "voxel (" << px.x << "," << px.y << "," << px.z << ")"; + ilo.close(); +} + +// End-to-end through the wired volumetric consumer for EVERY (channel,timeframe): +// scan_trivial_wholevolume(vroi, .., c, t) must fill the voxel cloud with THAT c/t +// plane's encoded intensity (before the wiring it always read c=0,t=0). +void test_ometiff_wholevolume_consumer_ct(const char* store, int T, int C, int Z) +{ + const int Y = 6, X = 8; + fs::path ds = ometiff_data_path(store); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + SlideProps p; + p.fname_int = ds.string(); + p.fname_seg = ""; + FpImageOptions fp; + ImageLoader ilo; + ASSERT_TRUE(ilo.open(p, fp)) << ds.string(); + + for (int t = 0; t < T; ++t) + for (int c = 0; c < C; ++c) + { + LR vroi; + ASSERT_TRUE(Nyxus::scan_trivial_wholevolume(vroi, ds.string(), ilo, c, t)) << store; + ASSERT_EQ(vroi.raw_pixels_3D.size(), (size_t)X * Y * Z) << store; + for (const Pixel3& px : vroi.raw_pixels_3D) + ASSERT_EQ((uint32_t)px.inten, ometiff_enc((int)px.x, (int)px.y, (int)px.z, c, t)) + << store << " (c" << c << " t" << t << ") voxel (" << px.x << "," << px.y << "," << px.z << ")"; + } + ilo.close(); +} + +// Whole-volume assembly through the ImageLoader facade: load_volume() stacks all +// Z-planes (per (channel,timeframe)) into one X*Y*Z buffer -- the foundation that +// lets the volumetric pipeline consume a multi-page OME-TIFF. +void test_ometiff_facade_volume(const char* store, int T, int C, int Z) +{ + const int Y = 6, X = 8; + fs::path ds = ometiff_data_path(store); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + SlideProps p; + p.fname_int = ds.string(); + p.fname_seg = ""; + FpImageOptions fp; + ImageLoader il; + ASSERT_TRUE(il.open(p, fp)) << ds.string(); + ASSERT_EQ(il.get_full_width(), (size_t)X); + ASSERT_EQ(il.get_full_height(), (size_t)Y); + ASSERT_EQ(il.get_full_depth(), (size_t)Z); + + for (int t = 0; t < T; ++t) + for (int c = 0; c < C; ++c) + { + ASSERT_TRUE(il.load_volume(c, t)); + const std::vector& vol = il.get_int_volume_buffer(); + ASSERT_EQ(vol.size(), (size_t)X * Y * Z); + for (int z = 0; z < Z; ++z) + for (int y = 0; y < Y; ++y) + for (int x = 0; x < X; ++x) + ASSERT_EQ(vol[(size_t)z * X * Y + (size_t)y * X + x], ometiff_enc(x, y, z, c, t)) + << store << " vol (x" << x << " y" << y << " z" << z << " c" << c << " t" << t << ")"; + } + il.close(); +} + +// The multitile fixtures carry their own (larger Y/X) shape because a TIFF tile size must be +// a multiple of 16: only a plane bigger than 16x16 yields a real tile grid. Their coordinate +// encoding is encoded_tczyx with each fixture's own C/Z/Y/X (see gen_ome_tiff.write_multitile). +inline uint32_t ometiff_enc_dims(int x, int y, int z, int c, int t, int C, int Z, int Y, int X) +{ + return (uint32_t)(1 + ((((t * C + c) * Z + z) * Y + y) * X + x)); +} + +// Multi-TILE plane: each plane spans a grid of 16x16 tiles, unlike every other fixture (one +// tile/chunk per plane), so assembling a volume from tile (0,0) alone silently returns wrong +// data outside the first 16x16 corner -- and over-reads that tile's buffer. With Y or X not a +// multiple of 16 the last tile of the row/column is PARTIAL, exercising the validH/validW seam +// clamp. Asserting the exact value at every voxel IS the seam check. +void test_ometiff_multitile_facade_volume(const char* store, int T, int C, int Z, int Y, int X) +{ + fs::path ds = ometiff_data_path(store); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + SlideProps p; + p.fname_int = ds.string(); + p.fname_seg = ""; + FpImageOptions fp; + ImageLoader il; + ASSERT_TRUE(il.open(p, fp)) << ds.string(); + ASSERT_EQ(il.get_full_width(), (size_t)X); + ASSERT_EQ(il.get_full_height(), (size_t)Y); + ASSERT_EQ(il.get_full_depth(), (size_t)Z); + + for (int t = 0; t < T; ++t) + for (int c = 0; c < C; ++c) + { + ASSERT_TRUE(il.load_volume(c, t)); + const std::vector& vol = il.get_int_volume_buffer(); + ASSERT_EQ(vol.size(), (size_t)X * Y * Z); + for (int z = 0; z < Z; ++z) + for (int y = 0; y < Y; ++y) + for (int x = 0; x < X; ++x) + ASSERT_EQ(vol[(size_t)z * X * Y + (size_t)y * X + x], ometiff_enc_dims(x, y, z, c, t, C, Z, Y, X)) + << store << " vol (x" << x << " y" << y << " z" << z << " c" << c << " t" << t << ")"; + } + il.close(); +} + +// Every one of the 6 legal OME DimensionOrder values (all orderings of {T,C,Z} +// before Y,X) must read correctly -- proves ifdForPlane honors DimensionOrder. +void test_ometiff_all_5d_permutations() +{ + for (const char* s : { "dim5.ome.tif", "dim5_tzcyx.ome.tif", "dim5_ctzyx.ome.tif", + "dim5_cztyx.ome.tif", "dim5_ztcyx.ome.tif", "dim5_zctyx.ome.tif" }) + { + test_ometiff_addressing(s, 2, 3, 4); + test_raw_ometiff_addressing(s, 2, 3, 4); + if (::testing::Test::HasFatalFailure()) return; + } +} + +// TILED multi-plane OME-TIFF: the TILE loaders (not just the strip loaders) must map +// (z,c,t) -> IFD. dim5_tiled.ome.tif is 5D TCZYX with one 16x16 tile per 6x8 plane. +void test_ometiff_tiled_addressing() +{ + const int T = 2, C = 3, Z = 4, Y = 6, X = 8; + fs::path ds = ometiff_data_path("dim5_tiled.ome.tif"); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + // AbstractTileLoader stack + auto ldr = NyxusGrayscaleTiffTileLoader(1, ds.string(), true, 0.0f, 1.0f, 1e4f, false); + ASSERT_EQ(ldr.fullWidth(0), (size_t)X); + ASSERT_EQ(ldr.fullHeight(0), (size_t)Y); + ASSERT_EQ(ldr.fullDepth(0), (size_t)Z); // SizeZ, not the 24 IFDs + ASSERT_EQ(ldr.numberChannels(), (size_t)C); + ASSERT_EQ(ldr.fullTimestamps(0), (size_t)T); + const size_t tw = ldr.tileWidth(0); + auto tile = std::make_shared>(ldr.tileHeight(0) * tw, 0u); + for (int t = 0; t < T; ++t) + for (int c = 0; c < C; ++c) + for (int z = 0; z < Z; ++z) + { + std::fill(tile->begin(), tile->end(), 0u); + ASSERT_NO_THROW(ldr.loadTileFromFile(tile, 0, 0, z, c, t, 0)); + const std::vector& buf = *tile; + for (int y = 0; y < Y; ++y) + for (int x = 0; x < X; ++x) + ASSERT_EQ(buf[y * tw + x], ometiff_enc(x, y, z, c, t)) + << "tile plane (z" << z << " c" << c << " t" << t << ") at (" << x << "," << y << ")"; + } + EXPECT_ANY_THROW(ldr.loadTileFromFile(tile, 0, 0, 0, 99, 0, 0)); // channel out of range + EXPECT_ANY_THROW(ldr.loadTileFromFile(tile, 0, 0, 99, 0, 0, 0)); // z out of range + + // RawFormatLoader stack + auto raw = RawTiffTileLoader(ds.string()); + ASSERT_EQ(raw.fullDepth(0), (size_t)Z); + ASSERT_EQ(raw.numberChannels(), (size_t)C); + ASSERT_EQ(raw.fullTimestamps(0), (size_t)T); + const size_t rw = raw.tileWidth(0); + for (int t = 0; t < T; ++t) + for (int c = 0; c < C; ++c) + for (int z = 0; z < Z; ++z) + { + ASSERT_NO_THROW(raw.loadTileFromFile(0, 0, z, c, t, 0)); + for (int y = 0; y < Y; ++y) + for (int x = 0; x < X; ++x) + ASSERT_EQ(raw.get_uint32_pixel(y * rw + x), ometiff_enc(x, y, z, c, t)) + << "raw tile plane (z" << z << " c" << c << " t" << t << ") at (" << x << "," << y << ")"; + raw.free_tile(); + } + EXPECT_ANY_THROW(raw.loadTileFromFile(0, 0, 0, 0, 99, 0)); // timeframe out of range + EXPECT_ANY_THROW(raw.loadTileFromFile(0, 0, 99, 0, 0, 0)); // z out of range +} + +// Regression: a single-channel mask paired with a multi-channel intensity. The mask is +// channel-agnostic and must be REUSED for every intensity channel (not indexed at the +// intensity's channel, which would read it out of range and drop the ROI for c>0). +// End-to-end this bug produced only the c=0 rows; here we assert at the facade. +void test_ometiff_multichannel_mask_pairing() +{ + fs::path ipath = ometiff_data_path("dim5.ome.tif"); // intensity C=3, T=2, Z=4 + fs::path mpath = ometiff_data_path("dim3_mask.ome.tif"); // single-channel ZYX label mask + ASSERT_TRUE(fs::exists(ipath)) << ipath.string(); + ASSERT_TRUE(fs::exists(mpath)) << mpath.string(); + + SlideProps p; + p.fname_int = ipath.string(); + p.fname_seg = mpath.string(); + FpImageOptions fp; + ImageLoader il; + ASSERT_TRUE(il.open(p, fp)) << ipath.string(); + + ASSERT_TRUE(il.load_volume(0, 0)); + const std::vector seg0 = il.get_seg_volume_buffer(); + const std::vector int0 = il.get_int_volume_buffer(); + size_t nz = 0; for (auto v : seg0) if (v) ++nz; + ASSERT_GT(nz, 0u) << "mask has no ROI voxels"; + + for (int c = 0; c < 3; ++c) + { + ASSERT_TRUE(il.load_volume(c, 0)) << "channel " << c; // must not throw/fail for c>0 + ASSERT_EQ(il.get_seg_volume_buffer(), seg0) << "mask changed for channel " << c; // reused identically + if (c > 0) + ASSERT_NE(il.get_int_volume_buffer(), int0) << "intensity channel " << c << " read c=0 data"; + } + il.close(); +} + +// Negative: the whole-volume facade read must propagate an out-of-range channel +// or timeframe as a throw (the (z,c,t)->IFD map range-guards). dim5 has C=3, T=2. +void test_ometiff_load_volume_out_of_range() +{ + fs::path ds = ometiff_data_path("dim5.ome.tif"); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + SlideProps p; + p.fname_int = ds.string(); + p.fname_seg = ""; + FpImageOptions fp; + ImageLoader il; + ASSERT_TRUE(il.open(p, fp)) << ds.string(); + + EXPECT_ANY_THROW(il.load_volume(99, 0)); // channel out of range (C=3) + EXPECT_ANY_THROW(il.load_volume(0, 99)); // timeframe out of range (T=2) + // in-range still works + EXPECT_TRUE(il.load_volume(2, 1)); + il.close(); +} + +// The strip loaders must ADVERTISE the OME C/T extents via numberChannels() / +// fullTimestamps() -- what the volumetric pipeline keys off to iterate channels +// and timeframes. A plain (non-OME) multi-page TIFF has no OME-XML, so it must keep +// the single-plane default of 1 for both (its pages are all Z-slices). +void test_ometiff_ct_counts(const char* store, int T, int C, int Z) +{ + fs::path ds = ometiff_data_path(store); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + auto ldr = NyxusGrayscaleTiffStripLoader(1, ds.string()); + ASSERT_EQ(ldr.numberChannels(), (size_t)C) << store; + ASSERT_EQ(ldr.fullTimestamps(0), (size_t)T) << store; + ASSERT_EQ(ldr.fullDepth(0), (size_t)Z) << store; + + auto raw = RawTiffStripLoader(1, ds.string()); + ASSERT_EQ(raw.numberChannels(), (size_t)C) << store; + ASSERT_EQ(raw.fullTimestamps(0), (size_t)T) << store; + ASSERT_EQ(raw.fullDepth(0), (size_t)Z) << store; +} + +// Negative: an out-of-range Z/C/T plane maps to a non-existent IFD and must throw. +// dim5.ome.tif has T=2, C=3, Z=4 (24 IFDs). +void test_ometiff_out_of_range_throws() +{ + fs::path ds = ometiff_data_path("dim5.ome.tif"); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + auto ldr = NyxusGrayscaleTiffStripLoader(1, ds.string()); + auto tile = std::make_shared>(ldr.tileHeight(0) * ldr.tileWidth(0), 0u); + EXPECT_ANY_THROW(ldr.loadTileFromFile(tile, 0, 0, 0, 99, 0, 0)); // channel out of range + EXPECT_ANY_THROW(ldr.loadTileFromFile(tile, 0, 0, 0, 0, 99, 0)); // timeframe out of range + EXPECT_ANY_THROW(ldr.loadTileFromFile(tile, 0, 0, 99, 0, 0, 0)); // z out of range + + auto raw = RawTiffStripLoader(1, ds.string()); + EXPECT_ANY_THROW(raw.loadTileFromFile(0, 0, 0, 99, 0, 0)); + EXPECT_ANY_THROW(raw.loadTileFromFile(0, 0, 0, 0, 99, 0)); + EXPECT_ANY_THROW(raw.loadTileFromFile(0, 0, 99, 0, 0, 0)); +} + +// Illegal / adversarial: a non-grayscale (RGB) OME-TIFF and a corrupt/non-TIFF file +// must be rejected cleanly (throw), not crash. +void test_ometiff_malformed_throws() +{ + for (const char* s : { "bad_rgb.ome.tif", "bad_corrupt.tif" }) + { + fs::path ds = ometiff_data_path(s); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + EXPECT_ANY_THROW(NyxusGrayscaleTiffStripLoader(1, ds.string())) << s; + EXPECT_ANY_THROW(RawTiffStripLoader(1, ds.string())) << s; + } + // a path that does not exist + EXPECT_ANY_THROW(RawTiffStripLoader(1, ometiff_data_path("does_not_exist.tif").string())); +} diff --git a/tests/test_omezarr_mechanics.h b/tests/test_omezarr_mechanics.h index cdd2eb12..b33196d5 100644 --- a/tests/test_omezarr_mechanics.h +++ b/tests/test_omezarr_mechanics.h @@ -8,6 +8,8 @@ #include #include "../src/nyx/omezarr.h" #include "../src/nyx/raw_omezarr.h" +#include "../src/nyx/image_loader.h" +#include "../src/nyx/globals.h" // scan_trivial_wholevolume, LR #include "../src/nyx/helpers/fsystem.h" // The OME-Zarr test datasets under tests/data/omezarr are generated with bfio @@ -60,7 +62,7 @@ void test_omezarr_tileloader_content() size_t tw = ldr.tileWidth(0); auto tile = std::make_shared>(th * tw, 0u); - ASSERT_NO_THROW(ldr.loadTileFromFile(tile, 0, 0, 0, 0)); + ASSERT_NO_THROW(ldr.loadTileFromFile(tile, 0, 0, 0, 0/*channel*/, 0/*timeframe*/, 0)); const std::vector& buf = *tile; @@ -106,7 +108,7 @@ void test_omezarr_tileloader_multitile() for (size_t tc = 0; tc < nCols; ++tc) { std::fill(tile->begin(), tile->end(), 0u); - ASSERT_NO_THROW(ldr.loadTileFromFile(tile, tr, tc, 0, 0)); + ASSERT_NO_THROW(ldr.loadTileFromFile(tile, tr, tc, 0, 0/*channel*/, 0/*timeframe*/, 0)); const std::vector& buf = *tile; const size_t row0 = tr * th; @@ -157,7 +159,7 @@ void test_raw_omezarr_content() auto ldr = RawOmezarrLoader(ds.string()); const size_t tw = ldr.tileWidth(0); - ASSERT_NO_THROW(ldr.loadTileFromFile(0, 0, 0, 0)); + ASSERT_NO_THROW(ldr.loadTileFromFile(0, 0, 0, 0/*channel*/, 0/*timeframe*/, 0)); // get_uint32_pixel indexes into the internal tile buffer (stride tileWidth) ASSERT_EQ(ldr.get_uint32_pixel(0 * tw + 0), 0u); @@ -195,7 +197,7 @@ void test_raw_omezarr_multitile() { for (size_t tc = 0; tc < nCols; ++tc) { - ASSERT_NO_THROW(ldr.loadTileFromFile(tr, tc, 0, 0)); + ASSERT_NO_THROW(ldr.loadTileFromFile(tr, tc, 0, 0/*channel*/, 0/*timeframe*/, 0)); const size_t row0 = tr * th; const size_t col0 = tc * tw; const size_t validH = std::min(th, H - row0); @@ -216,4 +218,313 @@ void test_raw_omezarr_multitile() ASSERT_EQ(total, 12681000000ull); } +// --------------------------------------------------------------------------- +// 5D (T,C,Z,Y,X) channel/timeframe addressability. +// +// dim5.ome.zarr (see gen_dim5.py) encodes every voxel as +// value(x,y,z,c,t) = 1 + ((((t*C + c)*Z + z)*Y + y)*X + x), C=3,Z=4,Y=6,X=8 +// chunked one z/c/t-plane per chunk. Reading plane (z,c,t) must return exactly +// that plane's values; a loader that ignored C/T (offset pinned to {0,0,...}, +// the pre-fix behavior) would return the c=0/t=0 plane for every (c,t). +// --------------------------------------------------------------------------- + +static inline uint32_t dim5_enc(int x, int y, int z, int c, int t) +{ + const int C = 3, Z = 4, Y = 6, X = 8; + return static_cast(1 + ((((t * C + c) * Z + z) * Y + y) * X + x)); +} + +// AbstractTileLoader stack. (T,C,Z) are the store's extents (X=8,Y=6 fixed). The +// encoded value is axis-order- AND rank-invariant, so one body covers the default +// TCZYX, the non-default CTZYX, the lower-rank 3D/2D stores, and the no-axes +// (legacy fallback) store — all of which must return the same encoded values. +void test_omezarr_addressing(const char* store, int T, int C, int Z) +{ + const int Y = 6, X = 8; + fs::path ds = omezarr_data_path(store); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + auto ldr = NyxusOmeZarrLoader(1, ds.string()); + ASSERT_EQ(ldr.fullWidth(0), (size_t)X); + ASSERT_EQ(ldr.fullHeight(0), (size_t)Y); + ASSERT_EQ(ldr.fullDepth(0), (size_t)Z); + const size_t tw = ldr.tileWidth(0); + auto tile = std::make_shared>(ldr.tileHeight(0) * tw, 0u); + + for (int t = 0; t < T; ++t) + for (int c = 0; c < C; ++c) + for (int z = 0; z < Z; ++z) + { + std::fill(tile->begin(), tile->end(), 0u); + ASSERT_NO_THROW(ldr.loadTileFromFile(tile, 0, 0, z, c, t, 0)); + const std::vector& buf = *tile; + for (int y = 0; y < Y; ++y) + for (int x = 0; x < X; ++x) + ASSERT_EQ(buf[y * tw + x], dim5_enc(x, y, z, c, t)) + << store << " plane (z" << z << " c" << c << " t" << t << ") at (" << x << "," << y << ")"; + } +} + +// RawFormatLoader stack (same coverage as above). +void test_raw_omezarr_addressing(const char* store, int T, int C, int Z) +{ + const int Y = 6, X = 8; + fs::path ds = omezarr_data_path(store); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + auto ldr = RawOmezarrLoader(ds.string()); + ASSERT_EQ(ldr.fullDepth(0), (size_t)Z); + const size_t tw = ldr.tileWidth(0); + + for (int t = 0; t < T; ++t) + for (int c = 0; c < C; ++c) + for (int z = 0; z < Z; ++z) + { + ASSERT_NO_THROW(ldr.loadTileFromFile(0, 0, z, c, t, 0)); + for (int y = 0; y < Y; ++y) + for (int x = 0; x < X; ++x) + ASSERT_EQ(ldr.get_uint32_pixel(y * tw + x), dim5_enc(x, y, z, c, t)) + << store << " plane (z" << z << " c" << c << " t" << t << ") at (" << x << "," << y << ")"; + } + + if (C > 1 && T > 1) // distinct (c,t) planes must differ -> the offset really uses C/T + { + ldr.loadTileFromFile(0, 0, 0, 0, 0, 0); uint32_t p000 = ldr.get_uint32_pixel(0); + ldr.loadTileFromFile(0, 0, 0, 1, 0, 0); uint32_t p010 = ldr.get_uint32_pixel(0); + ldr.loadTileFromFile(0, 0, 0, 0, 1, 0); uint32_t p001 = ldr.get_uint32_pixel(0); + ASSERT_NE(p000, p010) << "channel index ignored"; + ASSERT_NE(p000, p001) << "timeframe index ignored"; + } +} + +// End-to-end through the WIRED volumetric consumer: scan_trivial_wholevolume must +// feed the whole X*Y*Z volume (all Z planes) into the ROI's voxel cloud with the +// correct encoded intensity -- before the wiring it read only plane z=0. +void test_omezarr_wholevolume_consumer(const char* store, int Z) +{ + const int Y = 6, X = 8; + fs::path ds = omezarr_data_path(store); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + SlideProps p; + p.fname_int = ds.string(); + p.fname_seg = ""; + FpImageOptions fp; + ImageLoader ilo; + ASSERT_TRUE(ilo.open(p, fp)) << ds.string(); + + LR vroi; + ASSERT_TRUE(Nyxus::scan_trivial_wholevolume(vroi, ds.string(), ilo, 0/*channel*/, 0/*timeframe*/)); + + ASSERT_EQ(vroi.raw_pixels_3D.size(), (size_t)X * Y * Z); // all voxels, not just plane 0 (48) + for (const Pixel3& px : vroi.raw_pixels_3D) + ASSERT_EQ((uint32_t)px.inten, dim5_enc((int)px.x, (int)px.y, (int)px.z, 0, 0)) + << "voxel (" << px.x << "," << px.y << "," << px.z << ")"; + ilo.close(); +} + +// End-to-end through the wired volumetric consumer, but for EVERY (channel,timeframe): +// scan_trivial_wholevolume(vroi, .., c, t) must fill the ROI's voxel cloud with the +// encoded intensity of THAT c/t plane. Before the channel/timeframe wiring the consumer +// always read (c=0,t=0), so any c>0 / t>0 plane would carry the wrong values. +void test_omezarr_wholevolume_consumer_ct(const char* store, int T, int C, int Z) +{ + const int Y = 6, X = 8; + fs::path ds = omezarr_data_path(store); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + SlideProps p; + p.fname_int = ds.string(); + p.fname_seg = ""; + FpImageOptions fp; + ImageLoader ilo; + ASSERT_TRUE(ilo.open(p, fp)) << ds.string(); + + for (int t = 0; t < T; ++t) + for (int c = 0; c < C; ++c) + { + LR vroi; + ASSERT_TRUE(Nyxus::scan_trivial_wholevolume(vroi, ds.string(), ilo, c, t)) << store; + ASSERT_EQ(vroi.raw_pixels_3D.size(), (size_t)X * Y * Z) << store; + for (const Pixel3& px : vroi.raw_pixels_3D) + ASSERT_EQ((uint32_t)px.inten, dim5_enc((int)px.x, (int)px.y, (int)px.z, c, t)) + << store << " (c" << c << " t" << t << ") voxel (" << px.x << "," << px.y << "," << px.z << ")"; + } + ilo.close(); +} + +// Whole-volume assembly through the ImageLoader facade: load_volume() must stack +// all Z-planes (per (channel,timeframe)) into one X*Y*Z buffer. This is the +// foundation that lets the volumetric pipeline consume plane-by-plane OME-Zarr. +void test_omezarr_facade_volume(const char* store, int T, int C, int Z) +{ + const int Y = 6, X = 8; + fs::path ds = omezarr_data_path(store); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + SlideProps p; + p.fname_int = ds.string(); + p.fname_seg = ""; // whole-slide: intensity only + FpImageOptions fp; + ImageLoader il; + ASSERT_TRUE(il.open(p, fp)) << ds.string(); + ASSERT_EQ(il.get_full_width(), (size_t)X); + ASSERT_EQ(il.get_full_height(), (size_t)Y); + ASSERT_EQ(il.get_full_depth(), (size_t)Z); + + for (int t = 0; t < T; ++t) + for (int c = 0; c < C; ++c) + { + ASSERT_TRUE(il.load_volume(c, t)); + const std::vector& vol = il.get_int_volume_buffer(); + ASSERT_EQ(vol.size(), (size_t)X * Y * Z); + for (int z = 0; z < Z; ++z) + for (int y = 0; y < Y; ++y) + for (int x = 0; x < X; ++x) + ASSERT_EQ(vol[(size_t)z * X * Y + (size_t)y * X + x], dim5_enc(x, y, z, c, t)) + << store << " vol (x" << x << " y" << y << " z" << z << " c" << c << " t" << t << ")"; + } + il.close(); +} + +// Every one of the 6 legal orderings of {t,c,z} before y,x must read correctly +// (proves the axis-role resolution, not just the default TCZYX). +void test_omezarr_all_5d_permutations() +{ + for (const char* s : { "dim5.ome.zarr", "dim5_tzcyx.ome.zarr", "dim5_ctzyx.ome.zarr", + "dim5_cztyx.ome.zarr", "dim5_ztcyx.ome.zarr", "dim5_zctyx.ome.zarr" }) + { + test_omezarr_addressing(s, 2, 3, 4); + test_raw_omezarr_addressing(s, 2, 3, 4); + if (::testing::Test::HasFatalFailure()) return; + } +} + +// Negative: requesting a Z/C/T plane beyond the array extent must throw, not read +// out-of-bounds / wrong data. dim5.ome.zarr has T=2, C=3, Z=4. +void test_omezarr_out_of_range_throws() +{ + fs::path ds = omezarr_data_path("dim5.ome.zarr"); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + auto ldr = NyxusOmeZarrLoader(1, ds.string()); + auto tile = std::make_shared>(ldr.tileHeight(0) * ldr.tileWidth(0), 0u); + EXPECT_ANY_THROW(ldr.loadTileFromFile(tile, 0, 0, 0, 99, 0, 0)); // channel out of range + EXPECT_ANY_THROW(ldr.loadTileFromFile(tile, 0, 0, 0, 0, 99, 0)); // timeframe out of range + EXPECT_ANY_THROW(ldr.loadTileFromFile(tile, 0, 0, 99, 0, 0, 0)); // z out of range + + auto raw = RawOmezarrLoader(ds.string()); + EXPECT_ANY_THROW(raw.loadTileFromFile(0, 0, 0, 99, 0, 0)); + EXPECT_ANY_THROW(raw.loadTileFromFile(0, 0, 0, 0, 99, 0)); + EXPECT_ANY_THROW(raw.loadTileFromFile(0, 0, 99, 0, 0, 0)); +} + +// The loaders must ADVERTISE the real C/T extents via numberChannels() / +// fullTimestamps(). This is what the volumetric pipeline keys off to iterate +// channels and timeframes; before this both fell back to the base-class default +// of 1, which pinned the pipeline to plane (c=0, t=0) regardless of the store. +// Covers the parsed-axes path and (via dim5_noaxes) the positional fallback. +void test_omezarr_ct_counts(const char* store, int T, int C, int Z) +{ + fs::path ds = omezarr_data_path(store); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + auto ldr = NyxusOmeZarrLoader(1, ds.string()); + ASSERT_EQ(ldr.numberChannels(), (size_t)C) << store; + ASSERT_EQ(ldr.fullTimestamps(0), (size_t)T) << store; + ASSERT_EQ(ldr.fullDepth(0), (size_t)Z) << store; + + auto raw = RawOmezarrLoader(ds.string()); + ASSERT_EQ(raw.numberChannels(), (size_t)C) << store; + ASSERT_EQ(raw.fullTimestamps(0), (size_t)T) << store; + ASSERT_EQ(raw.fullDepth(0), (size_t)Z) << store; +} + +// Negative: the whole-volume facade read must propagate an out-of-range channel +// or timeframe as a throw (not silently read plane 0 or OOB memory). dim5 has C=3, T=2. +void test_omezarr_load_volume_out_of_range() +{ + fs::path ds = omezarr_data_path("dim5.ome.zarr"); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + + SlideProps p; + p.fname_int = ds.string(); + p.fname_seg = ""; + FpImageOptions fp; + ImageLoader il; + ASSERT_TRUE(il.open(p, fp)) << ds.string(); + + EXPECT_ANY_THROW(il.load_volume(99, 0)); // channel out of range (C=3) + EXPECT_ANY_THROW(il.load_volume(0, 99)); // timeframe out of range (T=2) + // in-range still works + EXPECT_TRUE(il.load_volume(2, 1)); + il.close(); +} + +// Physical calibration: the loaders must surface the OME-Zarr coordinateTransformations +// 'scale' as physicalSizeX/Y/Z and the space-axis 'unit'. dim5_calibrated encodes +// scale (t,c,z,y,x)=(1,1,2,0.5,0.5) micrometer -> physX=physY=0.5, physZ=2.0. An +// uncalibrated store (scale all 1) must report 1.0 and no unit. +void test_omezarr_physical_calibration() +{ + fs::path cal = omezarr_data_path("dim5_calibrated.ome.zarr"); + ASSERT_TRUE(fs::exists(cal)) << cal.string(); + + auto ldr = NyxusOmeZarrLoader(1, cal.string()); + ASSERT_DOUBLE_EQ(ldr.physicalSizeX(), 0.5); + ASSERT_DOUBLE_EQ(ldr.physicalSizeY(), 0.5); + ASSERT_DOUBLE_EQ(ldr.physicalSizeZ(), 2.0); + ASSERT_EQ(ldr.physicalSizeUnit(), "micrometer"); + + auto raw = RawOmezarrLoader(cal.string()); + ASSERT_DOUBLE_EQ(raw.physicalSizeX(), 0.5); + ASSERT_DOUBLE_EQ(raw.physicalSizeY(), 0.5); + ASSERT_DOUBLE_EQ(raw.physicalSizeZ(), 2.0); + ASSERT_EQ(raw.physicalSizeUnit(), "micrometer"); + + // uncalibrated store: scale all 1.0 -> physical sizes default to 1.0 + fs::path plain = omezarr_data_path("dim5.ome.zarr"); + auto ldr2 = NyxusOmeZarrLoader(1, plain.string()); + ASSERT_DOUBLE_EQ(ldr2.physicalSizeX(), 1.0); + ASSERT_DOUBLE_EQ(ldr2.physicalSizeZ(), 1.0); +} + +// Unit canonicalization: dim5_calibrated_nm declares the SAME physical spacing as +// dim5_calibrated above, but in nanometer (2000/500/500 nm == 2.0/0.5/0.5 um). The loader +// must report the SAME canonicalized values and unit as the micrometer fixture -- proving +// actual conversion happens, not just passthrough of whatever unit string the file declares. +void test_omezarr_unit_canonicalization() +{ + fs::path cal_nm = omezarr_data_path("dim5_calibrated_nm.ome.zarr"); + ASSERT_TRUE(fs::exists(cal_nm)) << cal_nm.string(); + + auto ldr = NyxusOmeZarrLoader(1, cal_nm.string()); + ASSERT_DOUBLE_EQ(ldr.physicalSizeX(), 0.5); + ASSERT_DOUBLE_EQ(ldr.physicalSizeY(), 0.5); + ASSERT_DOUBLE_EQ(ldr.physicalSizeZ(), 2.0); + ASSERT_EQ(ldr.physicalSizeUnit(), "micrometer"); + + auto raw = RawOmezarrLoader(cal_nm.string()); + ASSERT_DOUBLE_EQ(raw.physicalSizeX(), 0.5); + ASSERT_DOUBLE_EQ(raw.physicalSizeY(), 0.5); + ASSERT_DOUBLE_EQ(raw.physicalSizeZ(), 2.0); + ASSERT_EQ(raw.physicalSizeUnit(), "micrometer"); +} + +// Illegal / adversarial: self-inconsistent metadata must be rejected cleanly +// (throw), not crash. bad_axes_count declares 5 axes for a 3D array (indexing the +// shape by axis role would read OOB); bad_no_xy has axes but none labeled x/y. +void test_omezarr_malformed_throws() +{ + for (const char* s : { "bad_axes_count.ome.zarr", "bad_no_xy.ome.zarr" }) + { + fs::path ds = omezarr_data_path(s); + ASSERT_TRUE(fs::exists(ds)) << ds.string(); + EXPECT_ANY_THROW(NyxusOmeZarrLoader(1, ds.string())) << s; + EXPECT_ANY_THROW(RawOmezarrLoader(ds.string())) << s; + } + // a store path that does not exist at all + EXPECT_ANY_THROW(RawOmezarrLoader(omezarr_data_path("does_not_exist.ome.zarr").string())); +} + #endif // OMEZARR_SUPPORT diff --git a/tests/test_tiff_loader_mechanics.h b/tests/test_tiff_loader_mechanics.h index 9ff33d5c..6f1a6f9c 100644 --- a/tests/test_tiff_loader_mechanics.h +++ b/tests/test_tiff_loader_mechanics.h @@ -83,7 +83,7 @@ inline void test_uint32_strip_tiff_loader() auto tile = std::make_shared>(tw * th * td); // Single tile at (row,col,layer,level) = (0,0,0,0) covers this small image. - loader.loadTileFromFile(tile, 0, 0, 0, 0); + loader.loadTileFromFile(tile, 0, 0, 0, 0/*channel*/, 0/*timeframe*/, 0); for (uint32_t y = 0; y < h; ++y) for (uint32_t x = 0; x < w; ++x) diff --git a/tests/vetting/oracle_coverage.csv b/tests/vetting/oracle_coverage.csv index 8882b744..519ab7aa 100644 --- a/tests/vetting/oracle_coverage.csv +++ b/tests/vetting/oracle_coverage.csv @@ -1,6 +1,6 @@ dim,feature,family,status,oracle,agreement,config_recipe,tolerance,current_test,target_test,candidate_oracle,flag,source,notes 2D,COV,firstorder,vetted,matlab,agreed,"Not mode-specific. Any disagreement is a definition, coordinate-frame, spacing, aggregation, or implementation issue rather than Nyxus IBSI/radiomics mode.",,test_3d_inten.h;test_nyxus.py;test_firstorder_regression.h,test_firstorder_matlab.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A -2D,COVERED_IMAGE_INTENSITY_RANGE,firstorder,vetted,matlab,agreed,"Not mode-specific. Any disagreement is a definition, coordinate-frame, spacing, aggregation, or implementation issue rather than Nyxus IBSI/radiomics mode.",,test_3d_inten.h;test_nyxus.py;test_firstorder_regression.h,test_firstorder_matlab.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A +2D,COVERED_IMAGE_INTENSITY_RANGE,firstorder,vetted,matlab,agreed,"Not mode-specific. Any disagreement is a definition, coordinate-frame, spacing, aggregation, or implementation issue rather than Nyxus IBSI/radiomics mode.",,test_3d_inten.h;test_nyxus.py;test_firstorder_regression.h;test_covered_intensity_range_regression.py,test_firstorder_matlab.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A 2D,ENERGY,firstorder,vetted,matlab,agreed,"Not mode-specific. Any disagreement is a definition, coordinate-frame, spacing, aggregation, or implementation issue rather than Nyxus IBSI/radiomics mode.",,test_3d_inten.h;test_glcm.h;test_firstorder_ibsi.h;test_firstorder_regression.h;test_firstorder_pyradiomics.h,test_firstorder_matlab.h,,,tracker,"test_3d_inten.h ORPHANED (not #included; never run) - left per decision A | corroborated: pyradiomics exact (rel=0.00e+00), recipe=firstorder.pyradiomics_default, see test_firstorder_pyradiomics.h" 2D,ENTROPY,firstorder,vetted,pyradiomics,exact (rel=3.20e-16),firstorder.pyradiomics_default,exact,test_3d_inten.h;test_glcm.h;test_firstorder_regression.h;test_firstorder_pyradiomics.h,test_firstorder_regression.h,MIRP/IBSI check,promote-after-deepdive,tracker,"test_3d_inten.h ORPHANED (not #included; never run) - left per decision A | PROMOTED regression->vetted by pyradiomics exact (rel=3.20e-16), recipe=firstorder.pyradiomics_default, see test_firstorder_pyradiomics.h" 2D,EXCESS_KURTOSIS,firstorder,vetted,matlab,agreed,"Not mode-specific. Any disagreement is a definition, coordinate-frame, spacing, aggregation, or implementation issue rather than Nyxus IBSI/radiomics mode.",,test_3d_inten.h;test_firstorder_ibsi.h;test_firstorder_regression.h,test_firstorder_matlab.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A @@ -538,219 +538,219 @@ dim,feature,family,status,oracle,agreement,config_recipe,tolerance,current_test, 2D,IH_ROBUST_MEAN_IDX,intensity_histogram,vetted,analytic,,ih.ibsi_fbn,,test_intensity_histogram_ibsi.h,,mirp (IBSI Intensity-Histogram family); vet _IDX,not-in-tracker;oracle-identified,audit,"vetted by test_ih_dispersion_robust_analytic (hand-computed on the robust-window discriminating fixture; no clean IBSI anchor - robust-mean has no IBSI feature; QCoD_VAL/IQR_VAL are Nyxus continuous-percentile extensions of the IBSI discrete-percentile IQR/QCoD, so they have no IBSI counterpart by design)" 2D,IH_NUM_BINS,intensity_histogram,vetted,analytic,,,,test_intensity_histogram_regression.h;test_intensity_histogram.py,test_intensity_histogram_analytic.h,numpy/scipy analytic,not-in-tracker,audit, 2D,IH_BIN_SIZE,intensity_histogram,vetted,analytic,,,,test_intensity_histogram_regression.h;test_intensity_histogram.py,test_intensity_histogram_analytic.h,numpy/scipy analytic,not-in-tracker,audit, -3D,3COV,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h,test_3d_firstorder_regression.h,mirp:stat_cov (IBSI 7TET; no pyradiomics equiv),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) -3D,3COVERED_IMAGE_INTENSITY_RANGE,firstorder,regression,,na,,,test_3d_firstorder_coverage.h;test_3d_inten.h,test_3d_firstorder_regression.h,none (Nyxus-specific: (roi_max-roi_min)/(slide_max-slide_min)),bug-degenerate,tracker,BUG: 3D segmented workflow never sets roi.slide_idx (workflow_3d_segmented.cpp) so 3d_intensity.cpp takes the else branch and always emits degenerate 1; unvettable until slide_idx+min/max_preroi_inten are wired through the 3D pipeline (2D sets it via pixel_feed.cpp). Oracle (octave/oracle_3d) confirms intended value = (roi_range)/(slide_range) -3D,3ENERGY,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A -3D,3ENTROPY,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A -3D,3EXCESS_KURTOSIS,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h,test_3d_firstorder_regression.h,pyradiomics/mirp (native 3D first-order; kurtosis -3),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) -3D,3HYPERFLATNESS,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h,test_3d_firstorder_regression.h,none (scipy.moment order 5/6 only),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) -3D,3HYPERSKEWNESS,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h,test_3d_firstorder_regression.h,none (scipy.moment order 5/6 only),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) -3D,3INTEGRATED_INTENSITY,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h,test_3d_firstorder_regression.h,numpy (trivial closed-form; IBSI has only P10/P90),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) -3D,3INTERQUARTILE_RANGE,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A -3D,3KURTOSIS,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A -3D,3MAX,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A -3D,3MEAN,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A -3D,3MEAN_ABSOLUTE_DEVIATION,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A -3D,3MEDIAN,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A -3D,3MEDIAN_ABSOLUTE_DEVIATION,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h,test_3d_firstorder_regression.h,mirp:stat_medad (IBSI N72L),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) -3D,3MIN,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A -3D,3MODE,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h,test_3d_firstorder_regression.h,scipy.stats.mode on raw voxels (IBSI mode is on histogram),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) -3D,3P01,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h,test_3d_firstorder_regression.h,numpy (trivial closed-form; IBSI has only P10/P90),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d); Nyxus 100-bin CDF-interp reproduced exactly; ~0.2% vs prctile (definitional) -3D,3P10,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A -3D,3P25,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h,test_3d_firstorder_regression.h,numpy (trivial closed-form; IBSI has only P10/P90),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d); Nyxus 100-bin CDF-interp reproduced exactly; ~0.2% vs prctile (definitional) -3D,3P75,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h,test_3d_firstorder_regression.h,numpy (trivial closed-form; IBSI has only P10/P90),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d); Nyxus 100-bin CDF-interp reproduced exactly; ~0.2% vs prctile (definitional) -3D,3P90,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A -3D,3P99,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h,test_3d_firstorder_regression.h,numpy (trivial closed-form; IBSI has only P10/P90),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d); Nyxus 100-bin CDF-interp reproduced exactly; ~0.2% vs prctile (definitional) -3D,3QCOD,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h,test_3d_firstorder_regression.h,pyradiomics/mirp (native 3D first-order; kurtosis -3),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d); Nyxus 100-bin CDF-interp reproduced exactly; ~0.2% vs prctile (definitional) -3D,3RANGE,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A -3D,3ROBUST_MEAN,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h,test_3d_firstorder_regression.h,octave/MATLAB (mean of voxels in [P10,P90]),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; rel<1e-9) after fixing 3ROBUST_MEAN which was hardcoded 0 in both 3d_intensity.cpp paths -3D,3ROBUST_MEAN_ABSOLUTE_DEVIATION,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A -3D,3ROOT_MEAN_SQUARED,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A -3D,3SKEWNESS,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A -3D,3STANDARD_DEVIATION,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h,test_3d_firstorder_regression.h,pyradiomics/mirp (native 3D first-order; kurtosis -3),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) -3D,3STANDARD_DEVIATION_BIASED,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h,test_3d_firstorder_regression.h,pyradiomics/mirp (native 3D first-order; kurtosis -3),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) -3D,3STANDARD_ERROR,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h,test_3d_firstorder_regression.h,pyradiomics/mirp (native 3D first-order; kurtosis -3),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) -3D,3VARIANCE,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A -3D,3VARIANCE_BIASED,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h,test_3d_firstorder_regression.h,pyradiomics/mirp (native 3D first-order; kurtosis -3),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) -3D,3UNIFORMITY,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A -3D,3UNIFORMITY_PIU,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h,test_3d_firstorder_regression.h,pyradiomics/mirp (native 3D first-order; kurtosis -3),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) -3D,3AREA,morphology,regression,,needs_audit,,,test_3d_morphology_coverage.h;test_3d_morphology_regression.h,test_3d_morphology_regression.h,matlab,convention-mismatch,tracker,"convention: Nyxus surface area is voxel-face-based (3AREA 59992) vs MIRP/pyradiomics mesh area (46739, ~28% diff); all area/volume-derived features inherit it. Needs surface-convention decision (see Vetting-Work-Log)." -3D,3AREA_2_VOLUME,morphology,regression,,na,,,test_3d_morphology_coverage.h;test_3d_morphology_regression.h,test_3d_morphology_regression.h,pyradiomics:SurfaceVolumeRatio(3D),convention-mismatch,tracker,"convention: Nyxus surface area is voxel-face-based (3AREA 59992) vs MIRP/pyradiomics mesh area (46739, ~28% diff); all area/volume-derived features inherit it. Needs surface-convention decision (see Vetting-Work-Log)." -3D,3COMPACTNESS1,morphology,regression,,na,,,test_3d_morphology_coverage.h;test_3d_morphology_regression.h,test_3d_morphology_regression.h,pyradiomics:Compactness1/Compactness2(3D),convention-mismatch,tracker,"convention: Nyxus surface area is voxel-face-based (3AREA 59992) vs MIRP/pyradiomics mesh area (46739, ~28% diff); all area/volume-derived features inherit it. Needs surface-convention decision (see Vetting-Work-Log)." -3D,3COMPACTNESS2,morphology,regression,,na,,,test_3d_morphology_coverage.h;test_3d_morphology_regression.h,test_3d_morphology_regression.h,pyradiomics:Compactness1/Compactness2(3D),convention-mismatch,tracker,"convention: Nyxus surface area is voxel-face-based (3AREA 59992) vs MIRP/pyradiomics mesh area (46739, ~28% diff); all area/volume-derived features inherit it. Needs surface-convention decision (see Vetting-Work-Log)." -3D,3MESH_VOLUME,morphology,vetted,matlab,agreed,,,test_3d_morphology_coverage.h;test_3d_morphology_matlab.h,test_3d_morphology_matlab.h,,,tracker, -3D,3SPHERICAL_DISPROPORTION,morphology,regression,,na,,,test_3d_morphology_coverage.h;test_3d_morphology_regression.h,test_3d_morphology_regression.h,pyradiomics:SphericalDisproportion(3D shape),convention-mismatch,tracker,"convention: Nyxus surface area is voxel-face-based (3AREA 59992) vs MIRP/pyradiomics mesh area (46739, ~28% diff); all area/volume-derived features inherit it. Needs surface-convention decision (see Vetting-Work-Log)." -3D,3SPHERICITY,morphology,regression,,na,,,test_3d_morphology_coverage.h;test_3d_morphology_regression.h,test_3d_morphology_regression.h,pyradiomics:Sphericity(3D shape),convention-mismatch,tracker,"convention: Nyxus surface area is voxel-face-based (3AREA 59992) vs MIRP/pyradiomics mesh area (46739, ~28% diff); all area/volume-derived features inherit it. Needs surface-convention decision (see Vetting-Work-Log)." -3D,3VOLUME_CONVEXHULL,morphology,vetted,matlab,agreed,,,test_3d_morphology_coverage.h;test_3d_morphology_regression.h,test_3d_morphology_matlab.h,,,tracker, -3D,3VOXEL_VOLUME,morphology,vetted,matlab,agreed,,,test_3d_morphology_coverage.h;test_3d_morphology_regression.h,test_3d_morphology_matlab.h,,,tracker, -3D,3MAJOR_AXIS_LEN,morphology,vetted,mirp,agreed,,,test_3d_morphology_coverage.h,test_3d_morphology_regression.h,pyradiomics:MajorAxisLength(3D shape),,tracker,"FIXED axis-length mislabel (3d_surface.cpp used wrong eigenvalue indices -> LEAST>MAJOR, FLATNESS>1) + vetted vs MIRP morph_pca_* (exact): MAJOR 104.71, MINOR 88.30, LEAST 71.51, ELONGATION 0.843, FLATNESS 0.683" -3D,3MINOR_AXIS_LEN,morphology,vetted,mirp,agreed,,,test_3d_morphology_coverage.h,test_3d_morphology_regression.h,pyradiomics:MinorAxisLength(3D shape),,tracker,"FIXED axis-length mislabel (3d_surface.cpp used wrong eigenvalue indices -> LEAST>MAJOR, FLATNESS>1) + vetted vs MIRP morph_pca_* (exact): MAJOR 104.71, MINOR 88.30, LEAST 71.51, ELONGATION 0.843, FLATNESS 0.683" -3D,3LEAST_AXIS_LEN,morphology,vetted,mirp,agreed,,,test_3d_morphology_coverage.h,test_3d_morphology_regression.h,pyradiomics:LeastAxisLength(3D shape),,tracker,"FIXED axis-length mislabel (3d_surface.cpp used wrong eigenvalue indices -> LEAST>MAJOR, FLATNESS>1) + vetted vs MIRP morph_pca_* (exact): MAJOR 104.71, MINOR 88.30, LEAST 71.51, ELONGATION 0.843, FLATNESS 0.683" -3D,3ELONGATION,morphology,vetted,mirp,agreed,,,test_3d_morphology_coverage.h,test_3d_morphology_regression.h,pyradiomics:Elongation(3D shape),,tracker,"FIXED axis-length mislabel (3d_surface.cpp used wrong eigenvalue indices -> LEAST>MAJOR, FLATNESS>1) + vetted vs MIRP morph_pca_* (exact): MAJOR 104.71, MINOR 88.30, LEAST 71.51, ELONGATION 0.843, FLATNESS 0.683" -3D,3FLATNESS,morphology,vetted,mirp,agreed,,,test_3d_morphology_coverage.h,test_3d_morphology_regression.h,pyradiomics:Flatness(3D shape),,tracker,"FIXED axis-length mislabel (3d_surface.cpp used wrong eigenvalue indices -> LEAST>MAJOR, FLATNESS>1) + vetted vs MIRP morph_pca_* (exact): MAJOR 104.71, MINOR 88.30, LEAST 71.51, ELONGATION 0.843, FLATNESS 0.683" -3D,3GLCM_ACOR,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_ASM,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_CLUPROM,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_CLUSHADE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_CLUTEND,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_CONTRAST,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_CORRELATION,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_DIFAVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_DIFENTRO,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_DIFVAR,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_DIS,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_regression.h,"pyradiomics/mirp (DIS=DifferenceAverage, HOM1=Id, HOM2=Idm, SUMVAR=ClusterTendency, ENERGY=JointEnergy=ASM, VAR=mirp cm_var); CONFIG-SENSITIVE: symmetric+13dir vs Nyxus asym/1-offset/100lvl",,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_DIFAVE (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" -3D,3GLCM_ENERGY,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h,test_3d_glcm_regression.h,"pyradiomics/mirp (DIS=DifferenceAverage, HOM1=Id, HOM2=Idm, SUMVAR=ClusterTendency, ENERGY=JointEnergy=ASM, VAR=mirp cm_var); CONFIG-SENSITIVE: symmetric+13dir vs Nyxus asym/1-offset/100lvl",,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_ASM (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" -3D,3GLCM_ENTROPY,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h,test_3d_glcm_regression.h,"pyradiomics/mirp (DIS=DifferenceAverage, HOM1=Id, HOM2=Idm, SUMVAR=ClusterTendency, ENERGY=JointEnergy=ASM, VAR=mirp cm_var); CONFIG-SENSITIVE: symmetric+13dir vs Nyxus asym/1-offset/100lvl",,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_JE (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" -3D,3GLCM_HOM1,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h,test_3d_glcm_regression.h,"pyradiomics/mirp (DIS=DifferenceAverage, HOM1=Id, HOM2=Idm, SUMVAR=ClusterTendency, ENERGY=JointEnergy=ASM, VAR=mirp cm_var); CONFIG-SENSITIVE: symmetric+13dir vs Nyxus asym/1-offset/100lvl",,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_ID (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" -3D,3GLCM_HOM2,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h,test_3d_glcm_regression.h,"pyradiomics/mirp (DIS=DifferenceAverage, HOM1=Id, HOM2=Idm, SUMVAR=ClusterTendency, ENERGY=JointEnergy=ASM, VAR=mirp cm_var); CONFIG-SENSITIVE: symmetric+13dir vs Nyxus asym/1-offset/100lvl",,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_IDM (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" -3D,3GLCM_ID,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_IDN,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_IDM,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_IDMN,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_INFOMEAS1,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_INFOMEAS2,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_IV,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_JAVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_JE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_JMAX,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_JVAR,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_SUMAVERAGE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_SUMENTROPY,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A -3D,3GLCM_SUMVARIANCE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h,test_3d_glcm_regression.h,"pyradiomics/mirp (DIS=DifferenceAverage, HOM1=Id, HOM2=Idm, SUMVAR=ClusterTendency, ENERGY=JointEnergy=ASM, VAR=mirp cm_var); CONFIG-SENSITIVE: symmetric+13dir vs Nyxus asym/1-offset/100lvl",,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_CLUTEND (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" -3D,3GLCM_VARIANCE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h,test_3d_glcm_regression.h,"pyradiomics/mirp (DIS=DifferenceAverage, HOM1=Id, HOM2=Idm, SUMVAR=ClusterTendency, ENERGY=JointEnergy=ASM, VAR=mirp cm_var); CONFIG-SENSITIVE: symmetric+13dir vs Nyxus asym/1-offset/100lvl",,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_JVAR (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" -3D,3GLCM_ASM_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" -3D,3GLCM_ACOR_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" -3D,3GLCM_CLUPROM_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" -3D,3GLCM_CLUSHADE_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" -3D,3GLCM_CLUTEND_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" -3D,3GLCM_CONTRAST_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" -3D,3GLCM_CORRELATION_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" -3D,3GLCM_DIFAVE_AVE,glcm,vetted,mirp,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_mirp.h,,,tracker, -3D,3GLCM_DIFENTRO_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" -3D,3GLCM_DIFVAR_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" -3D,3GLCM_DIS_AVE,glcm,vetted,mirp,agreed,,,test_3d_glcm_coverage.h,test_3d_glcm_mirp.h,,,tracker, -3D,3GLCM_ENERGY_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_ASM_AVE (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" -3D,3GLCM_ENTROPY_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_JE_AVE (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" -3D,3GLCM_HOM1_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_ID_AVE (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" -3D,3GLCM_ID_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" -3D,3GLCM_IDN_AVE,glcm,vetted,mirp,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_mirp.h,,,tracker, -3D,3GLCM_IDM_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" -3D,3GLCM_IDMN_AVE,glcm,vetted,mirp,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_mirp.h,,,tracker, -3D,3GLCM_IV_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" -3D,3GLCM_JAVE_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" -3D,3GLCM_JE_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" -3D,3GLCM_INFOMEAS1_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" -3D,3GLCM_INFOMEAS2_AVE,glcm,vetted,mirp,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_mirp.h,,,tracker, -3D,3GLCM_VARIANCE_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_JVAR_AVE (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" -3D,3GLCM_JMAX_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" -3D,3GLCM_JVAR_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" -3D,3GLCM_SUMAVERAGE_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" -3D,3GLCM_SUMENTROPY_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" -3D,3GLCM_SUMVARIANCE_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_CLUTEND_AVE (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" -3D,3GLDM_SDE,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLDM_LDE,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLDM_GLN,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLDM_DN,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLDM_DNN,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLDM_GLV,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLDM_DV,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLDM_DE,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLDM_LGLE,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLDM_HGLE,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLDM_SDLGLE,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLDM_SDHGLE,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLDM_LDLGLE,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLDM_LDHGLE,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLDZM_SDE,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, -3D,3GLDZM_LDE,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, -3D,3GLDZM_LGLZE,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, -3D,3GLDZM_HGLZE,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, -3D,3GLDZM_SDLGLE,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, -3D,3GLDZM_SDHGLE,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, -3D,3GLDZM_LDLGLE,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, -3D,3GLDZM_LDHGLE,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, -3D,3GLDZM_GLNU,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, -3D,3GLDZM_GLNUN,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, -3D,3GLDZM_ZDNU,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, -3D,3GLDZM_ZDNUN,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, -3D,3GLDZM_ZP,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, -3D,3GLDZM_GLM,gldzm,regression,,na,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h,test_3d_gldzm_regression.h,none (intermediate mean; indirect via GLV/ZDV/DCV variances),no-mainstream-oracle,tracker, -3D,3GLDZM_GLV,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, -3D,3GLDZM_ZDM,gldzm,regression,,na,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h,test_3d_gldzm_regression.h,none (intermediate mean; indirect via GLV/ZDV/DCV variances),no-mainstream-oracle,tracker, -3D,3GLDZM_ZDV,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, -3D,3GLDZM_ZDE,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, -3D,3NGLDM_LDE,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_LDE: table=0.1 vs MIRP=0.2559." -3D,3NGLDM_HDE,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_HDE: table=261 vs MIRP=28.07." -3D,3NGLDM_LGLCE,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_LGLCE: table=0.00036 vs MIRP=0.0322." -3D,3NGLDM_HGLCE,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_HGLCE: table=740 vs MIRP=1324." -3D,3NGLDM_LDLGLE,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_LDLGLE: table=5.8e-05 vs MIRP=0.000685." -3D,3NGLDM_LDHGLE,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_LDHGLE: table=74 vs MIRP=474.8." -3D,3NGLDM_HDLGLE,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_HDLGLE: table=0.025 vs MIRP=8.714." -3D,3NGLDM_HDHGLE,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_HDHGLE: table=20030 vs MIRP=14942.8." -3D,3NGLDM_GLNU,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_GLNU: table=115443 vs MIRP=4350.3." -3D,3NGLDM_GLNUN,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_GLNUN: table=0.23 vs MIRP=0.01585." -3D,3NGLDM_DCNU,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_DCNU: table=115443 vs MIRP=40745.0." -3D,3NGLDM_DCNUN,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_DCNUN: table=0.23 vs MIRP=0.14847." -3D,3NGLDM_DCP,ngldm,vetted,mirp,agreed,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h,test_3d_ngldm_mirp.h,,,tracker, -3D,3NGLDM_GLM,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h,test_3d_ngldm_regression.h,mirp,no-external-oracle,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. Additionally 3NGLDM_GLM has NO counterpart in IBSI/MIRP at all (MIRP's NGLDM emits no grey-level-mean or dependence-count-mean column; the 2D table in test_ngldm_ibsi.h likewise marks GLM '--not in IBSI--'), so no external oracle exists for it." -3D,3NGLDM_GLV,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_GLV: table=190 vs MIRP=350.17." -3D,3NGLDM_DCM,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h,test_3d_ngldm_regression.h,mirp,no-external-oracle,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. Additionally 3NGLDM_DCM has NO counterpart in IBSI/MIRP at all (MIRP's NGLDM emits no grey-level-mean or dependence-count-mean column; the 2D table in test_ngldm_ibsi.h likewise marks GLM '--not in IBSI--'), so no external oracle exists for it." -3D,3NGLDM_DCV,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_DCV: table=86.17 vs MIRP=11.948." -3D,3NGLDM_DCENT,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_DCENT: table=5.23 vs MIRP=8.676." -3D,3NGLDM_DCENE,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_DCENE: table=0.14 vs MIRP=0.002875." -3D,3NGTDM_COARSENESS,ngtdm,vetted,pyradiomics,agreed,,,test_3d_ngtdm_coverage.h;test_3d_ngtdm.h;test_3d_ngtdm_pyradiomics.h;test_nyxus.py,test_3d_ngtdm_pyradiomics.h,,,tracker,test_3d_ngtdm.h ORPHANED (not #included; never run) - left per decision A -3D,3NGTDM_CONTRAST,ngtdm,vetted,pyradiomics,agreed,,,test_3d_ngtdm_coverage.h;test_3d_ngtdm.h;test_3d_ngtdm_pyradiomics.h;test_nyxus.py,test_3d_ngtdm_pyradiomics.h,,,tracker,test_3d_ngtdm.h ORPHANED (not #included; never run) - left per decision A -3D,3NGTDM_BUSYNESS,ngtdm,vetted,pyradiomics,agreed,,,test_3d_ngtdm_coverage.h;test_3d_ngtdm.h;test_3d_ngtdm_pyradiomics.h;test_nyxus.py,test_3d_ngtdm_pyradiomics.h,,,tracker,test_3d_ngtdm.h ORPHANED (not #included; never run) - left per decision A -3D,3NGTDM_COMPLEXITY,ngtdm,vetted,pyradiomics,agreed,,,test_3d_ngtdm_coverage.h;test_3d_ngtdm.h;test_3d_ngtdm_pyradiomics.h;test_nyxus.py,test_3d_ngtdm_pyradiomics.h,,,tracker,test_3d_ngtdm.h ORPHANED (not #included; never run) - left per decision A -3D,3NGTDM_STRENGTH,ngtdm,vetted,pyradiomics,agreed,,,test_3d_ngtdm_coverage.h;test_3d_ngtdm.h;test_3d_ngtdm_pyradiomics.h;test_nyxus.py,test_3d_ngtdm_pyradiomics.h,,,tracker,test_3d_ngtdm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLSZM_SAE,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLSZM_LAE,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLSZM_GLN,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLSZM_GLNN,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLSZM_SZN,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLSZM_SZNN,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLSZM_ZP,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLSZM_GLV,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLSZM_ZV,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLSZM_ZE,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLSZM_LGLZE,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLSZM_HGLZE,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLSZM_SALGLE,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLSZM_SAHGLE,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLSZM_LALGLE,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLSZM_LAHGLE,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLRLM_SRE,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLRLM_LRE,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLRLM_GLN,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLRLM_GLNN,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLRLM_RLN,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLRLM_RLNN,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLRLM_RP,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLRLM_GLV,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLRLM_RV,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLRLM_RE,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLRLM_LGLRE,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLRLM_HGLRE,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLRLM_SRLGLE,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLRLM_SRHGLE,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLRLM_LRLGLE,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLRLM_LRHGLE,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A -3D,3GLRLM_SRE_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." -3D,3GLRLM_LRE_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." -3D,3GLRLM_GLN_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." -3D,3GLRLM_GLNN_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." -3D,3GLRLM_RLN_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." -3D,3GLRLM_RLNN_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." -3D,3GLRLM_RP_AVE,glrlm,vetted,mirp,agreed,,,test_3d_glrlm_coverage.h;test_nyxus.py,test_3d_glrlm_mirp.h,,,tracker, -3D,3GLRLM_GLV_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." -3D,3GLRLM_RV_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." -3D,3GLRLM_RE_AVE,glrlm,vetted,mirp,agreed,,,test_3d_glrlm_coverage.h;test_nyxus.py,test_3d_glrlm_mirp.h,,,tracker, -3D,3GLRLM_LGLRE_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." -3D,3GLRLM_HGLRE_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." -3D,3GLRLM_SRLGLE_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." -3D,3GLRLM_SRHGLE_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." -3D,3GLRLM_LRLGLE_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." -3D,3GLRLM_LRHGLE_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." +3D,3COV,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_ooc_mechanics.py,test_3d_firstorder_regression.h,mirp:stat_cov (IBSI 7TET; no pyradiomics equiv),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) +3D,3COVERED_IMAGE_INTENSITY_RANGE,firstorder,regression,,na,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_ooc_mechanics.py;test_covered_intensity_range_regression.py,test_3d_firstorder_regression.h,none (Nyxus-specific: (roi_max-roi_min)/(slide_max-slide_min)),fixed-2026-07-22,tracker,"FIXED (2026-07-22): the real bug was NOT roi.slide_idx (that has been correctly wired since gatherRoisMetrics_3D was written -- verified via git history). The actual defect: gatherRoisMetrics_2_slideprops_3D (slideprops.cpp) computed the whole-slide min/max baseline via RawImageLoader::for_each_voxel, which only visited IN-MASK voxels, so for any segmented (non-whole-slide) image the baseline silently collapsed to the ROI's own min/max and the ratio was always exactly 1. Same root cause in the 2D counterpart (gatherRoisMetrics_2_slideprops), also fixed. Fix: read every voxel's intensity for the baseline, gate only ROI geometry on the mask. Verified against an independently-computed expected ratio in test_covered_intensity_range_regression.py (both 2D and 3D). Oracle (octave/oracle_3d) already confirmed the intended formula = (roi_range)/(slide_range) before this fix; vetting against that harness is now unblocked but not yet re-run." +3D,3ENERGY,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h;test_ooc_mechanics.py,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A +3D,3ENTROPY,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h;test_ooc_mechanics.py,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A +3D,3EXCESS_KURTOSIS,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_ooc_mechanics.py,test_3d_firstorder_regression.h,pyradiomics/mirp (native 3D first-order; kurtosis -3),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) +3D,3HYPERFLATNESS,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_ooc_mechanics.py,test_3d_firstorder_regression.h,none (scipy.moment order 5/6 only),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) +3D,3HYPERSKEWNESS,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_ooc_mechanics.py,test_3d_firstorder_regression.h,none (scipy.moment order 5/6 only),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) +3D,3INTEGRATED_INTENSITY,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_ooc_mechanics.py,test_3d_firstorder_regression.h,numpy (trivial closed-form; IBSI has only P10/P90),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) +3D,3INTERQUARTILE_RANGE,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h;test_ooc_mechanics.py,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A +3D,3KURTOSIS,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h;test_ooc_mechanics.py,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A +3D,3MAX,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h;test_ooc_mechanics.py,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A +3D,3MEAN,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h;test_ooc_mechanics.py,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A +3D,3MEAN_ABSOLUTE_DEVIATION,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h;test_ooc_mechanics.py,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A +3D,3MEDIAN,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h;test_ooc_mechanics.py,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A +3D,3MEDIAN_ABSOLUTE_DEVIATION,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_ooc_mechanics.py,test_3d_firstorder_regression.h,mirp:stat_medad (IBSI N72L),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) +3D,3MIN,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h;test_ooc_mechanics.py,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A +3D,3MODE,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_ooc_mechanics.py,test_3d_firstorder_regression.h,scipy.stats.mode on raw voxels (IBSI mode is on histogram),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) +3D,3P01,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_ooc_mechanics.py,test_3d_firstorder_regression.h,numpy (trivial closed-form; IBSI has only P10/P90),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d); Nyxus 100-bin CDF-interp reproduced exactly; ~0.2% vs prctile (definitional) +3D,3P10,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h;test_ooc_mechanics.py,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A +3D,3P25,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_ooc_mechanics.py,test_3d_firstorder_regression.h,numpy (trivial closed-form; IBSI has only P10/P90),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d); Nyxus 100-bin CDF-interp reproduced exactly; ~0.2% vs prctile (definitional) +3D,3P75,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_ooc_mechanics.py,test_3d_firstorder_regression.h,numpy (trivial closed-form; IBSI has only P10/P90),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d); Nyxus 100-bin CDF-interp reproduced exactly; ~0.2% vs prctile (definitional) +3D,3P90,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h;test_ooc_mechanics.py,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A +3D,3P99,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_ooc_mechanics.py,test_3d_firstorder_regression.h,numpy (trivial closed-form; IBSI has only P10/P90),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d); Nyxus 100-bin CDF-interp reproduced exactly; ~0.2% vs prctile (definitional) +3D,3QCOD,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_ooc_mechanics.py,test_3d_firstorder_regression.h,pyradiomics/mirp (native 3D first-order; kurtosis -3),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d); Nyxus 100-bin CDF-interp reproduced exactly; ~0.2% vs prctile (definitional) +3D,3RANGE,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h;test_ooc_mechanics.py,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A +3D,3ROBUST_MEAN,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_ooc_mechanics.py,test_3d_firstorder_regression.h,octave/MATLAB (mean of voxels in [P10,P90]),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; rel<1e-9) after fixing 3ROBUST_MEAN which was hardcoded 0 in both 3d_intensity.cpp paths +3D,3ROBUST_MEAN_ABSOLUTE_DEVIATION,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h;test_ooc_mechanics.py,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A +3D,3ROOT_MEAN_SQUARED,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h;test_ooc_mechanics.py,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A +3D,3SKEWNESS,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h;test_ooc_mechanics.py,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A +3D,3STANDARD_DEVIATION,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_ooc_mechanics.py,test_3d_firstorder_regression.h,pyradiomics/mirp (native 3D first-order; kurtosis -3),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) +3D,3STANDARD_DEVIATION_BIASED,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_ooc_mechanics.py,test_3d_firstorder_regression.h,pyradiomics/mirp (native 3D first-order; kurtosis -3),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) +3D,3STANDARD_ERROR,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_ooc_mechanics.py,test_3d_firstorder_regression.h,pyradiomics/mirp (native 3D first-order; kurtosis -3),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) +3D,3VARIANCE,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h;test_ooc_mechanics.py,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A +3D,3VARIANCE_BIASED,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_ooc_mechanics.py,test_3d_firstorder_regression.h,pyradiomics/mirp (native 3D first-order; kurtosis -3),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) +3D,3UNIFORMITY,firstorder,vetted,pyradiomics,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_3d_firstorder_pyradiomics.h;test_ooc_mechanics.py,test_3d_firstorder_pyradiomics.h,,,tracker,test_3d_inten.h ORPHANED (not #included; never run) - left per decision A +3D,3UNIFORMITY_PIU,firstorder,vetted,matlab,agreed,,,test_3d_firstorder_coverage.h;test_3d_inten.h;test_ooc_mechanics.py,test_3d_firstorder_regression.h,pyradiomics/mirp (native 3D first-order; kurtosis -3),,tracker,vetted vs Octave/MATLAB oracle (octave/oracle_3d; raw-formula rel<1e-9) +3D,3AREA,morphology,regression,,needs_audit,,,test_3d_morphology_coverage.h;test_3d_morphology_regression.h;test_ooc_mechanics.py,test_3d_morphology_regression.h,matlab,convention-mismatch,tracker,"convention: Nyxus surface area is voxel-face-based (3AREA 59992) vs MIRP/pyradiomics mesh area (46739, ~28% diff); all area/volume-derived features inherit it. Needs surface-convention decision (see Vetting-Work-Log)." +3D,3AREA_2_VOLUME,morphology,regression,,na,,,test_3d_morphology_coverage.h;test_3d_morphology_regression.h;test_ooc_mechanics.py,test_3d_morphology_regression.h,pyradiomics:SurfaceVolumeRatio(3D),convention-mismatch,tracker,"convention: Nyxus surface area is voxel-face-based (3AREA 59992) vs MIRP/pyradiomics mesh area (46739, ~28% diff); all area/volume-derived features inherit it. Needs surface-convention decision (see Vetting-Work-Log)." +3D,3COMPACTNESS1,morphology,regression,,na,,,test_3d_morphology_coverage.h;test_3d_morphology_regression.h;test_ooc_mechanics.py,test_3d_morphology_regression.h,pyradiomics:Compactness1/Compactness2(3D),convention-mismatch,tracker,"convention: Nyxus surface area is voxel-face-based (3AREA 59992) vs MIRP/pyradiomics mesh area (46739, ~28% diff); all area/volume-derived features inherit it. Needs surface-convention decision (see Vetting-Work-Log)." +3D,3COMPACTNESS2,morphology,regression,,na,,,test_3d_morphology_coverage.h;test_3d_morphology_regression.h;test_ooc_mechanics.py,test_3d_morphology_regression.h,pyradiomics:Compactness1/Compactness2(3D),convention-mismatch,tracker,"convention: Nyxus surface area is voxel-face-based (3AREA 59992) vs MIRP/pyradiomics mesh area (46739, ~28% diff); all area/volume-derived features inherit it. Needs surface-convention decision (see Vetting-Work-Log)." +3D,3MESH_VOLUME,morphology,vetted,matlab,agreed,,,test_3d_morphology_coverage.h;test_3d_morphology_matlab.h;test_ooc_mechanics.py,test_3d_morphology_matlab.h,,,tracker, +3D,3SPHERICAL_DISPROPORTION,morphology,regression,,na,,,test_3d_morphology_coverage.h;test_3d_morphology_regression.h;test_ooc_mechanics.py,test_3d_morphology_regression.h,pyradiomics:SphericalDisproportion(3D shape),convention-mismatch,tracker,"convention: Nyxus surface area is voxel-face-based (3AREA 59992) vs MIRP/pyradiomics mesh area (46739, ~28% diff); all area/volume-derived features inherit it. Needs surface-convention decision (see Vetting-Work-Log)." +3D,3SPHERICITY,morphology,regression,,na,,,test_3d_morphology_coverage.h;test_3d_morphology_regression.h;test_ooc_mechanics.py,test_3d_morphology_regression.h,pyradiomics:Sphericity(3D shape),convention-mismatch,tracker,"convention: Nyxus surface area is voxel-face-based (3AREA 59992) vs MIRP/pyradiomics mesh area (46739, ~28% diff); all area/volume-derived features inherit it. Needs surface-convention decision (see Vetting-Work-Log)." +3D,3VOLUME_CONVEXHULL,morphology,vetted,matlab,agreed,,,test_3d_morphology_coverage.h;test_3d_morphology_regression.h;test_ooc_mechanics.py,test_3d_morphology_matlab.h,,,tracker, +3D,3VOXEL_VOLUME,morphology,vetted,matlab,agreed,,,test_3d_morphology_coverage.h;test_3d_morphology_regression.h;test_ooc_mechanics.py,test_3d_morphology_matlab.h,,,tracker, +3D,3MAJOR_AXIS_LEN,morphology,vetted,mirp,agreed,,,test_3d_morphology_coverage.h;test_ooc_mechanics.py,test_3d_morphology_regression.h,pyradiomics:MajorAxisLength(3D shape),,tracker,"FIXED axis-length mislabel (3d_surface.cpp used wrong eigenvalue indices -> LEAST>MAJOR, FLATNESS>1) + vetted vs MIRP morph_pca_* (exact): MAJOR 104.71, MINOR 88.30, LEAST 71.51, ELONGATION 0.843, FLATNESS 0.683" +3D,3MINOR_AXIS_LEN,morphology,vetted,mirp,agreed,,,test_3d_morphology_coverage.h;test_ooc_mechanics.py,test_3d_morphology_regression.h,pyradiomics:MinorAxisLength(3D shape),,tracker,"FIXED axis-length mislabel (3d_surface.cpp used wrong eigenvalue indices -> LEAST>MAJOR, FLATNESS>1) + vetted vs MIRP morph_pca_* (exact): MAJOR 104.71, MINOR 88.30, LEAST 71.51, ELONGATION 0.843, FLATNESS 0.683" +3D,3LEAST_AXIS_LEN,morphology,vetted,mirp,agreed,,,test_3d_morphology_coverage.h;test_ooc_mechanics.py,test_3d_morphology_regression.h,pyradiomics:LeastAxisLength(3D shape),,tracker,"FIXED axis-length mislabel (3d_surface.cpp used wrong eigenvalue indices -> LEAST>MAJOR, FLATNESS>1) + vetted vs MIRP morph_pca_* (exact): MAJOR 104.71, MINOR 88.30, LEAST 71.51, ELONGATION 0.843, FLATNESS 0.683" +3D,3ELONGATION,morphology,vetted,mirp,agreed,,,test_3d_morphology_coverage.h;test_ooc_mechanics.py,test_3d_morphology_regression.h,pyradiomics:Elongation(3D shape),,tracker,"FIXED axis-length mislabel (3d_surface.cpp used wrong eigenvalue indices -> LEAST>MAJOR, FLATNESS>1) + vetted vs MIRP morph_pca_* (exact): MAJOR 104.71, MINOR 88.30, LEAST 71.51, ELONGATION 0.843, FLATNESS 0.683" +3D,3FLATNESS,morphology,vetted,mirp,agreed,,,test_3d_morphology_coverage.h;test_ooc_mechanics.py,test_3d_morphology_regression.h,pyradiomics:Flatness(3D shape),,tracker,"FIXED axis-length mislabel (3d_surface.cpp used wrong eigenvalue indices -> LEAST>MAJOR, FLATNESS>1) + vetted vs MIRP morph_pca_* (exact): MAJOR 104.71, MINOR 88.30, LEAST 71.51, ELONGATION 0.843, FLATNESS 0.683" +3D,3GLCM_ACOR,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_ASM,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_CLUPROM,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_CLUSHADE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_CLUTEND,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_CONTRAST,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_CORRELATION,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_DIFAVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_DIFENTRO,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_DIFVAR,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_DIS,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_regression.h,"pyradiomics/mirp (DIS=DifferenceAverage, HOM1=Id, HOM2=Idm, SUMVAR=ClusterTendency, ENERGY=JointEnergy=ASM, VAR=mirp cm_var); CONFIG-SENSITIVE: symmetric+13dir vs Nyxus asym/1-offset/100lvl",,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_DIFAVE (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" +3D,3GLCM_ENERGY,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_ooc_mechanics.py,test_3d_glcm_regression.h,"pyradiomics/mirp (DIS=DifferenceAverage, HOM1=Id, HOM2=Idm, SUMVAR=ClusterTendency, ENERGY=JointEnergy=ASM, VAR=mirp cm_var); CONFIG-SENSITIVE: symmetric+13dir vs Nyxus asym/1-offset/100lvl",,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_ASM (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" +3D,3GLCM_ENTROPY,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_ooc_mechanics.py,test_3d_glcm_regression.h,"pyradiomics/mirp (DIS=DifferenceAverage, HOM1=Id, HOM2=Idm, SUMVAR=ClusterTendency, ENERGY=JointEnergy=ASM, VAR=mirp cm_var); CONFIG-SENSITIVE: symmetric+13dir vs Nyxus asym/1-offset/100lvl",,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_JE (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" +3D,3GLCM_HOM1,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_ooc_mechanics.py,test_3d_glcm_regression.h,"pyradiomics/mirp (DIS=DifferenceAverage, HOM1=Id, HOM2=Idm, SUMVAR=ClusterTendency, ENERGY=JointEnergy=ASM, VAR=mirp cm_var); CONFIG-SENSITIVE: symmetric+13dir vs Nyxus asym/1-offset/100lvl",,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_ID (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" +3D,3GLCM_HOM2,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_ooc_mechanics.py,test_3d_glcm_regression.h,"pyradiomics/mirp (DIS=DifferenceAverage, HOM1=Id, HOM2=Idm, SUMVAR=ClusterTendency, ENERGY=JointEnergy=ASM, VAR=mirp cm_var); CONFIG-SENSITIVE: symmetric+13dir vs Nyxus asym/1-offset/100lvl",,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_IDM (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" +3D,3GLCM_ID,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_IDN,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_IDM,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_IDMN,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_INFOMEAS1,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_INFOMEAS2,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_IV,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_JAVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_JE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_JMAX,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_JVAR,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_SUMAVERAGE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_SUMENTROPY,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_3d_glcm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glcm_pyradiomics.h,,,tracker,test_3d_glcm.h ORPHANED (not #included in test_all.cc; 26 test_3glcm_* fns never run) - left as-is per decision A +3D,3GLCM_SUMVARIANCE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_3d_glcm.h;test_ooc_mechanics.py,test_3d_glcm_regression.h,"pyradiomics/mirp (DIS=DifferenceAverage, HOM1=Id, HOM2=Idm, SUMVAR=ClusterTendency, ENERGY=JointEnergy=ASM, VAR=mirp cm_var); CONFIG-SENSITIVE: symmetric+13dir vs Nyxus asym/1-offset/100lvl",,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_CLUTEND (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" +3D,3GLCM_VARIANCE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_ooc_mechanics.py,test_3d_glcm_regression.h,"pyradiomics/mirp (DIS=DifferenceAverage, HOM1=Id, HOM2=Idm, SUMVAR=ClusterTendency, ENERGY=JointEnergy=ASM, VAR=mirp cm_var); CONFIG-SENSITIVE: symmetric+13dir vs Nyxus asym/1-offset/100lvl",,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_JVAR (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" +3D,3GLCM_ASM_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" +3D,3GLCM_ACOR_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" +3D,3GLCM_CLUPROM_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" +3D,3GLCM_CLUSHADE_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" +3D,3GLCM_CLUTEND_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" +3D,3GLCM_CONTRAST_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" +3D,3GLCM_CORRELATION_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" +3D,3GLCM_DIFAVE_AVE,glcm,vetted,mirp,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_mirp.h,,,tracker, +3D,3GLCM_DIFENTRO_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" +3D,3GLCM_DIFVAR_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" +3D,3GLCM_DIS_AVE,glcm,vetted,mirp,agreed,,,test_3d_glcm_coverage.h;test_ooc_mechanics.py,test_3d_glcm_mirp.h,,,tracker, +3D,3GLCM_ENERGY_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_ASM_AVE (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" +3D,3GLCM_ENTROPY_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_JE_AVE (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" +3D,3GLCM_HOM1_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_ID_AVE (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" +3D,3GLCM_ID_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" +3D,3GLCM_IDN_AVE,glcm,vetted,mirp,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_mirp.h,,,tracker, +3D,3GLCM_IDM_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" +3D,3GLCM_IDMN_AVE,glcm,vetted,mirp,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_mirp.h,,,tracker, +3D,3GLCM_IV_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" +3D,3GLCM_JAVE_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" +3D,3GLCM_JE_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" +3D,3GLCM_INFOMEAS1_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" +3D,3GLCM_INFOMEAS2_AVE,glcm,vetted,mirp,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_mirp.h,,,tracker, +3D,3GLCM_VARIANCE_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_JVAR_AVE (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" +3D,3GLCM_JMAX_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" +3D,3GLCM_JVAR_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" +3D,3GLCM_SUMAVERAGE_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" +3D,3GLCM_SUMENTROPY_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: 3GLCM_*_AVE = calc_ave(per-direction base), which is exactly the value test_3d_glcm_pyradiomics.h asserts == pyradiomics for the vetted base feature" +3D,3GLCM_SUMVARIANCE_AVE,glcm,vetted,pyradiomics,agreed,,,test_3d_glcm_coverage.h;test_ooc_mechanics.py,test_3d_glcm_regression.h,mirp,,tracker,"vetted by equivalence: numerically identical to pyradiomics-vetted 3GLCM_CLUTEND_AVE (rel<1e-6, gtest TEST_3DGLCM_EQUIVALENCE_DUMP asserts it). ENTROPY/HOM2 also required the /sum_p fix" +3D,3GLDM_SDE,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLDM_LDE,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLDM_GLN,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLDM_DN,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLDM_DNN,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLDM_GLV,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLDM_DV,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLDM_DE,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLDM_LGLE,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLDM_HGLE,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLDM_SDLGLE,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLDM_SDHGLE,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLDM_LDLGLE,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLDM_LDHGLE,gldm,vetted,pyradiomics,agreed,,,test_3d_gldm_coverage.h;test_3d_gldm.h;test_3d_gldm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_gldm_pyradiomics.h,,,tracker,test_3d_gldm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLDZM_SDE,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h;test_ooc_mechanics.py,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, +3D,3GLDZM_LDE,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h;test_ooc_mechanics.py,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, +3D,3GLDZM_LGLZE,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h;test_ooc_mechanics.py,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, +3D,3GLDZM_HGLZE,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h;test_ooc_mechanics.py,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, +3D,3GLDZM_SDLGLE,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h;test_ooc_mechanics.py,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, +3D,3GLDZM_SDHGLE,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h;test_ooc_mechanics.py,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, +3D,3GLDZM_LDLGLE,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h;test_ooc_mechanics.py,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, +3D,3GLDZM_LDHGLE,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h;test_ooc_mechanics.py,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, +3D,3GLDZM_GLNU,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h;test_ooc_mechanics.py,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, +3D,3GLDZM_GLNUN,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h;test_ooc_mechanics.py,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, +3D,3GLDZM_ZDNU,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h;test_ooc_mechanics.py,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, +3D,3GLDZM_ZDNUN,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h;test_ooc_mechanics.py,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, +3D,3GLDZM_ZP,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h;test_ooc_mechanics.py,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, +3D,3GLDZM_GLM,gldzm,regression,,na,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h;test_ooc_mechanics.py,test_3d_gldzm_regression.h,none (intermediate mean; indirect via GLV/ZDV/DCV variances),no-mainstream-oracle,tracker, +3D,3GLDZM_GLV,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h;test_ooc_mechanics.py,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, +3D,3GLDZM_ZDM,gldzm,regression,,na,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h;test_ooc_mechanics.py,test_3d_gldzm_regression.h,none (intermediate mean; indirect via GLV/ZDV/DCV variances),no-mainstream-oracle,tracker, +3D,3GLDZM_ZDV,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h;test_ooc_mechanics.py,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, +3D,3GLDZM_ZDE,gldzm,regression,,needs_audit,,,test_3d_gldzm_coverage.h;test_3d_gldzm_ibsi.h;test_ooc_mechanics.py,test_3d_gldzm_regression.h,mirp,promote-after-deepdive,tracker, +3D,3NGLDM_LDE,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h;test_ooc_mechanics.py,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_LDE: table=0.1 vs MIRP=0.2559." +3D,3NGLDM_HDE,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h;test_ooc_mechanics.py,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_HDE: table=261 vs MIRP=28.07." +3D,3NGLDM_LGLCE,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h;test_ooc_mechanics.py,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_LGLCE: table=0.00036 vs MIRP=0.0322." +3D,3NGLDM_HGLCE,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h;test_ooc_mechanics.py,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_HGLCE: table=740 vs MIRP=1324." +3D,3NGLDM_LDLGLE,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h;test_ooc_mechanics.py,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_LDLGLE: table=5.8e-05 vs MIRP=0.000685." +3D,3NGLDM_LDHGLE,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h;test_ooc_mechanics.py,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_LDHGLE: table=74 vs MIRP=474.8." +3D,3NGLDM_HDLGLE,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h;test_ooc_mechanics.py,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_HDLGLE: table=0.025 vs MIRP=8.714." +3D,3NGLDM_HDHGLE,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h;test_ooc_mechanics.py,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_HDHGLE: table=20030 vs MIRP=14942.8." +3D,3NGLDM_GLNU,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h;test_ooc_mechanics.py,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_GLNU: table=115443 vs MIRP=4350.3." +3D,3NGLDM_GLNUN,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h;test_ooc_mechanics.py,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_GLNUN: table=0.23 vs MIRP=0.01585." +3D,3NGLDM_DCNU,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h;test_ooc_mechanics.py,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_DCNU: table=115443 vs MIRP=40745.0." +3D,3NGLDM_DCNUN,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h;test_ooc_mechanics.py,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_DCNUN: table=0.23 vs MIRP=0.14847." +3D,3NGLDM_DCP,ngldm,vetted,mirp,agreed,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h;test_ooc_mechanics.py,test_3d_ngldm_mirp.h,,,tracker, +3D,3NGLDM_GLM,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h;test_ooc_mechanics.py,test_3d_ngldm_regression.h,mirp,no-external-oracle,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. Additionally 3NGLDM_GLM has NO counterpart in IBSI/MIRP at all (MIRP's NGLDM emits no grey-level-mean or dependence-count-mean column; the 2D table in test_ngldm_ibsi.h likewise marks GLM '--not in IBSI--'), so no external oracle exists for it." +3D,3NGLDM_GLV,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h;test_ooc_mechanics.py,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_GLV: table=190 vs MIRP=350.17." +3D,3NGLDM_DCM,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h;test_ooc_mechanics.py,test_3d_ngldm_regression.h,mirp,no-external-oracle,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. Additionally 3NGLDM_DCM has NO counterpart in IBSI/MIRP at all (MIRP's NGLDM emits no grey-level-mean or dependence-count-mean column; the 2D table in test_ngldm_ibsi.h likewise marks GLM '--not in IBSI--'), so no external oracle exists for it." +3D,3NGLDM_DCV,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h;test_ooc_mechanics.py,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_DCV: table=86.17 vs MIRP=11.948." +3D,3NGLDM_DCENT,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h;test_ooc_mechanics.py,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_DCENT: table=5.23 vs MIRP=8.676." +3D,3NGLDM_DCENE,ngldm,regression,,needs_audit,,,test_3d_ngldm_coverage.h;test_3d_ngldm_regression.h;test_ooc_mechanics.py,test_3d_ngldm_regression.h,mirp,unproven-reference,tracker,"NOT externally vetted. The reference table `d3ngldm_GT` in test_3d_ngldm_regression.h has NO provenance (no tool/version/config, contrary to SPEC 6.4) and is evaluated on the Nyxus coverage phantom ut_inten.nii -- NOT the IBSI digital phantom -- so IBSI consensus values cannot apply to it. An independent MIRP NGLDM run on the same phantom at the same discretisation (fixed_bin_number n=64, 3D) disagrees on every comparable feature. The passing test is a drift guard, not an oracle; needs a documented, config-matched external oracle (MIRP) before promotion. Harness: morph_oracle/mirp_ngldm_3d.py. For 3NGLDM_DCENE: table=0.14 vs MIRP=0.002875." +3D,3NGTDM_COARSENESS,ngtdm,vetted,pyradiomics,agreed,,,test_3d_ngtdm_coverage.h;test_3d_ngtdm.h;test_3d_ngtdm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_ngtdm_pyradiomics.h,,,tracker,test_3d_ngtdm.h ORPHANED (not #included; never run) - left per decision A +3D,3NGTDM_CONTRAST,ngtdm,vetted,pyradiomics,agreed,,,test_3d_ngtdm_coverage.h;test_3d_ngtdm.h;test_3d_ngtdm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_ngtdm_pyradiomics.h,,,tracker,test_3d_ngtdm.h ORPHANED (not #included; never run) - left per decision A +3D,3NGTDM_BUSYNESS,ngtdm,vetted,pyradiomics,agreed,,,test_3d_ngtdm_coverage.h;test_3d_ngtdm.h;test_3d_ngtdm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_ngtdm_pyradiomics.h,,,tracker,test_3d_ngtdm.h ORPHANED (not #included; never run) - left per decision A +3D,3NGTDM_COMPLEXITY,ngtdm,vetted,pyradiomics,agreed,,,test_3d_ngtdm_coverage.h;test_3d_ngtdm.h;test_3d_ngtdm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_ngtdm_pyradiomics.h,,,tracker,test_3d_ngtdm.h ORPHANED (not #included; never run) - left per decision A +3D,3NGTDM_STRENGTH,ngtdm,vetted,pyradiomics,agreed,,,test_3d_ngtdm_coverage.h;test_3d_ngtdm.h;test_3d_ngtdm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_ngtdm_pyradiomics.h,,,tracker,test_3d_ngtdm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLSZM_SAE,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLSZM_LAE,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLSZM_GLN,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLSZM_GLNN,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLSZM_SZN,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLSZM_SZNN,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLSZM_ZP,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLSZM_GLV,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLSZM_ZV,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLSZM_ZE,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLSZM_LGLZE,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLSZM_HGLZE,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLSZM_SALGLE,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLSZM_SAHGLE,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLSZM_LALGLE,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLSZM_LAHGLE,glszm,vetted,pyradiomics,agreed,,,test_3d_glszm_coverage.h;test_3d_glszm.h;test_3d_glszm_pyradiomics.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glszm_pyradiomics.h,,,tracker,test_3d_glszm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLRLM_SRE,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLRLM_LRE,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLRLM_GLN,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLRLM_GLNN,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLRLM_RLN,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLRLM_RLNN,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLRLM_RP,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLRLM_GLV,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLRLM_RV,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLRLM_RE,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLRLM_LGLRE,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLRLM_HGLRE,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLRLM_SRLGLE,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLRLM_SRHGLE,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLRLM_LRLGLE,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLRLM_LRHGLE,glrlm,vetted,pyradiomics,agreed,,,test_3d_glrlm_coverage.h;test_3d_glrlm.h;test_3d_glrlm_pyradiomics.h;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,test_3d_glrlm.h ORPHANED (not #included; never run) - left per decision A +3D,3GLRLM_SRE_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." +3D,3GLRLM_LRE_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." +3D,3GLRLM_GLN_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." +3D,3GLRLM_GLNN_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." +3D,3GLRLM_RLN_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." +3D,3GLRLM_RLNN_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." +3D,3GLRLM_RP_AVE,glrlm,vetted,mirp,agreed,,,test_3d_glrlm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glrlm_mirp.h,,,tracker, +3D,3GLRLM_GLV_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." +3D,3GLRLM_RV_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." +3D,3GLRLM_RE_AVE,glrlm,vetted,mirp,agreed,,,test_3d_glrlm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glrlm_mirp.h,,,tracker, +3D,3GLRLM_LGLRE_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." +3D,3GLRLM_HGLRE_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." +3D,3GLRLM_SRLGLE_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." +3D,3GLRLM_SRHGLE_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." +3D,3GLRLM_LRLGLE_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." +3D,3GLRLM_LRHGLE_AVE,glrlm,vetted,pyradiomics,agreed,"glrlm.pyradiomics_bincount20 -- ibsi=false, GREYDEPTH=100, GLRLM_GREYDEPTH=-20 (radiomics binCount binning), 3D compat phantom, calc_ave over directions. VETTED ONLY ON THIS RECIPE (the pyradiomics-compat config); other configs are not covered.",,test_3d_glrlm_coverage.h;test_nyxus.py;test_ooc_mechanics.py,test_3d_glrlm_pyradiomics.h,,,tracker,"vetted vs PyRadiomics 3D GLRLM (binCount=20, weightingNorm=None) on the 3D compat phantom (ut_inten.nii label 57) at 10% in test_3d_glrlm_pyradiomics.h (TEST_COMPAT_3GLRLM_AVE_FEATURES). save_value stores fvals[X_AVE][0] = calc_ave(angled_X), the same direction-averaged quantity the base test asserts == PyRadiomics, so the _AVE slot is a direct PyRadiomics assertion. CONFIG SCOPE: vetted only on the pyradiomics-compat recipe above; Nyxus' default binning is not covered by this assertion." IMQ,FOCUS_SCORE,imq,regression,,,,,test_imq_regression.h,test_imq_regression.h,reference IQ library (e.g. skimage/BRISQUE) or analytic,not-in-tracker,audit, IMQ,LOCAL_FOCUS_SCORE,imq,regression,,,,,test_imq_regression.h,test_imq_regression.h,reference IQ library (e.g. skimage/BRISQUE) or analytic,not-in-tracker,audit, IMQ,POWER_SPECTRUM_SLOPE,imq,regression,,,,,test_imq_regression.h,test_imq_regression.h,reference IQ library (e.g. skimage/BRISQUE) or analytic,not-in-tracker,audit,