From c2baa0606f7e786ebbb3b0b415f2753c8c6df526 Mon Sep 17 00:00:00 2001 From: James Le Houx <37665786+jameslehoux@users.noreply.github.com> Date: Tue, 17 Nov 2026 22:11:52 +0000 Subject: [PATCH] feat: add Diamond I13-2 Zn-13X aligner, RRDB/RaGAN & 3D CNN baselines, 3D tortuosity physics, and OpenImpala bridge --- .github/workflows/ci.yml | 18 +- configs/i13-xct/zn13x_2x_1625_to_08125.yaml | 43 ++ configs/i13-xct/zn13x_8x_2600_to_0325.yaml | 43 ++ docs/api.md | 32 +- docs/benchmarks-and-metrics.md | 45 +- docs/datasets-and-configs.md | 49 +- scripts/benchmarks/aggregate_campaign.py | 233 +++++++++ scripts/benchmarks/zn13x_synchrotron_study.py | 469 ++++++++++++++++++ scripts/hpc/isambard_ai_array.slurm | 84 ++++ src/openlsrx/__init__.py | 54 +- src/openlsrx/cli.py | 147 +++++- src/openlsrx/data/__init__.py | 18 + src/openlsrx/data/transforms.py | 4 +- src/openlsrx/data/zn13x.py | 468 +++++++++++++++++ src/openlsrx/inference/orthogonal.py | 13 +- src/openlsrx/io/__init__.py | 8 + src/openlsrx/io/openimpala.py | 84 ++++ src/openlsrx/metrics/__init__.py | 14 +- src/openlsrx/metrics/image.py | 2 + src/openlsrx/metrics/transport.py | 366 ++++++++++++++ src/openlsrx/models/__init__.py | 23 +- src/openlsrx/models/rrdb.py | 192 +++++++ src/openlsrx/models/sr3d.py | 329 ++++++++++++ src/openlsrx/models/srgan.py | 98 +++- .../test_stage2_synchrotron_and_transport.py | 369 ++++++++++++++ 25 files changed, 3110 insertions(+), 95 deletions(-) create mode 100644 configs/i13-xct/zn13x_2x_1625_to_08125.yaml create mode 100644 configs/i13-xct/zn13x_8x_2600_to_0325.yaml create mode 100644 scripts/benchmarks/aggregate_campaign.py create mode 100644 scripts/benchmarks/zn13x_synchrotron_study.py create mode 100644 scripts/hpc/isambard_ai_array.slurm create mode 100644 src/openlsrx/data/zn13x.py create mode 100644 src/openlsrx/io/__init__.py create mode 100644 src/openlsrx/io/openimpala.py create mode 100644 src/openlsrx/metrics/transport.py create mode 100644 src/openlsrx/models/rrdb.py create mode 100644 src/openlsrx/models/sr3d.py create mode 100644 tests/test_stage2_synchrotron_and_transport.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index baea76f..56b24cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,10 @@ on: permissions: contents: read +env: + OMP_NUM_THREADS: "2" + MKL_NUM_THREADS: "2" + concurrency: group: ci-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} @@ -38,12 +42,12 @@ jobs: - name: Ruff lint & format check run: | - uv run --no-sync ruff check src tests scripts - uv run --no-sync ruff format --check src tests scripts + uv run --no-sync ruff check src tests scripts examples + uv run --no-sync ruff format --check src tests scripts examples - name: Static type check (Mypy) run: | - uv run --no-sync mypy src tests scripts + uv run --no-sync mypy src tests scripts examples - name: Build documentation strictly env: @@ -113,14 +117,20 @@ jobs: uv pip install --index-url https://download.pytorch.org/whl/cpu torch torchvision uv pip install -e "." - - name: Run fast synthetic + rock benchmarks and generate LaTeX/figures + - name: Run fast synthetic, rock, and I13-2 Zn-13X benchmarks + campaign aggregation run: | uv run --no-sync python scripts/benchmarks/porespy_study.py --fast --output-dir /tmp/bench/porespy uv run --no-sync python scripts/benchmarks/public_rock_xct.py --fast --output-dir /tmp/bench/rock --data-dir /tmp/bench/data + uv run --no-sync python scripts/benchmarks/zn13x_synchrotron_study.py --fast --output-dir /tmp/bench/zn13x --data-dir /tmp/bench/data uv run --no-sync python scripts/benchmarks/make_paper_outputs.py \ --porespy-csv /tmp/bench/porespy/porespy_study_results.csv \ --rock-csv /tmp/bench/rock/public_rock_results.csv \ --example-volumes /tmp/bench/porespy/example_volumes \ --out-dir /tmp/bench/paper_artifacts + uv run --no-sync python scripts/benchmarks/aggregate_campaign.py \ + --inputs /tmp/bench \ + --output-dir /tmp/bench/campaign_summary test -f /tmp/bench/paper_artifacts/tables/porespy_benchmark.tex test -f /tmp/bench/paper_artifacts/figures/orthogonal_sr_comparison.pdf + test -f /tmp/bench/campaign_summary/campaign_table.tex + test -f /tmp/bench/zn13x/openimpala_ready/zn13x_2x_rrdb_ragan_s42_openimpala.inputs diff --git a/configs/i13-xct/zn13x_2x_1625_to_08125.yaml b/configs/i13-xct/zn13x_2x_1625_to_08125.yaml new file mode 100644 index 0000000..801a76b --- /dev/null +++ b/configs/i13-xct/zn13x_2x_1625_to_08125.yaml @@ -0,0 +1,43 @@ +# Diamond Light Source I13-2 Experimental 2x Pair: 4x (1.625 um) -> 8x (0.8125 um) +# Prepare slabs via: openlsrx download-benchmarks --datasets zn13x_2x +seed_everything: 42 + +trainer: + accelerator: auto + devices: 1 + precision: 16-mixed + max_epochs: 300 + check_val_every_n_epoch: 10 + log_every_n_steps: 20 + +model: + learning_rate: 1.0e-4 + adaptive_size: 8 + accumulate_n_batches: 1 + super_res_factor: 2 + num_blocks: 10 + pretrain_epochs: 20 + use_adversarial_loss: true + use_vgg_loss: true + sample_id: Zn13X_2x_1625_to_08125 + +data: + class_path: openlsrx.data.PairedVolumeDataModule + init_args: + low_path: data/benchmarks/zn13x_2x_lr.tif + high_path: data/benchmarks/zn13x_2x_hr.tif + scale_factor: 2 + norm_mode: none + train_frac: 0.70 + val_frac: 0.15 + guard_slices_lr: 1 + train_axes: ["yz"] + test_axis: perpendicular + predict_axis: xy + patch_size_lr: 32 + augment: true + batch_size: 4 + eval_batch_size: 4 + num_workers: 2 + pin_memory: true + seed: 42 diff --git a/configs/i13-xct/zn13x_8x_2600_to_0325.yaml b/configs/i13-xct/zn13x_8x_2600_to_0325.yaml new file mode 100644 index 0000000..c399719 --- /dev/null +++ b/configs/i13-xct/zn13x_8x_2600_to_0325.yaml @@ -0,0 +1,43 @@ +# Diamond Light Source I13-2 Experimental 8x Pair: 2.5x (2.600 um) -> 20x (0.325 um) +# Prepare slabs via: openlsrx download-benchmarks --datasets zn13x_8x +seed_everything: 42 + +trainer: + accelerator: auto + devices: 1 + precision: 16-mixed + max_epochs: 400 + check_val_every_n_epoch: 10 + log_every_n_steps: 20 + +model: + learning_rate: 1.0e-4 + adaptive_size: 8 + accumulate_n_batches: 2 + super_res_factor: 8 + num_blocks: 12 + pretrain_epochs: 30 + use_adversarial_loss: true + use_vgg_loss: true + sample_id: Zn13X_8x_2600_to_0325 + +data: + class_path: openlsrx.data.PairedVolumeDataModule + init_args: + low_path: data/benchmarks/zn13x_8x_lr.tif + high_path: data/benchmarks/zn13x_8x_hr.tif + scale_factor: 8 + norm_mode: none + train_frac: 0.70 + val_frac: 0.15 + guard_slices_lr: 1 + train_axes: ["yz"] + test_axis: perpendicular + predict_axis: xy + patch_size_lr: 16 + augment: true + batch_size: 2 + eval_batch_size: 2 + num_workers: 2 + pin_memory: true + seed: 42 diff --git a/docs/api.md b/docs/api.md index a3feea7..a2ccc53 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,9 +1,21 @@ # API Reference -## Models +## Models (2D SRGAN, RRDB-ESRGAN & Native 3D CNN) ::: openlsrx.models.srgan.SRGAN +::: openlsrx.models.rrdb.RRDBGenerator + +::: openlsrx.models.rrdb.ArbitraryScaleUpsample + +::: openlsrx.models.rrdb.RRDB + +::: openlsrx.models.sr3d.SR3DResNet + +::: openlsrx.models.sr3d.Generator3D + +::: openlsrx.models.sr3d.super_resolve_volume_3d + ::: openlsrx.models.submodels.Generator ::: openlsrx.models.submodels.Discriminator @@ -36,7 +48,7 @@ --- -## Data Modules & Transforms +## Data Modules, Synchrotron Registration & Transforms ::: openlsrx.data.synthetic.SyntheticPoreSpyDataModule @@ -46,18 +58,32 @@ ::: openlsrx.data.volumes.load_volume_any +::: openlsrx.data.zn13x.prepare_zn13x_benchmark + +::: openlsrx.data.zn13x.register_multires_pair_3d + +::: openlsrx.data.zn13x.extract_aligned_slab_from_h5 + ::: openlsrx.data.transforms.split_volume_spatially ::: openlsrx.data.transforms.degrade_xct_volume --- -## Metrics & Baselines +## Metrics, Transport Physics & OpenImpala I/O ::: openlsrx.metrics.image.evaluate_image_metrics ::: openlsrx.metrics.petrophysics.evaluate_petrophysical_metrics +::: openlsrx.metrics.transport.evaluate_transport_metrics + +::: openlsrx.metrics.transport.compute_directional_tortuosity + +::: openlsrx.metrics.transport.compute_drainage_rmse + +::: openlsrx.io.openimpala.export_for_openimpala + ::: openlsrx.baselines.interpolation.upsample_2d ::: openlsrx.baselines.interpolation.upsample_3d diff --git a/docs/benchmarks-and-metrics.md b/docs/benchmarks-and-metrics.md index 32857fd..b59b8eb 100644 --- a/docs/benchmarks-and-metrics.md +++ b/docs/benchmarks-and-metrics.md @@ -1,6 +1,6 @@ -# Benchmarks & Petrophysical Metrics +# Benchmarks, Petrophysics & Pore-Scale Transport -Standard computer-vision metrics (PSNR and 2D SSIM) can favor overly smooth reconstructions that pinch off narrow pore throats and corrupt permeability predictions. OpenLSR-X pairs image-fidelity metrics with **porous-media petrophysical and topological diagnostics**. +Standard computer-vision metrics (PSNR and 2D SSIM) can favor overly smooth reconstructions that pinch off narrow pore throats and corrupt permeability and diffusivity predictions. OpenLSR-X pairs image-fidelity metrics with **porous-media petrophysical, topological, and 3D pore-scale transport physics diagnostics**. --- @@ -20,9 +20,30 @@ Standard computer-vision metrics (PSNR and 2D SSIM) can favor overly smooth reco - **Two-Point Correlation RMSE ($S_2(r)$):** Discrepancy in FFT-computed radially averaged two-point probability functions. - **Through-Plane Anisotropy Error:** Measures slice-stacking gradient discontinuities along the non-trained axis relative to ground-truth structural anisotropy. +### 3. 3D Pore-Scale Transport Physics (`openlsrx.metrics.transport`) +- **Directional Effective Diffusivity ($D_{\text{eff}}/D_0$) & Tortuosity ($\tau_z, \tau_y, \tau_x$):** Solves steady-state 3D Fickian Laplace diffusion ($\nabla \cdot \nabla c = 0$) across all three Cartesian directions on the 6-connected percolating pore network using a cell-centered finite-volume discretization matching `OpenImpala` and `TauFactor`. +- **Formation Factor ($F = (D_{\text{eff}}/D_0)^{-1}$) & Directional Transport Anisotropy:** Detects artificial slice-stacking bottlenecks where pores percolate in-plane (`yz`) but pinch off along the through-plane stacking axis (`x`). +- **Simulated Capillary Drainage Curve RMSE ($S_{\text{nwp}}(P_c)$):** Simulates inlet-connected morphological mercury intrusion / drainage via `PoreSpy` to quantify throat-entry capillary pressure errors. + +--- + +## Direct `OpenImpala` Simulation Bridge (`openlsrx.io.openimpala`) + +For full-volume HPC transport simulations on `AMReX` grids using [`BASE-Laboratory/OpenImpala`](https://github.com/BASE-Laboratory/OpenImpala), export any super-resolved volume directly from the CLI or Python API: + +```bash +openlsrx export-openimpala \ + --volume outputs/predicted_xy/predicted_stack.tiff \ + --output-dir outputs/openimpala_ready \ + --name zn13x_sr \ + --direction x +``` + +This writes both an uncompressed 8-bit phase-labeled 3D TIFF (`zn13x_sr.tif`) and a ready-to-run AMReX `ParmParse` input deck (`zn13x_sr_openimpala.inputs`). You can also combine transport evaluation and export in a single step via `openlsrx evaluate --include-transport --export-openimpala-dir outputs/openimpala_ready`. + --- -## 3D Orthogonal Fourier Stack Fusion +## 3D Orthogonal Fourier Stack Fusion & Model Backbones Given an isotropic low-resolution 3D scan $V_{\text{LR}} \in \mathbb{R}^{d \times h \times w}$ and a 2D generator $G_\theta$ trained only on slices from a single plane: @@ -31,9 +52,14 @@ Given an isotropic low-resolution 3D scan $V_{\text{LR}} \in \mathbb{R}^{d \time $$W_a(\mathbf{k}) = \frac{\sum_{j \neq a} k_j^2 + \epsilon}{2\,(k_x^2 + k_y^2 + k_z^2) + 3\epsilon}$$ suppressing low-pass interpolation blur and slice-stacking artifacts along the non-super-resolved axis of each individual pass. +OpenLSR-X includes three neural model families for controlled ablation: +- **Classical `SRGAN` / `SRResNet`** (`backbone="srgan"`): Residual blocks with optional pixel/instance/batch normalization. +- **Normalization-Free `RRDBGenerator` + `RaGAN`** (`backbone="rrdb"`, `gan_loss_type="ragan"`): Preserves absolute synchrotron X-ray attenuation contrasts and supports arbitrary fractional beamline magnification ratios (e.g., $2.5\times$ for $0.8125 \to 0.325\ \mu\text{m}$ and $5\times$ for $1.625 \to 0.325\ \mu\text{m}$). +- **Native 3D CNN Baseline (`SR3DResNet`)**: Volumetric 3D residual network with raised-cosine sliding-window inference (`super_resolve_volume_3d`) for benchmarking GPU memory footprint vs. 2D + 3-Axis Fourier Fusion. + --- -## Running the Reproducible Benchmark Suite +## Running the Reproducible Benchmark Suite & HPC Campaign 1. **Factorial PoreSpy Study** (`Low_Noise`/`High_Noise` × `Low_Feature`/`High_Feature`): ```bash @@ -43,8 +69,13 @@ Given an isotropic low-resolution 3D scan $V_{\text{LR}} \in \mathbb{R}^{d \time ```bash python scripts/benchmarks/public_rock_xct.py --output-dir results/public_rock ``` -3. **Generate LaTeX Tables & Figures:** +3. **Diamond Light Source I13-2 Multi-Resolution `Zn-13X` Synchrotron & Transport Study** ($2\times$ & $8\times$): + ```bash + python scripts/benchmarks/zn13x_synchrotron_study.py --output-dir results/zn13x_study + ``` +4. **Isambard-AI (`GH200`) Multi-Seed SLURM Array & Campaign Aggregator:** ```bash - python scripts/benchmarks/make_paper_outputs.py --out-dir results/paper_artifacts + sbatch scripts/hpc/isambard_ai_array.slurm + python scripts/benchmarks/aggregate_campaign.py --inputs results/campaign --output-dir results/campaign_summary ``` - Outputs `booktabs`-formatted `.tex` tables, `.md` summary tables, and 300-DPI `.png`/`.pdf` orthogonal comparison figures with pore-radius distributions. + Generates multi-seed `mean ± std` `booktabs` LaTeX tables (`campaign_table.tex`), Markdown summaries (`campaign_summary.md`), and PSNR-vs-transport Pareto plots (`campaign_tradeoffs.png`). diff --git a/docs/datasets-and-configs.md b/docs/datasets-and-configs.md index c3ffd2f..4799fb7 100644 --- a/docs/datasets-and-configs.md +++ b/docs/datasets-and-configs.md @@ -25,53 +25,41 @@ Both DataModules partition the 3D volume along `split_axis` into **strictly non- | `configs/porespy/High_Noise/*.yaml` | Factorial PoreSpy | High acquisition noise × Low/High feature frequency | | `configs/labxct/bs_config.yaml` | Bentheimer Sandstone | Well-sorted quartz sandstone (~22% intergranular porosity) | | `configs/labxct/il_config.yaml` | Indiana Limestone | Bimodal vuggy + microporous carbonate (~18% total porosity) | -| `configs/i13-xct/i13_xct_config.yaml` | Diamond I13 (`Zn13X`) | Synchrotron TomopyRecon HDF5/NeXus template with ROI crops | +| `configs/i13-xct/i13_xct_config.yaml` | Diamond I13 (`Zn13X`) | Raw Savu TomopyRecon HDF5/NeXus template with ROI crops | +| `configs/i13-xct/zn13x_2x_1625_to_08125.yaml` | Diamond I13 (`Zn13X` $2\times$) | Experimental $4\times$ ($1.625\ \mu\text{m}$) $\to 8\times$ ($0.8125\ \mu\text{m}$) aligned slab | +| `configs/i13-xct/zn13x_8x_2600_to_0325.yaml` | Diamond I13 (`Zn13X` $8\times$) | Experimental $2.5\times$ ($2.600\ \mu\text{m}$) $\to 20\times$ ($0.325\ \mu\text{m}$) aligned slab | --- -## Preparing Open Rock Micro-CT Benchmarks +## Preparing Open Rock & Diamond I13-2 `Zn-13X` Benchmarks -Run the built-in fetcher to prepare `bentheimer_sandstone` and `indiana_limestone` 3D reference volumes with SHA-256 provenance manifests: +Run the built-in fetcher to prepare `bentheimer_sandstone`, `indiana_limestone`, and aligned Diamond Light Source I13-2 `Zn-13X` ($2\times$ & $8\times$) 3D reference volumes with SHA-256 provenance manifests: ```bash +# Prepare laboratory rock benchmarks (Bentheimer & Indiana Limestone) openlsrx download-benchmarks --dest data/benchmarks --size 128 + +# Prepare Diamond Light Source I13-2 Zn-13X synchrotron pairs (from local .h5, Zenodo, or calibrated phantom) +openlsrx download-benchmarks --datasets zn13x_2x zn13x_8x --dest data/benchmarks --size 96 ``` -When running on air-gapped HPC compute nodes without outbound internet access, `download-benchmarks` automatically synthesizes calibrated sandstone and bimodal carbonate digital-rock phantoms and records `"source_kind": "offline_calibrated_phantom"` in `_manifest.json`. +When running on air-gapped HPC compute nodes without outbound internet access (and when `--raw-h5-dir` is not supplied), `download-benchmarks` automatically synthesizes calibrated digital-rock and two-phase `Zn-13X` synchrotron phantoms and records `"source_kind"` in `_manifest.json`. --- -## Using Custom Paired Laboratory or Synchrotron Scans - -For paired low-resolution and high-resolution reconstructions (such as registered micro-CT scans or Diamond Light Source I13 NeXus files), configure `PairedVolumeDataModule` with explicit paths, alignment boxes, and intensity bounds: - -```yaml -data: - class_path: openlsrx.data.PairedVolumeDataModule - init_args: - low_path: /path/to/low_res_recon.nxs - high_path: /path/to/high_res_recon.nxs - h5_dataset_key: 4-TomopyRecon-tomo/data - transpose_axes: [1, 2, 0] - scale_factor: 2 - norm_mode: fixed - low_vmin: -5.23e-4 - low_vmax: 1.11e-3 - high_vmin: -2.50e-4 - high_vmax: 6.79e-4 - low_z_range: [845, 1485] - low_y_range: [1010, 1650] - low_x_range: [858, 1498] - high_z_range: [701, 1981] - high_y_range: [763, 2043] - high_x_range: [466, 1746] -``` +## Vectorized 3D Registration for Multi-Magnification Synchrotron Scans + +Experimental multi-magnification XCT scans suffer from sub-millimetre stage drift and distinct scintillator point-spread functions across objectives. `openlsrx.data.register_multires_pair_3d` and `openlsrx.data.extract_aligned_slab_from_h5` perform: + +1. Chunked ROI slab extraction directly from $>50\ \text{GB}$ Savu `4-TomopyRecon-tomo/data` HDF5 files without loading full cubes into host RAM. +2. Calibrated attenuation normalization using verified I13-2 dynamic bounds. +3. Coarse-grid 3D FFT phase cross-correlation (`skimage.registration.phase_cross_correlation`) followed by local 3D SSIM refinement to produce strictly shape-matched `(LR, HR)` slabs. --- ## Open Synchrotron Multi-Resolution Benchmark (`Zn-13X`, Diamond I13-2) -The `configs/i13-xct/i13_xct_config.yaml` profile corresponds directly to the open multi-resolution synchrotron XCT dataset published in *Scientific Data* ([Green et al., 2025, `10.1038/s41597-025-04605-9`](https://doi.org/10.1038/s41597-025-04605-9)): +The `configs/i13-xct/*.yaml` profiles correspond directly to the open multi-resolution synchrotron XCT dataset published in *Scientific Data* ([Green et al., 2025, `10.1038/s41597-025-04605-9`](https://doi.org/10.1038/s41597-025-04605-9)): | Magnification | Voxel Size ($\mu\text{m}$) | Field of View ($\text{mm}$) | Reconstructed `.h5` / `.nxs` Zenodo DOI | | :--- | :--- | :--- | :--- | @@ -80,4 +68,3 @@ The `configs/i13-xct/i13_xct_config.yaml` profile corresponds directly to the op | `8x` | `0.8125` | $2.1 \times 1.8$ | [`10.5281/zenodo.13327682`](https://doi.org/10.5281/zenodo.13327682) | | `20x` | `0.325` | $0.83 \times 0.70$ | [`10.5281/zenodo.13327692`](https://doi.org/10.5281/zenodo.13327692) | | *Preview* | *8-bit sub-volumes* | *Cropped ROI* | [`10.5281/zenodo.14245751`](https://doi.org/10.5281/zenodo.14245751) | - diff --git a/scripts/benchmarks/aggregate_campaign.py b/scripts/benchmarks/aggregate_campaign.py new file mode 100644 index 0000000..b4aa720 --- /dev/null +++ b/scripts/benchmarks/aggregate_campaign.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Multi-seed campaign aggregator producing Markdown summaries, booktabs LaTeX tables, and Pareto plots. + +Scans benchmark CSV outputs (``porespy_study_results.csv``, ``public_rock_results.csv``, +``zn13x_synchrotron_results.csv``) across one or more result directories, aggregates multi-seed +``mean +/- std`` statistics, and exports publication-ready artifacts. +""" + +from __future__ import annotations + +import argparse +import csv +import json +from collections import defaultdict +from pathlib import Path + +import matplotlib +import numpy as np + +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +CORE_METRICS: list[tuple[str, str, str]] = [ + ("psnr", "PSNR (dB)", "higher"), + ("ssim_3d_triplanar", "3D SSIM", "higher"), + ("pore_dice", "Pore Dice", "higher"), + ("porosity_abs_error", "|dPhi|", "lower"), + ("through_plane_anisotropy_error", "Stack Aniso.", "lower"), + ("d_eff_mae", "D_eff MAE", "lower"), + ("drainage_snwp_rmse", "Drainage RMSE", "lower"), +] + + +def _discover_csvs(input_dirs: list[Path]) -> list[Path]: + found: list[Path] = [] + for root in input_dirs: + if root.is_file() and root.suffix == ".csv": + found.append(root) + elif root.is_dir(): + found.extend(sorted(root.rglob("*_results.csv"))) + return found + + +def _parse_float(val: object) -> float | None: + if val is None: + return None + s = str(val).strip() + if s in ("", "None", "nan", "NaN", "inf", "-inf", "Infinity", "-Infinity"): + return None + try: + out = float(s) + return out if np.isfinite(out) else None + except ValueError: + return None + + +def aggregate_csv_files(csv_paths: list[Path], output_dir: Path) -> dict[str, object]: + output_dir.mkdir(parents=True, exist_ok=True) + if not csv_paths: + raise FileNotFoundError("No *_results.csv files found in provided input paths.") + + # Group by (dataset_or_regime, method) -> metric -> list[float] + grouped: dict[tuple[str, str], dict[str, list[float]]] = defaultdict(lambda: defaultdict(list)) + for p in csv_paths: + with p.open("r", encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + ds_key = str(row.get("dataset") or row.get("regime") or p.stem) + method = str(row.get("method", "unknown")) + for k, v in row.items(): + if k in ("dataset", "regime", "pair", "method", "seed"): + continue + fv = _parse_float(v) + if fv is not None: + grouped[(ds_key, method)][k].append(fv) + + summary_records: list[dict[str, object]] = [] + for (ds_key, method), metric_map in sorted(grouped.items()): + rec: dict[str, object] = { + "dataset": ds_key, + "method": method, + "n_seeds": max((len(vals) for vals in metric_map.values()), default=0), + } + for m_key, vals in sorted(metric_map.items()): + arr = np.asarray(vals, dtype=np.float64) + rec[f"{m_key}_mean"] = float(np.mean(arr)) + rec[f"{m_key}_std"] = float(np.std(arr, ddof=0)) + summary_records.append(rec) + + # 1) JSON summary + json_out = output_dir / "campaign_summary.json" + json_out.write_text(json.dumps(summary_records, indent=2) + "\n", encoding="utf-8") + + # 2) Markdown table + active_metrics = [ + (m, label, direction) + for m, label, direction in CORE_METRICS + if any(f"{m}_mean" in r for r in summary_records) + ] + md_lines = [ + "# OpenLSR-X Multi-Seed Benchmark Campaign Summary", + "", + "| Dataset / Regime | Method | Seeds | " + + " | ".join(label for _, label, _ in active_metrics) + + " |", + "|---|---|---:|" + "|".join("---:" for _ in active_metrics) + "|", + ] + for rec in summary_records: + n_seeds = rec["n_seeds"] + assert isinstance(n_seeds, int) + cells: list[str] = [] + for m, _, _ in active_metrics: + mean_v = rec.get(f"{m}_mean") + std_v = rec.get(f"{m}_std") + if isinstance(mean_v, float) and isinstance(std_v, float): + if n_seeds > 1: + cells.append(f"{mean_v:.4f} ± {std_v:.4f}") + else: + cells.append(f"{mean_v:.4f}") + else: + cells.append("—") + md_lines.append( + f"| `{rec['dataset']}` | `{rec['method']}` | {n_seeds} | " + " | ".join(cells) + " |" + ) + md_out = output_dir / "campaign_summary.md" + md_out.write_text("\n".join(md_lines) + "\n", encoding="utf-8") + + # 3) Booktabs LaTeX table for manuscript inclusion + col_spec = "ll" + ("r" * len(active_metrics)) + tex_lines = [ + "% Auto-generated by scripts/benchmarks/aggregate_campaign.py", + "\\begin{table}[htbp]", + "\\centering", + "\\small", + f"\\begin{{tabular}}{{{col_spec}}}", + "\\toprule", + "Dataset & Method & " + " & ".join(label for _, label, _ in active_metrics) + " \\\\", + "\\midrule", + ] + for rec in summary_records: + n_seeds = rec["n_seeds"] + assert isinstance(n_seeds, int) + ds_tex = str(rec["dataset"]).replace("_", "\\_") + meth_tex = str(rec["method"]).replace("_", "\\_") + tex_cells: list[str] = [] + for m, _, _ in active_metrics: + mean_v = rec.get(f"{m}_mean") + std_v = rec.get(f"{m}_std") + if isinstance(mean_v, float) and isinstance(std_v, float): + if n_seeds > 1: + tex_cells.append(f"${mean_v:.4f} \\pm {std_v:.4f}$") + else: + tex_cells.append(f"${mean_v:.4f}$") + else: + tex_cells.append("--") + tex_lines.append(f"{ds_tex} & {meth_tex} & " + " & ".join(tex_cells) + " \\\\") + tex_lines.extend( + [ + "\\bottomrule", + "\\end{tabular}", + "\\caption{Multi-seed quantitative image, morphological, and transport XCT benchmark summary.}", + "\\label{tab:openlsrx_campaign}", + "\\end{table}", + ] + ) + tex_out = output_dir / "campaign_table.tex" + tex_out.write_text("\n".join(tex_lines) + "\n", encoding="utf-8") + + # 4) Trade-off / Pareto figure (PSNR vs Petrophysical/Transport Error) + fig, axes = plt.subplots(1, 2, figsize=(11, 4.4), dpi=160) + ax0, ax1 = axes + for rec in summary_records: + psnr_m = rec.get("psnr_mean") + aniso_m = rec.get("through_plane_anisotropy_error_mean") + deff_m = rec.get("d_eff_mae_mean") + label = f"{rec['dataset']}:{rec['method']}" + if isinstance(psnr_m, float) and isinstance(aniso_m, float): + ax0.scatter(psnr_m, aniso_m, s=55, alpha=0.85) + ax0.annotate(str(rec["method"]), (psnr_m, aniso_m), fontsize=6.5, alpha=0.85) + if isinstance(psnr_m, float) and isinstance(deff_m, float): + ax1.scatter(psnr_m, deff_m, s=55, alpha=0.85, color="#1f77b4") + ax1.annotate(label.split(":")[-1], (psnr_m, deff_m), fontsize=6.5, alpha=0.85) + + ax0.set_xlabel("3D PSNR (dB, higher is better)") + ax0.set_ylabel("Through-Plane Stacking Anisotropy (lower is better)") + ax0.set_title("Voxel PSNR vs. 3D Isotropy") + ax0.grid(True, alpha=0.25) + + ax1.set_xlabel("3D PSNR (dB, higher is better)") + ax1.set_ylabel("Effective Diffusivity MAE |D_eff - D_eff^HR| (lower is better)") + ax1.set_title("Voxel PSNR vs. Pore-Scale Transport Fidelity") + ax1.grid(True, alpha=0.25) + + fig.tight_layout() + fig_out = output_dir / "campaign_tradeoffs.png" + fig.savefig(fig_out) + plt.close(fig) + + return { + "n_groups": len(summary_records), + "json_path": str(json_out), + "markdown_path": str(md_out), + "latex_path": str(tex_out), + "figure_path": str(fig_out), + } + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Aggregate multi-seed OpenLSR-X benchmark CSVs into Markdown, LaTeX, and plots." + ) + parser.add_argument( + "--inputs", + nargs="+", + default=["results"], + help="Directories or CSV files to aggregate (default: results).", + ) + parser.add_argument( + "--output-dir", + default="results/campaign_summary", + help="Output directory for aggregated tables and figures.", + ) + ns = parser.parse_args() + csvs = _discover_csvs([Path(p) for p in ns.inputs]) + info = aggregate_csv_files(csvs, Path(ns.output_dir)) + print( + f"[aggregate_campaign] Aggregated {info['n_groups']} method groups -> " + f"{info['markdown_path']} & {info['latex_path']}" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/benchmarks/zn13x_synchrotron_study.py b/scripts/benchmarks/zn13x_synchrotron_study.py new file mode 100644 index 0000000..9b23ec9 --- /dev/null +++ b/scripts/benchmarks/zn13x_synchrotron_study.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +"""Diamond Light Source I13-2 Multi-Resolution Zn-13X Synchrotron XCT & Transport Study. + +Benchmarks 3D interpolation, 2D+Fourier SRResNet, memory-matched Native 3D CNN (SR3DResNet), +classical SRGAN (1-axis vs 3-axis Fourier), and normalization-free RRDB-RaGAN (3-axis Fourier) +across image fidelity, 3D morphology/topology, and 3D Fickian diffusive tortuosity + drainage. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import time +from pathlib import Path +from typing import Literal + +import numpy as np +import pytorch_lightning as pl +import torch +from torch import nn + +from openlsrx.baselines import upsample_3d +from openlsrx.data import ( + PairedVolumeDataModule, + prepare_zn13x_benchmark, + synthesize_zn13x_phantom_pair, +) +from openlsrx.inference import super_resolve_volume_orthogonal +from openlsrx.io import export_for_openimpala +from openlsrx.metrics import ( + evaluate_image_metrics, + evaluate_petrophysical_metrics, + evaluate_transport_metrics, +) +from openlsrx.models import ( + SRGAN, + SR3DResNet, + build_3d_subcube_dataloaders, + super_resolve_volume_3d, +) + + +def _count_params(module: nn.Module) -> int: + return sum(p.numel() for p in module.parameters() if p.requires_grad) + + +def _evaluate_method( + dataset_name: str, + pair: str, + seed: int, + method: str, + pred_vol: np.ndarray, + hr_test: np.ndarray, + voxel_size_um: float, + stack_axis: int = 0, + gen_params: int = 0, + train_sec: float = 0.0, + infer_sec: float = 0.0, +) -> dict[str, object]: + row: dict[str, object] = { + "dataset": dataset_name, + "pair": pair, + "seed": seed, + "method": method, + "gen_params": gen_params, + "train_sec": round(train_sec, 4), + "infer_sec": round(infer_sec, 4), + } + row.update(evaluate_image_metrics(pred_vol, hr_test)) + row.update( + evaluate_petrophysical_metrics( + pred_vol, + hr_test, + pore_is_low=True, + voxel_size=voxel_size_um, + stack_axis=stack_axis, + ) + ) + row.update( + evaluate_transport_metrics( + pred_vol, + hr_test, + pore_is_low=True, + voxel_size=voxel_size_um, + drainage_steps=10, + ) + ) + return row + + +def run_zn13x_study( + output_dir: Path, + data_dir: Path = Path("data/benchmarks"), + pairs: list[str] | None = None, + lr_size: int = 48, + epochs: int = 120, + seeds: list[int] | None = None, + raw_h5_dir: Path | None = None, + fast: bool = False, +) -> Path: + output_dir.mkdir(parents=True, exist_ok=True) + pair_list = pairs or (["2x"] if fast else ["2x", "8x"]) + seed_list = seeds or ([42] if fast else [42, 43, 44]) + + if fast: + lr_size = 20 + epochs = 3 + srgan_blocks = 2 + rrdb_blocks = 2 + out_channels = 16 + growth_channels = 8 + disc_features: list[int] | None = [16, 32] + c3d_channels = 12 + c3d_blocks = 2 + c3d_samples = 8 + else: + srgan_blocks = 10 + rrdb_blocks = 8 + out_channels = 64 + growth_channels = 32 + disc_features = [32, 64, 128, 128] + c3d_channels = 28 + c3d_blocks = 4 + c3d_samples = 48 + + rows: list[dict[str, object]] = [] + + for pair in pair_list: + pair_lit: Literal["2x", "8x"] = "2x" if pair == "2x" else "8x" + scale = 2 if pair_lit == "2x" else 8 + cur_lr_size = max(12, lr_size // 2) if (pair_lit == "8x" and fast) else lr_size + lr_tif, hr_tif, manifest = prepare_zn13x_benchmark( + pair=pair_lit, + dest_dir=data_dir, + lr_size=cur_lr_size, + raw_h5_dir=raw_h5_dir, + allow_offline_fallback=True, + seed=42, + ) + dataset_name = f"zn13x_{pair_lit}" + voxel_um = float(manifest["high_resolution_um"]) # type: ignore[arg-type] + + for seed in seed_list: + print( + f"[zn13x_study] Running {dataset_name} ({scale}x, source={manifest['source_kind']}) seed={seed} ..." + ) + dm = PairedVolumeDataModule( + low_path=str(lr_tif), + high_path=str(hr_tif), + scale_factor=scale, + norm_mode="none", + train_frac=0.65, + val_frac=0.20, + guard_slices_lr=1, + patch_size_lr=min(16, cur_lr_size), + batch_size=4 if scale == 2 else 2, + eval_batch_size=4 if scale == 2 else 2, + seed=seed, + ) + dm.setup("fit") + lr_train, hr_train = dm.splits_3d["train"] + lr_val, hr_val = dm.splits_3d["val"] + + if manifest["source_kind"] == "offline_calibrated_zn13x_phantom": + lr_test, hr_test = synthesize_zn13x_phantom_pair( + pair=pair_lit, + lr_shape=(cur_lr_size, cur_lr_size, cur_lr_size), + seed=seed + 1000, + ) + else: + lr_test, hr_test = dm.test_volumes + + # 1) 3D Tricubic baseline + t0 = time.perf_counter() + pred_tricubic = upsample_3d(lr_test, scale_factors=scale, mode="tricubic") + infer_tricubic = time.perf_counter() - t0 + rows.append( + _evaluate_method( + dataset_name, + pair_lit, + seed, + "Tricubic-3D", + pred_tricubic, + hr_test, + voxel_um, + infer_sec=infer_tricubic, + ) + ) + + # 2) SRResNet (2D + 3-Axis Fourier, generator-only ablation) + pl.seed_everything(seed, workers=True) + srresnet = SRGAN( + learning_rate=2e-4 if fast else 1e-4, + adaptive_size=4 if fast else 8, + super_res_factor=scale, + num_blocks=srgan_blocks, + out_channels=out_channels, + disc_features=disc_features, + use_adversarial_loss=False, + use_vgg_loss=False, + output_dir=str(output_dir / "runs" / f"{dataset_name}_s{seed}_srresnet"), + save_val_images=False, + ) + trainer_srres = pl.Trainer( + default_root_dir=str(output_dir / "runs"), + max_epochs=epochs, + accelerator="auto", + devices=1, + logger=False, + enable_checkpointing=False, + enable_progress_bar=False, + enable_model_summary=False, + ) + t0 = time.perf_counter() + trainer_srres.fit(srresnet, datamodule=dm) + train_srres = time.perf_counter() - t0 + + t0 = time.perf_counter() + srres_vols = super_resolve_volume_orthogonal( + srresnet.gen, lr_test, scale_factor=scale, fusion="fourier" + ) + infer_srres = time.perf_counter() - t0 + rows.append( + _evaluate_method( + dataset_name, + pair_lit, + seed, + "SRResNet-3Axis-Fourier", + srres_vols["fused"], + hr_test, + voxel_um, + gen_params=_count_params(srresnet.gen), + train_sec=train_srres, + infer_sec=infer_srres, + ) + ) + + # 3) Native 3D CNN baseline (SR3DResNet) + pl.seed_everything(seed, workers=True) + sr3d = SR3DResNet( + learning_rate=2e-4, + super_res_factor=scale, + out_channels=c3d_channels, + num_blocks=c3d_blocks, + ) + tr_3d_loader, va_3d_loader = build_3d_subcube_dataloaders( + lr_train, + hr_train, + lr_val, + hr_val, + scale_factor=scale, + patch_size_lr=min(12, int(min(lr_train.shape))), + train_samples=c3d_samples, + val_samples=max(2, c3d_samples // 4), + batch_size=2 if scale == 2 else 1, + seed=seed, + ) + trainer_3d = pl.Trainer( + default_root_dir=str(output_dir / "runs"), + max_epochs=epochs, + accelerator="auto", + devices=1, + logger=False, + enable_checkpointing=False, + enable_progress_bar=False, + enable_model_summary=False, + ) + t0 = time.perf_counter() + trainer_3d.fit(sr3d, train_dataloaders=tr_3d_loader, val_dataloaders=va_3d_loader) + train_3d = time.perf_counter() - t0 + + t0 = time.perf_counter() + pred_3d = super_resolve_volume_3d( + sr3d.gen, + lr_test, + scale_factor=scale, + patch_size_lr=min(16, cur_lr_size), + overlap_lr=4, + ) + infer_3d = time.perf_counter() - t0 + rows.append( + _evaluate_method( + dataset_name, + pair_lit, + seed, + "SR3DResNet-Native3D", + pred_3d, + hr_test, + voxel_um, + gen_params=_count_params(sr3d.gen), + train_sec=train_3d, + infer_sec=infer_3d, + ) + ) + + # 4 & 5) Classical OpenLSR-X SRGAN (Single-Axis yz vs 3-Axis Fourier) + pl.seed_everything(seed, workers=True) + srgan = SRGAN( + learning_rate=2e-4 if fast else 1e-4, + adaptive_size=4 if fast else 8, + super_res_factor=scale, + num_blocks=srgan_blocks, + out_channels=out_channels, + disc_features=disc_features, + pretrain_epochs=max(1, epochs // 3), + use_adversarial_loss=True, + use_vgg_loss=False, + output_dir=str(output_dir / "runs" / f"{dataset_name}_s{seed}_srgan"), + save_val_images=False, + ) + trainer_srgan = pl.Trainer( + default_root_dir=str(output_dir / "runs"), + max_epochs=epochs, + accelerator="auto", + devices=1, + logger=False, + enable_checkpointing=False, + enable_progress_bar=False, + enable_model_summary=False, + ) + t0 = time.perf_counter() + trainer_srgan.fit(srgan, datamodule=dm) + train_srgan = time.perf_counter() - t0 + + t0 = time.perf_counter() + srgan_vols = super_resolve_volume_orthogonal( + srgan.gen, lr_test, scale_factor=scale, fusion="fourier" + ) + infer_srgan = time.perf_counter() - t0 + rows.append( + _evaluate_method( + dataset_name, + pair_lit, + seed, + "OpenLSR-X-SingleAxis(yz)", + srgan_vols["yz"], + hr_test, + voxel_um, + stack_axis=0, + gen_params=_count_params(srgan.gen), + train_sec=train_srgan, + infer_sec=infer_srgan / 3.0, + ) + ) + rows.append( + _evaluate_method( + dataset_name, + pair_lit, + seed, + "OpenLSR-X-3Axis-Fourier", + srgan_vols["fused"], + hr_test, + voxel_um, + stack_axis=0, + gen_params=_count_params(srgan.gen), + train_sec=train_srgan, + infer_sec=infer_srgan, + ) + ) + + # 6) Normalization-free RRDB-RaGAN + 3-Axis Fourier Fusion + pl.seed_everything(seed, workers=True) + rrdb_ragan = SRGAN( + learning_rate=2e-4 if fast else 1e-4, + adaptive_size=4 if fast else 8, + super_res_factor=scale, + num_blocks=rrdb_blocks, + out_channels=out_channels, + growth_channels=growth_channels, + backbone="rrdb", + gan_loss_type="ragan", + output_activation="clamp", + pixel_loss_type="l1", + disc_features=disc_features, + pretrain_epochs=max(1, epochs // 3), + use_adversarial_loss=True, + use_vgg_loss=False, + output_dir=str(output_dir / "runs" / f"{dataset_name}_s{seed}_rrdb_ragan"), + save_val_images=False, + ) + trainer_rrdb = pl.Trainer( + default_root_dir=str(output_dir / "runs"), + max_epochs=epochs, + accelerator="auto", + devices=1, + logger=False, + enable_checkpointing=False, + enable_progress_bar=False, + enable_model_summary=False, + ) + t0 = time.perf_counter() + trainer_rrdb.fit(rrdb_ragan, datamodule=dm) + train_rrdb = time.perf_counter() - t0 + + t0 = time.perf_counter() + rrdb_vols = super_resolve_volume_orthogonal( + rrdb_ragan.gen, lr_test, scale_factor=scale, fusion="fourier" + ) + infer_rrdb = time.perf_counter() - t0 + rows.append( + _evaluate_method( + dataset_name, + pair_lit, + seed, + "OpenLSR-X-RRDB-RaGAN-3Axis-Fourier", + rrdb_vols["fused"], + hr_test, + voxel_um, + stack_axis=0, + gen_params=_count_params(rrdb_ragan.gen), + train_sec=train_rrdb, + infer_sec=infer_rrdb, + ) + ) + + if seed == seed_list[0]: + export_for_openimpala( + rrdb_vols["fused"], + output_dir=output_dir / "openimpala_ready", + name=f"{dataset_name}_rrdb_ragan_s{seed}", + pore_is_low=True, + ) + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + csv_path = output_dir / "zn13x_synchrotron_results.csv" + json_path = output_dir / "zn13x_synchrotron_results.json" + with csv_path.open("w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=list(rows[0].keys())) + writer.writeheader() + writer.writerows(rows) + json_path.write_text(json.dumps(rows, indent=2) + "\n", encoding="utf-8") + print(f"[zn13x_study] Saved {len(rows)} benchmark rows to {csv_path}") + return csv_path + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run Diamond I13-2 Zn-13X Synchrotron SR Study.") + parser.add_argument("--output-dir", default="results/zn13x_study", help="Output directory.") + parser.add_argument("--data-dir", default="data/benchmarks", help="Benchmark cache directory.") + parser.add_argument( + "--pairs", + nargs="+", + choices=["2x", "8x"], + default=None, + help="I13-2 magnification pairs to evaluate.", + ) + parser.add_argument("--lr-size", type=int, default=48, help="LR cubic side length.") + parser.add_argument("--epochs", type=int, default=120, help="Training epochs.") + parser.add_argument("--seeds", type=int, nargs="+", default=None, help="Random seeds.") + parser.add_argument("--raw-h5-dir", type=str, default=None, help="Optional raw I13-2 HDF5 dir.") + parser.add_argument("--fast", action="store_true", help="Rapid CPU smoke benchmark.") + ns = parser.parse_args() + + run_zn13x_study( + output_dir=Path(ns.output_dir), + data_dir=Path(ns.data_dir), + pairs=ns.pairs, + lr_size=ns.lr_size, + epochs=ns.epochs, + seeds=ns.seeds, + raw_h5_dir=Path(ns.raw_h5_dir) if ns.raw_h5_dir else None, + fast=ns.fast, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/hpc/isambard_ai_array.slurm b/scripts/hpc/isambard_ai_array.slurm new file mode 100644 index 0000000..571b42a --- /dev/null +++ b/scripts/hpc/isambard_ai_array.slurm @@ -0,0 +1,84 @@ +#!/bin/bash +#SBATCH --job-name=openlsrx-campaign +#SBATCH --partition=workq +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --gpus-per-node=1 +#SBATCH --cpus-per-task=16 +#SBATCH --mem=120G +#SBATCH --time=08:00:00 +#SBATCH --array=0-8 +#SBATCH --output=logs/slurm_%A_%a.out +#SBATCH --error=logs/slurm_%A_%a.err + +# ============================================================================== +# OpenLSR-X Reproducible Multi-Seed HPC Campaign for Isambard-AI (NVIDIA GH200) +# Acknowledgements: UKRI Isambard-AI National AIRR (FIELF001) & Diamond I13-2 +# +# Submits 9 independent GPU jobs (3 benchmark suites x 3 random seeds: 42,43,44): +# Array 0..2 -> Factorial PoreSpy 3D XCT Study (seeds 42, 43, 44) +# Array 3..5 -> Public Micro-CT Rock Suite (Bentheimer & Indiana, seeds 42, 43, 44) +# Array 6..8 -> Diamond I13-2 Multi-Resolution Zn-13X Synchrotron Study (2x & 8x) +# +# Usage: +# mkdir -p logs +# export I13_RAW_H5_DIR=/lus/lfs1aip1/projects/openlsrx/i13_zn13x_h5 # optional +# sbatch scripts/hpc/isambard_ai_array.slurm +# +# After array completion, aggregate into booktabs LaTeX + Pareto figures: +# python scripts/benchmarks/aggregate_campaign.py --inputs results/campaign --output-dir results/campaign_summary +# ============================================================================== + +set -euo pipefail + +mkdir -p logs results/campaign + +# Activate project virtual environment (uv / venv) +if [ -f ".venv/bin/activate" ]; then + # shellcheck disable=SC1091 + source .venv/bin/activate +fi + +export OMP_NUM_THREADS=8 +export PYTORCH_CUDA_ALLOC_CONF="expandable_segments:True" + +SEEDS=(42 43 44) +SUITE_IDX=$(( SLURM_ARRAY_TASK_ID / 3 )) +SEED_IDX=$(( SLURM_ARRAY_TASK_ID % 3 )) +SEED="${SEEDS[$SEED_IDX]}" + +echo "[OpenLSR-X] Task ${SLURM_ARRAY_TASK_ID}: suite=${SUITE_IDX}, seed=${SEED}, host=$(hostname)" +nvidia-smi || true + +if [ "${SUITE_IDX}" -eq 0 ]; then + python scripts/benchmarks/porespy_study.py \ + --output-dir "results/campaign/porespy_s${SEED}" \ + --volume-size 128 \ + --scale-factor 2 \ + --epochs 120 \ + --seeds "${SEED}" \ + --use-vgg +elif [ "${SUITE_IDX}" -eq 1 ]; then + python scripts/benchmarks/public_rock_xct.py \ + --output-dir "results/campaign/public_rock_s${SEED}" \ + --data-dir "data/benchmarks" \ + --volume-size 128 \ + --scale-factor 2 \ + --epochs 150 \ + --seeds "${SEED}" +elif [ "${SUITE_IDX}" -eq 2 ]; then + EXTRA_H5_ARGS=() + if [ -n "${I13_RAW_H5_DIR:-}" ] && [ -d "${I13_RAW_H5_DIR}" ]; then + EXTRA_H5_ARGS=(--raw-h5-dir "${I13_RAW_H5_DIR}") + fi + python scripts/benchmarks/zn13x_synchrotron_study.py \ + --output-dir "results/campaign/zn13x_s${SEED}" \ + --data-dir "data/benchmarks" \ + --pairs 2x 8x \ + --lr-size 64 \ + --epochs 150 \ + --seeds "${SEED}" \ + "${EXTRA_H5_ARGS[@]}" +fi + +echo "[OpenLSR-X] Completed Task ${SLURM_ARRAY_TASK_ID} cleanly." diff --git a/src/openlsrx/__init__.py b/src/openlsrx/__init__.py index 946955e..503e336 100644 --- a/src/openlsrx/__init__.py +++ b/src/openlsrx/__init__.py @@ -2,12 +2,17 @@ from .baselines import upsample_2d, upsample_3d from .data import ( + ZN13X_I13_RESOLUTIONS, DatasetObject, PairedSliceDataset, PairedVolumeDataModule, SyntheticPoreSpyDataModule, + extract_aligned_slab_from_h5, generate_porespy_volume, load_volume_any, + prepare_zn13x_benchmark, + register_multires_pair_3d, + synthesize_zn13x_phantom_pair, ) from .inference import ( fuse_orthogonal_volumes, @@ -15,35 +20,80 @@ super_resolve_volume_orthogonal, super_resolve_volume_single_axis, ) +from .io import export_for_openimpala from .losses import PixelNorm, VGGLoss, custom_sigmoid -from .metrics import evaluate_image_metrics, evaluate_petrophysical_metrics -from .models import SRGAN, ConvBlock, Discriminator, Generator, ResidualBlock, UpsampleBlock +from .metrics import ( + compute_directional_tortuosity, + compute_drainage_curve, + compute_drainage_rmse, + evaluate_image_metrics, + evaluate_petrophysical_metrics, + evaluate_transport_metrics, + solve_steady_state_diffusion_axis, +) +from .models import ( + RRDB, + SRGAN, + ArbitraryScaleUpsample, + ConvBlock, + DenseResidualBlock, + Discriminator, + Generator, + Generator3D, + ResidualBlock, + ResidualBlock3D, + RRDBGenerator, + SR3DResNet, + UpsampleBlock, + build_3d_subcube_dataloaders, + super_resolve_volume_3d, +) __version__ = "0.2.0" __all__ = [ + "ArbitraryScaleUpsample", "ConvBlock", "DatasetObject", + "DenseResidualBlock", "Discriminator", "Generator", + "Generator3D", "PairedSliceDataset", "PairedVolumeDataModule", "PixelNorm", + "RRDB", + "RRDBGenerator", "ResidualBlock", + "ResidualBlock3D", + "SR3DResNet", "SRGAN", "SyntheticPoreSpyDataModule", "UpsampleBlock", "VGGLoss", + "ZN13X_I13_RESOLUTIONS", "__version__", + "build_3d_subcube_dataloaders", + "compute_directional_tortuosity", + "compute_drainage_curve", + "compute_drainage_rmse", "custom_sigmoid", "evaluate_image_metrics", "evaluate_petrophysical_metrics", + "evaluate_transport_metrics", + "export_for_openimpala", + "extract_aligned_slab_from_h5", "fuse_orthogonal_volumes", "generate_porespy_volume", "load_volume_any", + "prepare_zn13x_benchmark", + "register_multires_pair_3d", + "solve_steady_state_diffusion_axis", "super_resolve_slice_stack", + "super_resolve_volume_3d", "super_resolve_volume_orthogonal", "super_resolve_volume_single_axis", + "synthesize_zn13x_phantom_pair", "upsample_2d", "upsample_3d", ] diff --git a/src/openlsrx/cli.py b/src/openlsrx/cli.py index cee21ad..ee0d404 100644 --- a/src/openlsrx/cli.py +++ b/src/openlsrx/cli.py @@ -7,6 +7,7 @@ import sys from collections.abc import Sequence from pathlib import Path +from typing import Any, Literal import pytorch_lightning as pl from pytorch_lightning.cli import LightningCLI @@ -17,15 +18,23 @@ SyntheticPoreSpyDataModule, # noqa: F401 - registered for LightningCLI subclass discovery fetch_or_generate_rock_sample, load_volume_any, + prepare_zn13x_benchmark, +) +from .io import export_for_openimpala +from .metrics import ( + evaluate_image_metrics, + evaluate_petrophysical_metrics, + evaluate_transport_metrics, ) -from .metrics import evaluate_image_metrics, evaluate_petrophysical_metrics from .models import SRGAN +ZN13X_BENCHMARK_KEYS = ("zn13x_2x", "zn13x_8x") + def _run_evaluate(argv: Sequence[str]) -> int: parser = argparse.ArgumentParser( prog="openlsrx evaluate", - description="Compute 2D/3D image-fidelity and porous-media petrophysical metrics on XCT volumes.", + description="Compute 2D/3D image-fidelity, petrophysical, and transport metrics on XCT volumes.", ) parser.add_argument( "--pred", required=True, help="Path to super-resolved TIFF/HDF5/NPY volume." @@ -54,6 +63,17 @@ def _run_evaluate(argv: Sequence[str]) -> int: parser.add_argument( "--stack-axis", type=int, default=0, help="Stacking axis for anisotropy check." ) + parser.add_argument( + "--include-transport", + action="store_true", + help="Also solve 3D Fickian diffusive tortuosity (D_eff/D_0) and capillary drainage curves.", + ) + parser.add_argument( + "--export-openimpala-dir", + type=str, + default=None, + help="Optional directory to export segmented 8-bit TIFF + AMReX .inputs for OpenImpala.", + ) parser.add_argument("--output", type=str, default=None, help="Optional JSON output file path.") ns = parser.parse_args(list(argv)) @@ -64,17 +84,44 @@ def _run_evaluate(argv: Sequence[str]) -> int: f"Shape mismatch between --pred {pred_vol.shape} and --target {target_vol.shape}." ) + pore_is_low = not ns.pore_is_high img_metrics = evaluate_image_metrics(pred_vol, target_vol) petro_metrics = evaluate_petrophysical_metrics( pred_vol, target_vol, threshold_method=ns.threshold, fixed_threshold=ns.fixed_threshold, - pore_is_low=(not ns.pore_is_high), + pore_is_low=pore_is_low, voxel_size=ns.voxel_size, stack_axis=ns.stack_axis, ) - combined = {"image_metrics": img_metrics, "petrophysical_metrics": petro_metrics} + combined: dict[str, Any] = { + "image_metrics": img_metrics, + "petrophysical_metrics": petro_metrics, + } + if ns.include_transport and pred_vol.ndim == 3: + combined["transport_metrics"] = evaluate_transport_metrics( + pred_vol, + target_vol, + threshold_method=ns.threshold, + fixed_threshold=ns.fixed_threshold, + pore_is_low=pore_is_low, + voxel_size=ns.voxel_size, + ) + if ns.export_openimpala_dir and pred_vol.ndim == 3: + tif_p, inp_p = export_for_openimpala( + pred_vol, + output_dir=ns.export_openimpala_dir, + name=Path(ns.pred).stem, + threshold_method="fixed", + fixed_threshold=float(petro_metrics["threshold_true"]), + pore_is_low=pore_is_low, + ) + combined["openimpala_export"] = { + "tiff_path": str(tif_p), + "inputs_path": str(inp_p), + } + formatted = json.dumps(combined, indent=2) print(formatted) if ns.output: @@ -84,16 +131,53 @@ def _run_evaluate(argv: Sequence[str]) -> int: return 0 +def _run_export_openimpala(argv: Sequence[str]) -> int: + parser = argparse.ArgumentParser( + prog="openlsrx export-openimpala", + description="Export a 3D XCT volume as an uncompressed 8-bit TIFF + AMReX .inputs deck for OpenImpala.", + ) + parser.add_argument("--volume", required=True, help="Path to 3D TIFF/HDF5/NPY volume.") + parser.add_argument( + "--output-dir", required=True, help="Output directory for OpenImpala files." + ) + parser.add_argument("--name", default=None, help="Sample base name (defaults to volume stem).") + parser.add_argument( + "--threshold", + choices=["otsu", "yen", "fixed"], + default="otsu", + help="Thresholding algorithm for greyscale inputs (default: otsu).", + ) + parser.add_argument("--fixed-threshold", type=float, default=None) + parser.add_argument("--pore-is-high", action="store_true") + parser.add_argument("--direction", choices=["x", "y", "z"], default="x") + ns = parser.parse_args(list(argv)) + + vol = load_volume_any(ns.volume) + sample_name = ns.name or Path(ns.volume).stem + tif_path, inputs_path = export_for_openimpala( + vol, + output_dir=ns.output_dir, + name=sample_name, + threshold_method=ns.threshold, + fixed_threshold=ns.fixed_threshold, + pore_is_low=(not ns.pore_is_high), + direction=ns.direction, + ) + print(f"[OpenImpala] geometry: {tif_path} | inputs: {inputs_path}") + return 0 + + def _run_download_benchmarks(argv: Sequence[str]) -> int: + all_choices = sorted([*AVAILABLE_PUBLIC_DATASETS.keys(), *ZN13X_BENCHMARK_KEYS]) parser = argparse.ArgumentParser( prog="openlsrx download-benchmarks", - description="Fetch open micro-CT rock benchmark volumes (with offline digital-rock fallback).", + description="Fetch open micro-CT rock & Diamond I13-2 Zn-13X synchrotron benchmark volumes.", ) parser.add_argument( "--datasets", nargs="+", default=sorted(AVAILABLE_PUBLIC_DATASETS.keys()), - choices=sorted(AVAILABLE_PUBLIC_DATASETS.keys()), + choices=all_choices, help="Benchmark datasets to prepare.", ) parser.add_argument( @@ -105,29 +189,52 @@ def _run_download_benchmarks(argv: Sequence[str]) -> int: "--size", type=int, default=128, - help="Cubic HR volume side length (default: 128).", + help="Cubic HR side length for rock benchmarks, or LR side length for zn13x_* if <= 64 (default: 128).", + ) + parser.add_argument( + "--raw-h5-dir", + type=str, + default=None, + help="Optional local directory containing raw Diamond I13-2 16906*_recon.h5 files.", ) parser.add_argument( "--no-offline-fallback", action="store_true", - help="Fail instead of synthesizing calibrated digital-rock phantoms if offline.", + help="Fail instead of synthesizing calibrated digital-rock/synchrotron phantoms if offline.", ) parser.add_argument("--seed", type=int, default=42, help="Deterministic RNG seed.") ns = parser.parse_args(list(argv)) shape = (ns.size, ns.size, ns.size) for name in ns.datasets: - tif_path, manifest = fetch_or_generate_rock_sample( - dataset_name=name, - dest_dir=ns.dest, - shape=shape, - allow_offline_fallback=(not ns.no_offline_fallback), - seed=ns.seed, - ) - print( - f"[{name}] ready at {tif_path} " - f"(source={manifest['source_kind']}, sha256={str(manifest['sha256'])[:12]}...)" - ) + if name in ZN13X_BENCHMARK_KEYS: + pair: Literal["2x", "8x"] = "2x" if name == "zn13x_2x" else "8x" + scale = 2 if pair == "2x" else 8 + lr_size = max(16, ns.size // scale) if ns.size >= 64 else max(16, ns.size) + lr_tif, hr_tif, manifest = prepare_zn13x_benchmark( + pair=pair, + dest_dir=ns.dest, + lr_size=lr_size, + raw_h5_dir=ns.raw_h5_dir, + allow_offline_fallback=(not ns.no_offline_fallback), + seed=ns.seed, + ) + print( + f"[{name}] ready at {lr_tif} & {hr_tif} " + f"(source={manifest['source_kind']}, hr_sha256={str(manifest['hr_sha256'])[:12]}...)" + ) + else: + tif_path, manifest = fetch_or_generate_rock_sample( + dataset_name=name, + dest_dir=ns.dest, + shape=shape, + allow_offline_fallback=(not ns.no_offline_fallback), + seed=ns.seed, + ) + print( + f"[{name}] ready at {tif_path} " + f"(source={manifest['source_kind']}, sha256={str(manifest['sha256'])[:12]}...)" + ) return 0 @@ -140,6 +247,8 @@ def main(args: Sequence[str] | None = None) -> None: raw_args = list(sys.argv[1:] if args is None else args) if raw_args and raw_args[0] == "evaluate": raise SystemExit(_run_evaluate(raw_args[1:])) + if raw_args and raw_args[0] == "export-openimpala": + raise SystemExit(_run_export_openimpala(raw_args[1:])) if raw_args and raw_args[0] == "download-benchmarks": raise SystemExit(_run_download_benchmarks(raw_args[1:])) diff --git a/src/openlsrx/data/__init__.py b/src/openlsrx/data/__init__.py index be9a96b..53d5d27 100644 --- a/src/openlsrx/data/__init__.py +++ b/src/openlsrx/data/__init__.py @@ -21,6 +21,16 @@ split_volume_spatially, ) from .volumes import PairedVolumeDataModule, load_volume_any +from .zn13x import ( + ZN13X_I13_RESOLUTIONS, + ZN13X_SUBVOLUMES_ZENODO_RECORD, + Zn13XPairName, + Zn13XResolutionSpec, + extract_aligned_slab_from_h5, + prepare_zn13x_benchmark, + register_multires_pair_3d, + synthesize_zn13x_phantom_pair, +) __all__ = [ "AVAILABLE_PUBLIC_DATASETS", @@ -33,13 +43,21 @@ "SyntheticPoreSpyDataModule", "SyntheticRepresentation", "VolumeAlignment", + "ZN13X_I13_RESOLUTIONS", + "ZN13X_SUBVOLUMES_ZENODO_RECORD", + "Zn13XPairName", + "Zn13XResolutionSpec", "apply_alignment", "degrade_xct_volume", "downsample_orthogonal_slabs", + "extract_aligned_slab_from_h5", "extract_paired_slices_from_subvolume", "fetch_or_generate_rock_sample", "generate_porespy_volume", "load_volume_any", "normalize_volume", + "prepare_zn13x_benchmark", + "register_multires_pair_3d", "split_volume_spatially", + "synthesize_zn13x_phantom_pair", ] diff --git a/src/openlsrx/data/transforms.py b/src/openlsrx/data/transforms.py index 9bc6087..c3f6638 100644 --- a/src/openlsrx/data/transforms.py +++ b/src/openlsrx/data/transforms.py @@ -12,7 +12,7 @@ from torch import Tensor from torch.utils.data import Dataset -NormMode = Literal["minmax", "percentile", "fixed"] +NormMode = Literal["minmax", "percentile", "fixed", "none"] @dataclass(frozen=True) @@ -36,6 +36,8 @@ def normalize_volume( ) -> np.ndarray: """Vectorized normalization of a 2D/3D XCT array into ``[0, 1]`` float32.""" arr = np.asarray(volume, dtype=np.float32) + if mode == "none": + return np.clip(arr, 0.0, 1.0).astype(np.float32, copy=False) if mode == "minmax": lo = float(arr.min()) hi = float(arr.max()) diff --git a/src/openlsrx/data/zn13x.py b/src/openlsrx/data/zn13x.py new file mode 100644 index 0000000..c6f4b87 --- /dev/null +++ b/src/openlsrx/data/zn13x.py @@ -0,0 +1,468 @@ +"""Diamond Light Source I13-2 multi-resolution Zn-13X synchrotron dataset & 3D aligner. + +Implements automated preparation, Zenodo fetching, offline calibrated phantom synthesis, +and vectorized 3D phase-cross-correlation + SSIM registration for the open multi-resolution +zinc-doped Zeolite 13X dataset published in: + + Green, C., Ahmed, S., Marathe, S. et al., "Three-dimensional, multimodal synchrotron + data for machine learning applications", Scientific Data 12, 329 (2025), + https://doi.org/10.1038/s41597-025-04605-9 +""" + +from __future__ import annotations + +import hashlib +import json +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +import h5py +import numpy as np +import tifffile +from scipy.ndimage import gaussian_filter +from skimage.metrics import structural_similarity +from skimage.registration import phase_cross_correlation + +from .transforms import normalize_volume + +Zn13XPairName = Literal["2x", "8x"] +Zn13XTier = Literal["preview", "full"] + + +@dataclass(frozen=True) +class Zn13XResolutionSpec: + """Metadata and calibrated alignment anchors for one I13-2 magnification.""" + + key: str + magnification: str + voxel_size_um: float + zenodo_record: str + recon_filename: str + vmin: float + vmax: float + anchor_z: int + roi_1040um_yx: tuple[tuple[int, int], tuple[int, int]] | None + roi_812um_yx: tuple[tuple[int, int], tuple[int, int]] + + +ZN13X_SUBVOLUMES_ZENODO_RECORD = "14245751" + +ZN13X_I13_RESOLUTIONS: dict[str, Zn13XResolutionSpec] = { + "2p600um": Zn13XResolutionSpec( + key="2p600um", + magnification="2.5x", + voxel_size_um=2.600, + zenodo_record="12206815", + recon_filename="169068_recon.h5", + vmin=-5.41e-4, + vmax=1.68e-3, + anchor_z=934, + roi_1040um_yx=((1102, 1502), (1006, 1406)), # 400 x 400 px (1040 um) + roi_812um_yx=((1146, 1458), (1050, 1362)), # 312 x 312 px (811.2 um) -> 8x = 2496 px + ), + "1p625um": Zn13XResolutionSpec( + key="1p625um", + magnification="4x", + voxel_size_um=1.625, + zenodo_record="13327651", + recon_filename="169067_recon.h5", + vmin=-5.23e-4, + vmax=1.11e-3, + anchor_z=894, + roi_1040um_yx=((1010, 1650), (858, 1498)), # 640 x 640 px (1040 um) -> 2x = 1280 px + roi_812um_yx=((1080, 1580), (928, 1428)), # 500 x 500 px (812.5 um) -> 5x = 2500 px + ), + "0p8125um": Zn13XResolutionSpec( + key="0p8125um", + magnification="8x", + voxel_size_um=0.8125, + zenodo_record="13327682", + recon_filename="169066_recon.h5", + vmin=-2.50e-4, + vmax=6.79e-4, + anchor_z=804, + roi_1040um_yx=((763, 2043), (466, 1746)), # 1280 x 1280 px (1040 um) + roi_812um_yx=((753, 1753), (753, 1753)), # 1000 x 1000 px (812.5 um) -> 2.5x = 2500 px + ), + "0p325um": Zn13XResolutionSpec( + key="0p325um", + magnification="20x", + voxel_size_um=0.325, + zenodo_record="13327692", + recon_filename="169065_recon.h5", + vmin=-2.50e-4, + vmax=6.79e-4, + anchor_z=509, + roi_1040um_yx=None, + roi_812um_yx=((0, 2496), (0, 2496)), # 2496 x 2496 px (811.2 um) + ), +} + + +def _sha256_file(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(65536), b""): + h.update(chunk) + return h.hexdigest() + + +def _block_downsample_3d(vol: np.ndarray, factor: int) -> np.ndarray: + """Exact block-average 3D downsampling along all 3 axes by integer ``factor``.""" + d, h, w = vol.shape + cd = (d // factor) * factor + ch = (h // factor) * factor + cw = (w // factor) * factor + cropped = vol[:cd, :ch, :cw] + return cropped.reshape( + cd // factor, + factor, + ch // factor, + factor, + cw // factor, + factor, + ).mean(axis=(1, 3, 5), dtype=np.float32) + + +def register_multires_pair_3d( + low_vol: np.ndarray, + high_vol: np.ndarray, + scale_factor: int = 2, + local_refine_radius: int = 1, +) -> tuple[np.ndarray, np.ndarray, dict[str, object]]: + """Vectorized 3D registration of paired multi-magnification XCT volumes. + + Replaces brute-force 3D slice scanning with FFT phase cross-correlation at coarse + resolution followed by local 3D SSIM refinement, returning strictly shape-matched + ``(aligned_lr, aligned_hr, metadata)`` with ``aligned_hr.shape == s * aligned_lr.shape``. + """ + if scale_factor < 1: + raise ValueError(f"scale_factor must be >= 1, got {scale_factor}.") + + lr = np.asarray(low_vol, dtype=np.float32) + hr = np.asarray(high_vol, dtype=np.float32) + if lr.ndim != 3 or hr.ndim != 3: + raise ValueError(f"Expected 3D volumes, got {lr.shape} and {hr.shape}.") + + hr_coarse = _block_downsample_3d(hr, scale_factor) + common_lr_shape = tuple(min(a, b) for a, b in zip(lr.shape, hr_coarse.shape, strict=True)) + lr_ref = lr[: common_lr_shape[0], : common_lr_shape[1], : common_lr_shape[2]] + hr_ref = hr_coarse[: common_lr_shape[0], : common_lr_shape[1], : common_lr_shape[2]] + + shift_est, _, _ = phase_cross_correlation(lr_ref, hr_ref, upsample_factor=1) + base_shift = tuple(int(round(float(s))) for s in shift_est) + + def _extract_overlap( + shift_lr: tuple[int, int, int], + ) -> tuple[np.ndarray, np.ndarray, np.ndarray] | None: + dz, dy, dx = shift_lr + # Map coarse HR coordinate c -> LR coordinate c + shift_lr + z0_hr = max(0, -dz) + y0_hr = max(0, -dy) + x0_hr = max(0, -dx) + + z1_hr = min(hr_coarse.shape[0], lr.shape[0] - dz) + y1_hr = min(hr_coarse.shape[1], lr.shape[1] - dy) + x1_hr = min(hr_coarse.shape[2], lr.shape[2] - dx) + + if (z1_hr - z0_hr) < 8 or (y1_hr - y0_hr) < 8 or (x1_hr - x0_hr) < 8: + return None + + lr_sub = lr[z0_hr + dz : z1_hr + dz, y0_hr + dy : y1_hr + dy, x0_hr + dx : x1_hr + dx] + hr_c_sub = hr_coarse[z0_hr:z1_hr, y0_hr:y1_hr, x0_hr:x1_hr] + s = scale_factor + hr_sub = hr[z0_hr * s : z1_hr * s, y0_hr * s : y1_hr * s, x0_hr * s : x1_hr * s] + return lr_sub, hr_c_sub, hr_sub + + best_score = -2.0 + best_shift = (0, 0, 0) + best_pair: tuple[np.ndarray, np.ndarray] | None = None + + candidates = [ + (base_shift[0] + dz, base_shift[1] + dy, base_shift[2] + dx) + for dz in range(-local_refine_radius, local_refine_radius + 1) + for dy in range(-local_refine_radius, local_refine_radius + 1) + for dx in range(-local_refine_radius, local_refine_radius + 1) + ] + if (0, 0, 0) not in candidates: + candidates.append((0, 0, 0)) + + for cand in candidates: + extracted = _extract_overlap(cand) + if extracted is None: + continue + lr_sub, hr_c_sub, hr_sub = extracted + mid = lr_sub.shape[0] // 2 + win = min(7, min(lr_sub.shape[1], lr_sub.shape[2])) + if win % 2 == 0: + win -= 1 + score = ( + float( + structural_similarity( + lr_sub[mid], + hr_c_sub[mid], + data_range=1.0, + win_size=max(3, win), + ) + ) + if win >= 3 + else -float(np.mean((lr_sub - hr_c_sub) ** 2)) + ) + if score > best_score: + best_score = score + best_shift = cand + best_pair = (lr_sub.copy(), hr_sub.copy()) + + if best_pair is None: + d, h, w = common_lr_shape + s = scale_factor + best_pair = (lr[:d, :h, :w].copy(), hr[: d * s, : h * s, : w * s].copy()) + best_shift = (0, 0, 0) + + info: dict[str, object] = { + "coarse_shift_zyx": list(best_shift), + "coarse_mid_slice_ssim": best_score, + "aligned_lr_shape": list(best_pair[0].shape), + "aligned_hr_shape": list(best_pair[1].shape), + "scale_factor": scale_factor, + } + return best_pair[0], best_pair[1], info + + +def synthesize_zn13x_phantom_pair( + pair: Zn13XPairName = "2x", + lr_shape: tuple[int, int, int] = (48, 48, 48), + seed: int = 42, +) -> tuple[np.ndarray, np.ndarray]: + """Synthesize a calibrated two-phase Zn-doped Zeolite 13X synchrotron XCT pair. + + Models the physical structure described in Green et al. (*Sci. Data* 2025): + - Cleaved Zeolite 13X fragment inside a thin-walled capillary FOV + - Higher-attenuation outer Zn-exchanged rim (~0.88) and Na-13X core (~0.68) + - Hierarchical connected macro- and meso-pore network (~0.14 attenuation) + - Distinct objective-specific scintillator PSF and detector noise at LR vs HR + """ + scale = 2 if pair == "2x" else 8 + hr_shape = (lr_shape[0] * scale, lr_shape[1] * scale, lr_shape[2] * scale) + rng = np.random.default_rng(seed) + + # Coordinate grid in [-1, 1]^3 for cleaved pellet fragment + Zn diffusion rim + zz, yy, xx = np.meshgrid( + np.linspace(-1.0, 1.0, hr_shape[0], dtype=np.float32), + np.linspace(-1.0, 1.0, hr_shape[1], dtype=np.float32), + np.linspace(-1.0, 1.0, hr_shape[2], dtype=np.float32), + indexing="ij", + ) + radial = np.sqrt(0.85 * zz**2 + yy**2 + 1.1 * xx**2) + + # Hierarchical macropores + mesopores generated via multi-scale Gaussian random fields + coarse_field = gaussian_filter( + rng.standard_normal(hr_shape).astype(np.float32), sigma=max(2.0, scale * 1.4) + ) + fine_field = gaussian_filter( + rng.standard_normal(hr_shape).astype(np.float32), sigma=max(0.9, scale * 0.45) + ) + macro_pores = coarse_field > np.quantile(coarse_field, 0.78) + meso_pores = fine_field > np.quantile(fine_field, 0.84) + pore_mask = np.logical_or(macro_pores, meso_pores) + + # Zn2+ ion-exchange rim (outer shell of pellet has higher X-ray attenuation at 27 keV) + zn_rim = radial > 0.52 + hr_clean = np.where(zn_rim, 0.88, 0.68).astype(np.float32) + hr_clean[pore_mask] = 0.14 + + # High-magnification GGG:Eu / LuAG scintillator PSF + mild FBP noise + hr_vol = gaussian_filter(hr_clean, sigma=0.55, mode="reflect") + hr_vol = hr_vol + rng.normal(0.0, 0.015, size=hr_shape).astype(np.float32) + hr_vol = np.clip(hr_vol, 0.0, 1.0).astype(np.float32, copy=False) + + # Lower-magnification thick LuAG (500 um) scintillator blur + lens browning noise + lr_blurred = gaussian_filter(hr_clean, sigma=0.85 * scale, mode="reflect") + lr_binned = _block_downsample_3d(lr_blurred, scale) + noise_sigma = 0.024 if pair == "2x" else 0.035 # 2.5x lens browning increases photon noise + lr_vol = lr_binned + rng.normal(0.0, noise_sigma, size=lr_binned.shape).astype(np.float32) + lr_vol = np.clip(lr_vol, 0.0, 1.0).astype(np.float32, copy=False) + return lr_vol, hr_vol + + +def extract_aligned_slab_from_h5( + low_h5_path: str | Path, + high_h5_path: str | Path, + pair: Zn13XPairName = "2x", + lr_slab_depth: int = 64, + lr_crop_yx: int | None = None, + dataset_key: str = "4-TomopyRecon-tomo/data", +) -> tuple[np.ndarray, np.ndarray, dict[str, object]]: + """Extract and normalize an aligned 3D slab from raw >50 GB I13-2 HDF5 files without loading full cubes.""" + if pair == "2x": + lo_spec = ZN13X_I13_RESOLUTIONS["1p625um"] + hi_spec = ZN13X_I13_RESOLUTIONS["0p8125um"] + scale = 2 + lo_yx = lo_spec.roi_1040um_yx + hi_yx = hi_spec.roi_1040um_yx + else: + lo_spec = ZN13X_I13_RESOLUTIONS["2p600um"] + hi_spec = ZN13X_I13_RESOLUTIONS["0p325um"] + scale = 8 + lo_yx = lo_spec.roi_812um_yx + hi_yx = hi_spec.roi_812um_yx + + assert lo_yx is not None and hi_yx is not None + half_d = lr_slab_depth // 2 + lo_z0 = max(0, lo_spec.anchor_z - half_d) + lo_z1 = lo_z0 + lr_slab_depth + hi_z0 = max(0, hi_spec.anchor_z - half_d * scale) + hi_z1 = hi_z0 + lr_slab_depth * scale + + lo_y0, lo_y1 = lo_yx[0] + lo_x0, lo_x1 = lo_yx[1] + hi_y0, hi_y1 = hi_yx[0] + hi_x0, hi_x1 = hi_yx[1] + + if lr_crop_yx is not None: + lo_y1 = min(lo_y1, lo_y0 + lr_crop_yx) + lo_x1 = min(lo_x1, lo_x0 + lr_crop_yx) + hi_y1 = min(hi_y1, hi_y0 + lr_crop_yx * scale) + hi_x1 = min(hi_x1, hi_x0 + lr_crop_yx * scale) + + # Note: Raw Savu TomopyRecon HDF5 stores axes (y_orig, z_orig, x_orig) -> transposed (1, 2, 0) in Calum's notebook + with h5py.File(low_h5_path, "r") as f_lo: + dset_lo = f_lo[dataset_key] + raw_lo = np.asarray(dset_lo[lo_x0:lo_x1, lo_z0:lo_z1, lo_y0:lo_y1], dtype=np.float32) + slab_lo = np.transpose(raw_lo, (1, 2, 0)) + + with h5py.File(high_h5_path, "r") as f_hi: + dset_hi = f_hi[dataset_key] + raw_hi = np.asarray(dset_hi[hi_x0:hi_x1, hi_z0:hi_z1, hi_y0:hi_y1], dtype=np.float32) + slab_hi = np.transpose(raw_hi, (1, 2, 0)) + + norm_lo = normalize_volume(slab_lo, mode="fixed", vmin=lo_spec.vmin, vmax=lo_spec.vmax) + norm_hi = normalize_volume(slab_hi, mode="fixed", vmin=hi_spec.vmin, vmax=hi_spec.vmax) + return register_multires_pair_3d(norm_lo, norm_hi, scale_factor=scale) + + +def prepare_zn13x_benchmark( + pair: Zn13XPairName = "2x", + dest_dir: str | Path = "data/benchmarks", + tier: Zn13XTier = "preview", + lr_size: int = 48, + raw_h5_dir: str | Path | None = None, + allow_offline_fallback: bool = True, + seed: int = 42, +) -> tuple[Path, Path, dict[str, object]]: + """Prepare aligned LR & HR 3D TIFF volumes + provenance manifest for Diamond I13-2 ``Zn-13X``. + + Resolution priority: + 1. Cached ``zn13x__lr.tif`` + ``zn13x__hr.tif`` + manifest if already present. + 2. Local synchrotron HDF5 reconstructions in ``raw_h5_dir`` (e.g., on Diamond/Isambard Lustre). + 3. Public Zenodo 8-bit preview sub-volumes (``10.5281/zenodo.14245751``) when online. + 4. Calibrated two-phase ``Zn-13X`` synchrotron phantom when offline or air-gapped. + """ + out_dir = Path(dest_dir) + out_dir.mkdir(parents=True, exist_ok=True) + name = f"zn13x_{pair}" + lr_tif = out_dir / f"{name}_lr.tif" + hr_tif = out_dir / f"{name}_hr.tif" + manifest_path = out_dir / f"{name}_manifest.json" + + if lr_tif.is_file() and hr_tif.is_file() and manifest_path.is_file(): + cached: dict[str, object] = json.loads(manifest_path.read_text(encoding="utf-8")) + return lr_tif, hr_tif, cached + + scale = 2 if pair == "2x" else 8 + lo_key, hi_key = ("1p625um", "0p8125um") if pair == "2x" else ("2p600um", "0p325um") + lo_spec = ZN13X_I13_RESOLUTIONS[lo_key] + hi_spec = ZN13X_I13_RESOLUTIONS[hi_key] + + lr_vol: np.ndarray | None = None + hr_vol: np.ndarray | None = None + reg_info: dict[str, object] = {} + source_kind = "offline_calibrated_zn13x_phantom" + + # 1. Check local HDF5 directory if provided + if raw_h5_dir is not None: + h5_root = Path(raw_h5_dir) + lo_h5 = h5_root / lo_spec.recon_filename + hi_h5 = h5_root / hi_spec.recon_filename + if lo_h5.is_file() and hi_h5.is_file(): + lr_vol, hr_vol, reg_info = extract_aligned_slab_from_h5( + lo_h5, + hi_h5, + pair=pair, + lr_slab_depth=lr_size, + lr_crop_yx=lr_size, + ) + source_kind = "local_i13_hdf5_slab" + + # 2. Try Zenodo 8-bit preview record API if online and tier == 'preview' + if lr_vol is None and tier == "preview": + api_url = f"https://zenodo.org/api/records/{ZN13X_SUBVOLUMES_ZENODO_RECORD}" + try: + with urllib.request.urlopen(api_url, timeout=8) as resp: # noqa: S310 + record_meta = json.loads(resp.read().decode("utf-8")) + files = {f["key"]: f["links"]["self"] for f in record_meta.get("files", [])} + # Match TIF/H5 files mentioning pixel sizes if present in record + lo_candidates = [ + u for k, u in files.items() if lo_spec.magnification in k or lo_key in k + ] + hi_candidates = [ + u for k, u in files.items() if hi_spec.magnification in k or hi_key in k + ] + if lo_candidates and hi_candidates: + tmp_lo = out_dir / f".{name}_lo.tmp" + tmp_hi = out_dir / f".{name}_hi.tmp" + try: + urllib.request.urlretrieve(lo_candidates[0], tmp_lo) # noqa: S310 + urllib.request.urlretrieve(hi_candidates[0], tmp_hi) # noqa: S310 + raw_lo_arr = normalize_volume(tifffile.imread(tmp_lo), mode="minmax") + raw_hi_arr = normalize_volume(tifffile.imread(tmp_hi), mode="minmax") + lr_vol, hr_vol, reg_info = register_multires_pair_3d( + raw_lo_arr[:lr_size, :lr_size, :lr_size], + raw_hi_arr[: lr_size * scale, : lr_size * scale, : lr_size * scale], + scale_factor=scale, + ) + source_kind = "zenodo_14245751_preview" + finally: + tmp_lo.unlink(missing_ok=True) + tmp_hi.unlink(missing_ok=True) + except Exception as exc: + if not allow_offline_fallback: + raise RuntimeError( + f"Unable to download Zn-13X preview from Zenodo ({api_url}) and fallback is disabled." + ) from exc + + # 3. Offline calibrated Zn-13X two-phase synchrotron phantom + if lr_vol is None or hr_vol is None: + if not allow_offline_fallback: + raise RuntimeError(f"No local HDF5 or online Zenodo files available for {name}.") + lr_vol, hr_vol = synthesize_zn13x_phantom_pair( + pair=pair, + lr_shape=(lr_size, lr_size, lr_size), + seed=seed, + ) + source_kind = "offline_calibrated_zn13x_phantom" + + tifffile.imwrite(lr_tif, lr_vol.astype(np.float32), photometric="minisblack") + tifffile.imwrite(hr_tif, hr_vol.astype(np.float32), photometric="minisblack") + + manifest: dict[str, object] = { + "name": name, + "pair": pair, + "scale_factor": scale, + "low_resolution_um": lo_spec.voxel_size_um, + "high_resolution_um": hi_spec.voxel_size_um, + "low_zenodo_doi": f"10.5281/zenodo.{lo_spec.zenodo_record}", + "high_zenodo_doi": f"10.5281/zenodo.{hi_spec.zenodo_record}", + "paper_doi": "10.1038/s41597-025-04605-9", + "source_kind": source_kind, + "lr_shape": list(lr_vol.shape), + "hr_shape": list(hr_vol.shape), + "lr_sha256": _sha256_file(lr_tif), + "hr_sha256": _sha256_file(hr_tif), + "registration": reg_info, + "seed": seed, + } + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + return lr_tif, hr_tif, manifest diff --git a/src/openlsrx/inference/orthogonal.py b/src/openlsrx/inference/orthogonal.py index 52768a5..c888836 100644 --- a/src/openlsrx/inference/orthogonal.py +++ b/src/openlsrx/inference/orthogonal.py @@ -68,7 +68,7 @@ def super_resolve_slice_stack( def super_resolve_volume_single_axis( generator: nn.Module, low_volume: np.ndarray, - scale_factor: int = 2, + scale_factor: int | float = 2, axis: AxisName = "yz", interpolate_stack_axis: bool = True, batch_size: int = 8, @@ -83,7 +83,7 @@ def super_resolve_volume_single_axis( by ``scale_factor`` before applying the 2D generator to each orthogonal plane. Set ``interpolate_stack_axis=False`` when ``low_volume`` is already high-resolution along ``axis`` (anisotropic slab input). - scale_factor: Spatial upsampling factor of ``generator``. + scale_factor: Spatial upsampling factor of ``generator`` (integer or fractional). axis: Orthogonal plane to super-resolve (``'yz'``, ``'xz'``, or ``'xy'``). interpolate_stack_axis: Whether to upsample the slicing axis by ``scale_factor``. batch_size: Number of 2D slices per forward pass. @@ -94,10 +94,13 @@ def super_resolve_volume_single_axis( raise ValueError(f"Expected 3D volume, got shape {vol.shape}.") ordered = _move_slice_axis_first(vol, axis) # (N_axis, h, w) - if interpolate_stack_axis and scale_factor > 1: + sf = float(scale_factor) + if interpolate_stack_axis and sf > 1.0: + target_n = int(round(ordered.shape[0] * sf)) + zoom_n = target_n / float(ordered.shape[0]) ordered = zoom( ordered, - zoom=(scale_factor, 1, 1), + zoom=(zoom_n, 1, 1), order=3, mode="reflect", ).astype(np.float32, copy=False) @@ -178,7 +181,7 @@ def fuse_orthogonal_volumes( def super_resolve_volume_orthogonal( generator: nn.Module, low_volume: np.ndarray, - scale_factor: int = 2, + scale_factor: int | float = 2, axes: Sequence[AxisName] = ("yz", "xz", "xy"), fusion: FusionMode = "fourier", batch_size: int = 8, diff --git a/src/openlsrx/io/__init__.py b/src/openlsrx/io/__init__.py new file mode 100644 index 0000000..fd21386 --- /dev/null +++ b/src/openlsrx/io/__init__.py @@ -0,0 +1,8 @@ +"""Simulation and external solver I/O bridges for OpenLSR-X.""" + +from .openimpala import ImpalaDirection, export_for_openimpala + +__all__ = [ + "ImpalaDirection", + "export_for_openimpala", +] diff --git a/src/openlsrx/io/openimpala.py b/src/openlsrx/io/openimpala.py new file mode 100644 index 0000000..ca6a223 --- /dev/null +++ b/src/openlsrx/io/openimpala.py @@ -0,0 +1,84 @@ +"""Direct geometry & AMReX ParmParse input exporter for BASE-Laboratory/OpenImpala.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Literal + +import numpy as np +import tifffile + +from ..metrics.petrophysics import ThresholdMethod, segment_pore_space + +ImpalaDirection = Literal["x", "y", "z"] + + +def export_for_openimpala( + volume_or_mask: np.ndarray, + output_dir: str | Path, + name: str = "sample", + threshold_method: ThresholdMethod = "otsu", + fixed_threshold: float | None = None, + pore_is_low: bool = True, + pore_label: int = 1, + solid_label: int = 0, + direction: ImpalaDirection = "x", + max_grid_size: int = 64, +) -> tuple[Path, Path]: + """Export a 3D XCT volume or binary mask for direct simulation in ``BASE-Laboratory/OpenImpala``. + + Writes: + 1. ``/.tif``: Uncompressed 8-bit multi-page TIFF (``uint8``, shape ``(D, H, W)``) + with ``pore_label`` marking the active diffusive phase and ``solid_label`` marking solid. + 2. ``/_openimpala.inputs``: Ready-to-run AMReX ``ParmParse`` input deck for + ``OpenImpala``'s ``Tortuosity`` / ``Diffusion`` executable. + """ + arr = np.asarray(volume_or_mask) + if arr.ndim != 3: + raise ValueError(f"OpenImpala requires a 3D volume (D, H, W), got shape {arr.shape}.") + if direction not in ("x", "y", "z"): + raise ValueError(f"direction must be 'x', 'y', or 'z', got {direction!r}.") + if not (0 <= pore_label <= 255 and 0 <= solid_label <= 255 and pore_label != solid_label): + raise ValueError( + f"pore_label ({pore_label}) and solid_label ({solid_label}) must be distinct uint8 integers." + ) + + if arr.dtype == bool: + pore_mask = arr + else: + pore_mask, _ = segment_pore_space( + arr.astype(np.float32, copy=False), + method=threshold_method, + fixed_value=fixed_threshold, + pore_is_low=pore_is_low, + ) + + labeled = np.where(pore_mask, np.uint8(pore_label), np.uint8(solid_label)) + out_root = Path(output_dir) + out_root.mkdir(parents=True, exist_ok=True) + + tiff_path = out_root / f"{name}.tif" + inputs_path = out_root / f"{name}_openimpala.inputs" + + tifffile.imwrite( + tiff_path, + labeled, + photometric="minisblack", + compression=None, + ) + + depth_z, height_y, width_x = labeled.shape + porosity = float(pore_mask.mean()) + inputs_text = ( + "# Generated by OpenLSR-X for BASE-Laboratory/OpenImpala (AMReX Tortuosity)\n" + f"# Sample: {name} | Shape (Z, Y, X) = ({depth_z}, {height_y}, {width_x}) | Porosity = {porosity:.6f}\n" + 'eb2.geom_type = "tiff"\n' + f'torteff.tiff_filename = "{tiff_path.name}"\n' + f"torteff.phase = {pore_label}\n" + f'torteff.direction = "{direction}"\n' + f"amr.n_cell = {width_x} {height_y} {depth_z}\n" + f"amr.max_grid_size = {int(max_grid_size)}\n" + "torteff.verbose = 1\n" + ) + inputs_path.write_text(inputs_text, encoding="utf-8") + return tiff_path, inputs_path diff --git a/src/openlsrx/metrics/__init__.py b/src/openlsrx/metrics/__init__.py index cd47445..aa468a5 100644 --- a/src/openlsrx/metrics/__init__.py +++ b/src/openlsrx/metrics/__init__.py @@ -1,4 +1,4 @@ -"""Quantitative image-fidelity and porous-media petrophysical metrics.""" +"""Quantitative image-fidelity, petrophysical, and pore-scale transport XCT metrics.""" from .image import ( compute_mae, @@ -22,9 +22,19 @@ segment_pore_space, two_point_correlation_radial, ) +from .transport import ( + compute_directional_tortuosity, + compute_drainage_curve, + compute_drainage_rmse, + evaluate_transport_metrics, + solve_steady_state_diffusion_axis, +) __all__ = [ "compute_dice_iou", + "compute_directional_tortuosity", + "compute_drainage_curve", + "compute_drainage_rmse", "compute_euler_density", "compute_mae", "compute_ms_ssim", @@ -40,6 +50,8 @@ "compute_through_plane_anisotropy_error", "evaluate_image_metrics", "evaluate_petrophysical_metrics", + "evaluate_transport_metrics", "segment_pore_space", + "solve_steady_state_diffusion_axis", "two_point_correlation_radial", ] diff --git a/src/openlsrx/metrics/image.py b/src/openlsrx/metrics/image.py index 3e2a8fc..edac5cb 100644 --- a/src/openlsrx/metrics/image.py +++ b/src/openlsrx/metrics/image.py @@ -70,6 +70,7 @@ def compute_ssim( if kernel_size < 3: return 1.0 if torch.allclose(p, t) else 0.0 + use_gaussian = kernel_size == 11 scores: list[float] = [] weights: list[int] = [] for start in range(0, p.shape[0], chunk_size): @@ -78,6 +79,7 @@ def compute_ssim( val = structural_similarity_index_measure( p_chunk, t_chunk, + gaussian_kernel=use_gaussian, data_range=data_range, kernel_size=kernel_size, ) diff --git a/src/openlsrx/metrics/transport.py b/src/openlsrx/metrics/transport.py new file mode 100644 index 0000000..adaeec5 --- /dev/null +++ b/src/openlsrx/metrics/transport.py @@ -0,0 +1,366 @@ +"""3D pore-scale diffusive tortuosity and capillary drainage physics metrics for XCT.""" + +from __future__ import annotations + +import numpy as np +import porespy as ps +from scipy import sparse +from scipy.ndimage import distance_transform_edt, generate_binary_structure, label +from scipy.sparse.linalg import LinearOperator, cg + +from .petrophysics import ThresholdMethod, segment_pore_space + + +def solve_steady_state_diffusion_axis( + pore_mask: np.ndarray, + axis: int = 0, + tol: float = 1e-6, + maxiter: int = 4000, +) -> dict[str, float]: + """Solve 3D steady-state Fickian Laplace diffusion ``div(grad c) = 0`` along ``axis``. + + Uses a cell-centered 6-neighbor finite-volume discretization matching ``OpenImpala`` + and ``TauFactor``: + - Unit voxel spacing ``dx = 1`` and unit intrinsic diffusivity ``D_0 = 1``. + - Dirichlet boundary conditions ``c = 1`` on the outer inlet face (``x_axis = 0``) and + ``c = 0`` on the outer outlet face (``x_axis = L``) with half-cell boundary conductance ``2``. + - Zero-flux Neumann conditions on all solid-pore interfaces and lateral domain boundaries. + + For an analytical straight pore channel of cross-sectional fraction ``phi``, this yields + ``D_eff / D_0 = phi`` and ``tortuosity = 1.0`` to solver tolerance. + """ + mask = np.asarray(pore_mask, dtype=bool) + if mask.ndim != 3: + raise ValueError(f"Expected 3D boolean pore_mask, got shape {mask.shape}.") + if axis not in (0, 1, 2): + raise ValueError(f"axis must be 0, 1, or 2, got {axis}.") + + porosity = float(mask.mean()) + if porosity == 0.0: + return { + "porosity": 0.0, + "percolating_porosity": 0.0, + "d_eff": 0.0, + "tortuosity": float("inf"), + "formation_factor": float("inf"), + "percolates": 0.0, + } + + ordered = np.moveaxis(mask, axis, 0) # (L, M, N) + length, m_dim, n_dim = ordered.shape + a_geom = float(m_dim * n_dim) + + # Identify 6-connected (face-sharing) pore clusters spanning inlet (0) to outlet (L - 1) + conn6 = generate_binary_structure(3, 1) + labels, _ = label(ordered, structure=conn6) + inlet_labels = set(np.unique(labels[0])) - {0} + outlet_labels = set(np.unique(labels[-1])) - {0} + perc_labels = inlet_labels & outlet_labels + + if not perc_labels: + return { + "porosity": porosity, + "percolating_porosity": 0.0, + "d_eff": 0.0, + "tortuosity": float("inf"), + "formation_factor": float("inf"), + "percolates": 0.0, + } + + perc_mask = np.isin(labels, list(perc_labels)) + perc_porosity = float(perc_mask.mean()) + + if length == 1: + d_eff = perc_porosity + return { + "porosity": porosity, + "percolating_porosity": perc_porosity, + "d_eff": d_eff, + "tortuosity": porosity / d_eff, + "formation_factor": 1.0 / d_eff, + "percolates": 1.0, + } + + # Map percolating pore voxels to compact [0, n_nodes) indices + active_flat = np.flatnonzero(perc_mask) + n_nodes = int(active_flat.size) + node_idx = np.full(perc_mask.shape, -1, dtype=np.int64) + node_idx.ravel()[active_flat] = np.arange(n_nodes, dtype=np.int64) + + rows_list: list[np.ndarray] = [] + cols_list: list[np.ndarray] = [] + diag = np.zeros(n_nodes, dtype=np.float64) + + # Interior 6-neighbor pore-pore faces (conductance = 1.0) + for dim in range(3): + sl_lo = [slice(None), slice(None), slice(None)] + sl_hi = [slice(None), slice(None), slice(None)] + sl_lo[dim] = slice(0, -1) + sl_hi[dim] = slice(1, None) + both = perc_mask[tuple(sl_lo)] & perc_mask[tuple(sl_hi)] + u = node_idx[tuple(sl_lo)][both] + v = node_idx[tuple(sl_hi)][both] + if u.size > 0: + rows_list.append(u) + cols_list.append(v) + rows_list.append(v) + cols_list.append(u) + np.add.at(diag, u, 1.0) + np.add.at(diag, v, 1.0) + + # Half-cell Dirichlet faces on inlet (c = 1.0) and outlet (c = 0.0), conductance = 2.0 + rhs = np.zeros(n_nodes, dtype=np.float64) + inlet_nodes = node_idx[0][perc_mask[0]] + outlet_nodes = node_idx[-1][perc_mask[-1]] + diag[inlet_nodes] += 2.0 + rhs[inlet_nodes] += 2.0 + diag[outlet_nodes] += 2.0 + + diag_idx = np.arange(n_nodes, dtype=np.int64) + if rows_list: + off_r = np.concatenate(rows_list) + off_c = np.concatenate(cols_list) + off_v = np.full(off_r.size, -1.0, dtype=np.float64) + all_r = np.concatenate([diag_idx, off_r]) + all_c = np.concatenate([diag_idx, off_c]) + all_v = np.concatenate([diag, off_v]) + else: + all_r = diag_idx + all_c = diag_idx + all_v = diag + + mat = sparse.csr_matrix((all_v, (all_r, all_c)), shape=(n_nodes, n_nodes)) + inv_diag = 1.0 / np.maximum(diag, 1e-12) + precond = LinearOperator( + shape=(n_nodes, n_nodes), + dtype=np.float64, + matvec=lambda x: inv_diag * x, + ) + conc, _ = cg(mat, rhs, rtol=tol, atol=0.0, maxiter=maxiter, M=precond) + + q_in = float(np.sum(2.0 * (1.0 - conc[inlet_nodes]))) + q_out = float(np.sum(2.0 * conc[outlet_nodes])) + q_mean = max(0.0, 0.5 * (q_in + q_out)) + + d_eff = (q_mean * float(length)) / a_geom + if d_eff <= 1e-12: + return { + "porosity": porosity, + "percolating_porosity": perc_porosity, + "d_eff": 0.0, + "tortuosity": float("inf"), + "formation_factor": float("inf"), + "percolates": 0.0, + } + + return { + "porosity": porosity, + "percolating_porosity": perc_porosity, + "d_eff": d_eff, + "tortuosity": porosity / d_eff, + "formation_factor": 1.0 / d_eff, + "percolates": 1.0, + } + + +def compute_directional_tortuosity( + pore_mask: np.ndarray, + tol: float = 1e-6, + maxiter: int = 4000, +) -> dict[str, float]: + """Compute directional effective diffusivity ``D_eff/D_0``, tortuosity ``tau``, and anisotropy across ``z, y, x``.""" + res_z = solve_steady_state_diffusion_axis(pore_mask, axis=0, tol=tol, maxiter=maxiter) + res_y = solve_steady_state_diffusion_axis(pore_mask, axis=1, tol=tol, maxiter=maxiter) + res_x = solve_steady_state_diffusion_axis(pore_mask, axis=2, tol=tol, maxiter=maxiter) + + d_vals = [res_z["d_eff"], res_y["d_eff"], res_x["d_eff"]] + d_mean = float(np.mean(d_vals)) + finite_taus = [r["tortuosity"] for r in (res_z, res_y, res_x) if np.isfinite(r["tortuosity"])] + tau_mean = float(np.mean(finite_taus)) if finite_taus else float("inf") + ff_mean = (1.0 / d_mean) if d_mean > 1e-12 else float("inf") + + min_d = min(d_vals) + max_d = max(d_vals) + aniso_ratio = (max_d / min_d) if min_d > 1e-12 else (1.0 if max_d <= 1e-12 else float("inf")) + + return { + "porosity": res_z["porosity"], + "d_eff_z": res_z["d_eff"], + "d_eff_y": res_y["d_eff"], + "d_eff_x": res_x["d_eff"], + "d_eff_mean": d_mean, + "tortuosity_z": res_z["tortuosity"], + "tortuosity_y": res_y["tortuosity"], + "tortuosity_x": res_x["tortuosity"], + "tortuosity_mean": tau_mean, + "formation_factor_mean": ff_mean, + "percolating_axes_count": res_z["percolates"] + res_y["percolates"] + res_x["percolates"], + "d_eff_anisotropy_spread": max_d - min_d, + "transport_anisotropy_ratio": aniso_ratio, + } + + +def compute_drainage_curve( + pore_mask: np.ndarray, + n_steps: int = 16, + voxel_size: float = 1.0, + inlet_axis: int = 0, + pc_grid: np.ndarray | None = None, +) -> tuple[np.ndarray, np.ndarray]: + """Simulate inlet-connected capillary drainage (mercury intrusion) on a 3D pore mask. + + Returns ``(pc_values, non_wetting_saturation)`` evaluated on ``pc_grid`` (or a geometric + capillary-pressure schedule spanning ``[2 / r_max, 2 / r_min]``). + """ + mask = np.asarray(pore_mask, dtype=bool) + if mask.ndim != 3: + raise ValueError(f"Expected 3D boolean pore_mask, got shape {mask.shape}.") + + n_pores = int(mask.sum()) + if n_pores == 0: + grid = ( + np.geomspace(0.1, 2.0, n_steps, dtype=np.float64) + if pc_grid is None + else np.asarray(pc_grid, dtype=np.float64) + ) + return grid, np.zeros_like(grid, dtype=np.float64) + + dt = distance_transform_edt(mask) * float(voxel_size) + if pc_grid is None: + r_vals = dt[mask] + r_min = max(float(r_vals.min()), 0.5 * float(voxel_size)) + r_max = max(float(r_vals.max()), r_min * 1.05) + grid = np.geomspace(2.0 / r_max, 2.0 / r_min, n_steps, dtype=np.float64) + else: + grid = np.asarray(pc_grid, dtype=np.float64) + + inlets = np.zeros_like(mask, dtype=bool) + sl: list[slice | int] = [slice(None), slice(None), slice(None)] + sl[inlet_axis] = 0 + inlets[tuple(sl)] = mask[tuple(sl)] + if not inlets.any(): + return grid, np.zeros_like(grid, dtype=np.float64) + + pc_transform = np.full(mask.shape, np.inf, dtype=np.float32) + pc_transform[mask] = 2.0 / np.maximum(dt[mask], 1e-6).astype(np.float32) + + drn = ps.simulations.drainage( + im=mask, + pc=pc_transform, + dt=dt.astype(np.float32), + inlets=inlets, + steps=max(4, int(n_steps)), + ) + im_pc = np.asarray(drn.im_pc, dtype=np.float64) + pore_pc = im_pc[mask] + snwp = np.array([float(np.sum(pore_pc <= p)) / float(n_pores) for p in grid], dtype=np.float64) + return grid, snwp + + +def compute_drainage_rmse( + pred_mask: np.ndarray, + true_mask: np.ndarray, + n_steps: int = 16, + voxel_size: float = 1.0, + inlet_axis: int = 0, +) -> float: + """RMSE between simulated capillary drainage saturation curves ``S_nwp(P_c)``.""" + pc_grid, true_snwp = compute_drainage_curve( + true_mask, + n_steps=n_steps, + voxel_size=voxel_size, + inlet_axis=inlet_axis, + ) + _, pred_snwp = compute_drainage_curve( + pred_mask, + n_steps=n_steps, + voxel_size=voxel_size, + inlet_axis=inlet_axis, + pc_grid=pc_grid, + ) + return float(np.sqrt(np.mean((pred_snwp - true_snwp) ** 2))) + + +def _json_safe(val: float) -> float | None: + return float(val) if np.isfinite(val) else None + + +def evaluate_transport_metrics( + pred_vol: np.ndarray, + true_vol: np.ndarray, + threshold_method: ThresholdMethod = "otsu", + fixed_threshold: float | None = None, + pore_is_low: bool = True, + voxel_size: float = 1.0, + drainage_steps: int = 16, +) -> dict[str, float | None]: + """Evaluate 3D diffusive transport and capillary drainage fidelity between SR and HR XCT volumes. + + Uses the ground-truth segmentation threshold consistently across ``true_vol`` and ``pred_vol`` + (matching ``evaluate_petrophysical_metrics``) so greyscale bias or blurring is faithfully + reflected in pore-throat connectivity and effective diffusivity. + """ + true_mask, thr = segment_pore_space( + true_vol, + method=threshold_method, + fixed_value=fixed_threshold, + pore_is_low=pore_is_low, + ) + pred_mask, _ = segment_pore_space( + pred_vol, + method="fixed", + fixed_value=thr, + pore_is_low=pore_is_low, + ) + + pred_t = compute_directional_tortuosity(pred_mask) + true_t = compute_directional_tortuosity(true_mask) + + d_eff_mae = float( + np.mean( + [ + abs(pred_t["d_eff_z"] - true_t["d_eff_z"]), + abs(pred_t["d_eff_y"] - true_t["d_eff_y"]), + abs(pred_t["d_eff_x"] - true_t["d_eff_x"]), + ] + ) + ) + d_eff_rel_err = ( + abs(pred_t["d_eff_mean"] - true_t["d_eff_mean"]) / true_t["d_eff_mean"] + if true_t["d_eff_mean"] > 1e-12 + else None + ) + tau_abs_err = ( + abs(pred_t["tortuosity_mean"] - true_t["tortuosity_mean"]) + if np.isfinite(pred_t["tortuosity_mean"]) and np.isfinite(true_t["tortuosity_mean"]) + else None + ) + drainage_rmse = compute_drainage_rmse( + pred_mask, + true_mask, + n_steps=drainage_steps, + voxel_size=voxel_size, + ) + + return { + "threshold_used": float(thr), + "pred_d_eff_z": _json_safe(pred_t["d_eff_z"]), + "pred_d_eff_y": _json_safe(pred_t["d_eff_y"]), + "pred_d_eff_x": _json_safe(pred_t["d_eff_x"]), + "pred_d_eff_mean": _json_safe(pred_t["d_eff_mean"]), + "true_d_eff_mean": _json_safe(true_t["d_eff_mean"]), + "d_eff_mae": d_eff_mae, + "d_eff_rel_error": _json_safe(d_eff_rel_err) if d_eff_rel_err is not None else None, + "pred_tortuosity_mean": _json_safe(pred_t["tortuosity_mean"]), + "true_tortuosity_mean": _json_safe(true_t["tortuosity_mean"]), + "tortuosity_abs_error": _json_safe(tau_abs_err) if tau_abs_err is not None else None, + "pred_formation_factor_mean": _json_safe(pred_t["formation_factor_mean"]), + "true_formation_factor_mean": _json_safe(true_t["formation_factor_mean"]), + "pred_percolating_axes": float(pred_t["percolating_axes_count"]), + "true_percolating_axes": float(true_t["percolating_axes_count"]), + "pred_d_eff_anisotropy_spread": float(pred_t["d_eff_anisotropy_spread"]), + "true_d_eff_anisotropy_spread": float(true_t["d_eff_anisotropy_spread"]), + "pred_transport_anisotropy_ratio": _json_safe(pred_t["transport_anisotropy_ratio"]), + "true_transport_anisotropy_ratio": _json_safe(true_t["transport_anisotropy_ratio"]), + "drainage_snwp_rmse": drainage_rmse, + } diff --git a/src/openlsrx/models/__init__.py b/src/openlsrx/models/__init__.py index 6bbb247..5422e53 100644 --- a/src/openlsrx/models/__init__.py +++ b/src/openlsrx/models/__init__.py @@ -1,16 +1,37 @@ """Neural network models and blocks for OpenLSR-X.""" from .blocks import ConvBlock, NormType, ResidualBlock, UpsampleBlock -from .srgan import SRGAN +from .rrdb import RRDB, ArbitraryScaleUpsample, DenseResidualBlock, RRDBGenerator +from .sr3d import ( + Generator3D, + PairedSubCubeDataset, + ResidualBlock3D, + SR3DResNet, + build_3d_subcube_dataloaders, + super_resolve_volume_3d, +) +from .srgan import SRGAN, BackboneType, GANLossType from .submodels import Discriminator, Generator, OutputActivation __all__ = [ + "ArbitraryScaleUpsample", + "BackboneType", "ConvBlock", + "DenseResidualBlock", "Discriminator", + "GANLossType", "Generator", + "Generator3D", "NormType", "OutputActivation", + "PairedSubCubeDataset", + "RRDB", + "RRDBGenerator", "ResidualBlock", + "ResidualBlock3D", + "SR3DResNet", "SRGAN", "UpsampleBlock", + "build_3d_subcube_dataloaders", + "super_resolve_volume_3d", ] diff --git a/src/openlsrx/models/rrdb.py b/src/openlsrx/models/rrdb.py new file mode 100644 index 0000000..42e9534 --- /dev/null +++ b/src/openlsrx/models/rrdb.py @@ -0,0 +1,192 @@ +"""Normalization-free RRDB (ESRGAN-style) 2D generator with arbitrary-scale XCT upsampling.""" + +from __future__ import annotations + +import math + +import torch +import torch.nn.functional as F +from torch import Tensor, nn + +from ..losses import custom_sigmoid +from .blocks import UpsampleBlock +from .submodels import OutputActivation + + +class DenseResidualBlock(nn.Module): + """5-layer densely connected convolutional block without batch/pixel normalization. + + Omitting feature normalization preserves absolute X-ray attenuation contrasts across + batches (Wang et al., ESRGAN, ECCV Workshops 2018). + """ + + def __init__( + self, + channels: int = 64, + growth_channels: int = 32, + res_scale: float = 0.2, + ) -> None: + super().__init__() + self.res_scale = float(res_scale) + self.act = nn.LeakyReLU(0.2, inplace=True) + + self.conv1 = nn.Conv2d(channels, growth_channels, 3, 1, 1, padding_mode="reflect") + self.conv2 = nn.Conv2d( + channels + growth_channels, growth_channels, 3, 1, 1, padding_mode="reflect" + ) + self.conv3 = nn.Conv2d( + channels + 2 * growth_channels, growth_channels, 3, 1, 1, padding_mode="reflect" + ) + self.conv4 = nn.Conv2d( + channels + 3 * growth_channels, growth_channels, 3, 1, 1, padding_mode="reflect" + ) + self.conv5 = nn.Conv2d( + channels + 4 * growth_channels, channels, 3, 1, 1, padding_mode="reflect" + ) + + def forward(self, x: Tensor) -> Tensor: + x1 = self.act(self.conv1(x)) + x2 = self.act(self.conv2(torch.cat([x, x1], dim=1))) + x3 = self.act(self.conv3(torch.cat([x, x1, x2], dim=1))) + x4 = self.act(self.conv4(torch.cat([x, x1, x2, x3], dim=1))) + x5 = self.conv5(torch.cat([x, x1, x2, x3, x4], dim=1)) + return x + self.res_scale * x5 + + +class RRDB(nn.Module): + """Residual-in-Residual Dense Block chaining three ``DenseResidualBlock`` units.""" + + def __init__( + self, + channels: int = 64, + growth_channels: int = 32, + res_scale: float = 0.2, + ) -> None: + super().__init__() + self.res_scale = float(res_scale) + self.rdb1 = DenseResidualBlock(channels, growth_channels, res_scale=res_scale) + self.rdb2 = DenseResidualBlock(channels, growth_channels, res_scale=res_scale) + self.rdb3 = DenseResidualBlock(channels, growth_channels, res_scale=res_scale) + + def forward(self, x: Tensor) -> Tensor: + out = self.rdb3(self.rdb2(self.rdb1(x))) + return x + self.res_scale * out + + +class ArbitraryScaleUpsample(nn.Module): + """Hybrid sub-pixel + resize-convolution upsampler supporting any scale >= 1.0. + + - Exact powers of two (``1, 2, 4, 8``) use learned ``UpsampleBlock`` (``PixelShuffle``) stages. + - Non-power-of-two beamline ratios (e.g., ``2.5`` for I13-2 ``0.8125 -> 0.325 um`` or ``5.0`` + for ``1.625 -> 0.325 um``) combine ``floor(log2(s))`` sub-pixel stages with an exact + bicubic resize-convolution refinement stage. + """ + + def __init__(self, channels: int = 64, scale_factor: float | int = 2) -> None: + super().__init__() + sf = float(scale_factor) + if sf < 1.0: + raise ValueError(f"scale_factor must be >= 1.0, got {scale_factor}.") + self.scale_factor = sf + + n_pow2 = int(math.floor(math.log2(sf) + 1e-9)) + self.pow2_scale = 2**n_pow2 + self.residual_scale = sf / float(self.pow2_scale) + + self.pow2_blocks = nn.Sequential( + *[UpsampleBlock(in_channels=channels, scale_factor=2) for _ in range(n_pow2)] + ) + self.has_residual = not math.isclose(self.residual_scale, 1.0, rel_tol=1e-5, abs_tol=1e-5) + self.refine_conv: nn.Module | None = ( + nn.Sequential( + nn.Conv2d(channels, channels, 3, 1, 1, padding_mode="reflect"), + nn.LeakyReLU(0.2, inplace=True), + ) + if self.has_residual + else None + ) + + def forward(self, x: Tensor) -> Tensor: + target_h = int(round(x.shape[-2] * self.scale_factor)) + target_w = int(round(x.shape[-1] * self.scale_factor)) + out = self.pow2_blocks(x) + if self.has_residual and self.refine_conv is not None: + out = F.interpolate(out, size=(target_h, target_w), mode="bicubic", align_corners=False) + out = self.refine_conv(out) + return out + + +class RRDBGenerator(nn.Module): + """ESRGAN-style RRDB Generator adapted for quantitative single-channel XCT slices.""" + + def __init__( + self, + in_channels: int = 1, + out_channels: int = 64, + growth_channels: int = 32, + num_blocks: int = 8, + super_res_factor: float | int = 2, + sigmoid_k: float = 1.0, + output_activation: OutputActivation = "clamp", + ) -> None: + super().__init__() + if output_activation not in ("custom_sigmoid", "sigmoid", "clamp", "identity"): + raise ValueError( + f"Unsupported output_activation={output_activation!r}. " + "Expected 'custom_sigmoid', 'sigmoid', 'clamp', or 'identity'." + ) + + self.in_channels = int(in_channels) + self.out_channels = int(out_channels) + self.growth_channels = int(growth_channels) + self.num_blocks = int(num_blocks) + self.super_res_factor = float(super_res_factor) + self.sigmoid_k = float(sigmoid_k) + self.output_activation: OutputActivation = output_activation + + self.conv_first = nn.Conv2d( + self.in_channels, self.out_channels, 3, 1, 1, padding_mode="reflect" + ) + self.trunk = nn.Sequential( + *[ + RRDB(channels=self.out_channels, growth_channels=self.growth_channels) + for _ in range(self.num_blocks) + ] + ) + self.trunk_conv = nn.Conv2d( + self.out_channels, self.out_channels, 3, 1, 1, padding_mode="reflect" + ) + self.upsample = ArbitraryScaleUpsample( + channels=self.out_channels, scale_factor=self.super_res_factor + ) + final_conv = nn.Conv2d(self.out_channels, self.in_channels, 3, 1, 1, padding_mode="reflect") + if final_conv.bias is not None: + nn.init.zeros_(final_conv.bias) + final_conv.weight.data.mul_(0.1) + self.hr_head = nn.Sequential( + nn.Conv2d(self.out_channels, self.out_channels, 3, 1, 1, padding_mode="reflect"), + nn.LeakyReLU(0.2, inplace=True), + final_conv, + ) + + def _apply_output_activation(self, x: Tensor) -> Tensor: + if self.output_activation == "custom_sigmoid": + return custom_sigmoid(x, k=self.sigmoid_k) + if self.output_activation == "sigmoid": + return torch.sigmoid(x) + if self.output_activation == "clamp": + return torch.clamp(x, 0.0, 1.0) + return x + + def forward(self, x: Tensor) -> Tensor: + target_h = int(round(x.shape[-2] * self.super_res_factor)) + target_w = int(round(x.shape[-1] * self.super_res_factor)) + base = F.interpolate(x, size=(target_h, target_w), mode="bicubic", align_corners=False) + feat = self.conv_first(x) + trunk = self.trunk_conv(self.trunk(feat)) + feat = feat + trunk + up = self.upsample(feat) + residual = self.hr_head(up) + if self.output_activation in ("clamp", "identity"): + return self._apply_output_activation(base + residual) + return self._apply_output_activation(residual) diff --git a/src/openlsrx/models/sr3d.py b/src/openlsrx/models/sr3d.py new file mode 100644 index 0000000..aca3f47 --- /dev/null +++ b/src/openlsrx/models/sr3d.py @@ -0,0 +1,329 @@ +"""Memory-matched native 3D convolutional residual super-resolution baseline (SR3DResNet). + +Provides a direct volumetric 3D CNN baseline to benchmark GPU memory footprint, training +throughput, and petrophysical fidelity against 2D + 3-Axis Orthogonal Fourier Fusion. +""" + +from __future__ import annotations + +import math +from typing import Literal + +import numpy as np +import pytorch_lightning as pl +import torch +import torch.nn.functional as F +from torch import Tensor, nn, optim +from torch.utils.data import DataLoader, Dataset + +from ..metrics import compute_mae, compute_psnr, compute_rmse, compute_ssim_3d_isotropic + + +class ResidualBlock3D(nn.Module): + """3D reflect-padded residual block without batch normalization.""" + + def __init__(self, channels: int = 32, res_scale: float = 0.2) -> None: + super().__init__() + self.res_scale = float(res_scale) + self.block = nn.Sequential( + nn.Conv3d(channels, channels, kernel_size=3, padding=1, padding_mode="reflect"), + nn.LeakyReLU(0.2, inplace=True), + nn.Conv3d(channels, channels, kernel_size=3, padding=1, padding_mode="reflect"), + ) + + def forward(self, x: Tensor) -> Tensor: + return x + self.res_scale * self.block(x) + + +class Generator3D(nn.Module): + """3D residual super-resolution network using trilinear resize-Conv3d upsampling.""" + + def __init__( + self, + in_channels: int = 1, + out_channels: int = 24, + num_blocks: int = 4, + super_res_factor: int = 2, + ) -> None: + super().__init__() + if super_res_factor < 1 or (super_res_factor & (super_res_factor - 1)) != 0: + raise ValueError( + f"Generator3D super_res_factor must be a power of 2, got {super_res_factor}." + ) + self.in_channels = int(in_channels) + self.out_channels = int(out_channels) + self.num_blocks = int(num_blocks) + self.super_res_factor = int(super_res_factor) + + self.conv_first = nn.Conv3d( + self.in_channels, self.out_channels, kernel_size=3, padding=1, padding_mode="reflect" + ) + self.trunk = nn.Sequential( + *[ResidualBlock3D(self.out_channels) for _ in range(self.num_blocks)] + ) + self.trunk_conv = nn.Conv3d( + self.out_channels, self.out_channels, kernel_size=3, padding=1, padding_mode="reflect" + ) + + n_up = int(math.log2(self.super_res_factor)) + up_layers: list[nn.Module] = [] + for _ in range(n_up): + up_layers.extend( + [ + nn.Upsample(scale_factor=2, mode="trilinear", align_corners=False), + nn.Conv3d( + self.out_channels, + self.out_channels, + kernel_size=3, + padding=1, + padding_mode="reflect", + ), + nn.LeakyReLU(0.2, inplace=True), + ] + ) + self.upsample = nn.Sequential(*up_layers) + self.conv_last = nn.Conv3d( + self.out_channels, self.in_channels, kernel_size=3, padding=1, padding_mode="reflect" + ) + nn.init.zeros_(self.conv_last.bias) # type: ignore[arg-type] + self.conv_last.weight.data.mul_(0.1) + + def forward(self, x: Tensor) -> Tensor: + base = F.interpolate( + x, + scale_factor=self.super_res_factor, + mode="trilinear", + align_corners=False, + ) + feat = self.conv_first(x) + feat = feat + self.trunk_conv(self.trunk(feat)) + out = base + self.conv_last(self.upsample(feat)) + return torch.clamp(out, 0.0, 1.0) + + +class PairedSubCubeDataset(Dataset[tuple[Tensor, Tensor]]): + """Random/grid 3D sub-cube sampler for training ``SR3DResNet`` on paired 3D slabs.""" + + def __init__( + self, + lr_vol: np.ndarray, + hr_vol: np.ndarray, + scale_factor: int = 2, + patch_size_lr: int = 16, + samples_per_epoch: int = 32, + deterministic: bool = False, + seed: int = 42, + ) -> None: + self.lr_vol = np.asarray(lr_vol, dtype=np.float32) + self.hr_vol = np.asarray(hr_vol, dtype=np.float32) + self.scale_factor = int(scale_factor) + self.patch_size_lr = tuple(min(int(patch_size_lr), s) for s in self.lr_vol.shape) + self.samples_per_epoch = int(samples_per_epoch) + self.deterministic = bool(deterministic) + self.rng = np.random.default_rng(seed) + + def __len__(self) -> int: + return self.samples_per_epoch + + def __getitem__(self, idx: int) -> tuple[Tensor, Tensor]: + pd, ph, pw = self.patch_size_lr + d, h, w = self.lr_vol.shape + if self.deterministic: + rng = np.random.default_rng(idx + 1000) + z0 = int(rng.integers(0, max(1, d - pd + 1))) + y0 = int(rng.integers(0, max(1, h - ph + 1))) + x0 = int(rng.integers(0, max(1, w - pw + 1))) + else: + z0 = int(self.rng.integers(0, max(1, d - pd + 1))) + y0 = int(self.rng.integers(0, max(1, h - ph + 1))) + x0 = int(self.rng.integers(0, max(1, w - pw + 1))) + + s = self.scale_factor + lr_patch = self.lr_vol[z0 : z0 + pd, y0 : y0 + ph, x0 : x0 + pw] + hr_patch = self.hr_vol[ + z0 * s : (z0 + pd) * s, + y0 * s : (y0 + ph) * s, + x0 * s : (x0 + pw) * s, + ] + return ( + torch.from_numpy(lr_patch.copy()).unsqueeze(0), + torch.from_numpy(hr_patch.copy()).unsqueeze(0), + ) + + +def build_3d_subcube_dataloaders( + lr_train: np.ndarray, + hr_train: np.ndarray, + lr_val: np.ndarray, + hr_val: np.ndarray, + scale_factor: int = 2, + patch_size_lr: int = 16, + train_samples: int = 32, + val_samples: int = 8, + batch_size: int = 2, + seed: int = 42, +) -> tuple[DataLoader[tuple[Tensor, Tensor]], DataLoader[tuple[Tensor, Tensor]]]: + """Build train and validation 3D sub-cube DataLoaders from disjoint 3D slabs.""" + tr_ds = PairedSubCubeDataset( + lr_train, + hr_train, + scale_factor=scale_factor, + patch_size_lr=patch_size_lr, + samples_per_epoch=train_samples, + deterministic=False, + seed=seed, + ) + va_ds = PairedSubCubeDataset( + lr_val, + hr_val, + scale_factor=scale_factor, + patch_size_lr=patch_size_lr, + samples_per_epoch=val_samples, + deterministic=True, + seed=seed, + ) + return ( + DataLoader(tr_ds, batch_size=batch_size, shuffle=False), + DataLoader(va_ds, batch_size=batch_size, shuffle=False), + ) + + +class SR3DResNet(pl.LightningModule): + """PyTorch Lightning 3D CNN super-resolution baseline for XCT volumes.""" + + def __init__( + self, + learning_rate: float = 2e-4, + super_res_factor: int = 2, + out_channels: int = 24, + num_blocks: int = 4, + loss_type: Literal["l1", "mse"] = "l1", + ) -> None: + super().__init__() + self.save_hyperparameters() + self.learning_rate = float(learning_rate) + self.super_res_factor = int(super_res_factor) + self.gen = Generator3D( + in_channels=1, + out_channels=out_channels, + num_blocks=num_blocks, + super_res_factor=super_res_factor, + ) + self.criterion: nn.Module = nn.L1Loss() if loss_type == "l1" else nn.MSELoss() + + def forward(self, x: Tensor) -> Tensor: + return self.gen(x) + + def training_step(self, batch: tuple[Tensor, Tensor], batch_idx: int) -> Tensor: + lr_cube, hr_cube = batch + pred = self.gen(lr_cube) + loss = self.criterion(pred, hr_cube) + self.log( + "train/loss_3d", + loss.detach(), + prog_bar=True, + on_step=False, + on_epoch=True, + batch_size=int(lr_cube.shape[0]), + ) + return loss + + @torch.no_grad() + def validation_step(self, batch: tuple[Tensor, Tensor], batch_idx: int) -> None: + lr_cube, hr_cube = batch + pred = self.gen(lr_cube).to(torch.float32) + target = hr_cube.to(torch.float32) + bs = int(lr_cube.shape[0]) + self.log( + "val/psnr", compute_psnr(pred, target), prog_bar=True, on_epoch=True, batch_size=bs + ) + self.log( + "val/ssim_3d", + compute_ssim_3d_isotropic(pred[0, 0], target[0, 0]), + on_epoch=True, + batch_size=bs, + ) + self.log("val/mae", compute_mae(pred, target), on_epoch=True, batch_size=bs) + self.log("val/rmse", compute_rmse(pred, target), on_epoch=True, batch_size=bs) + + def configure_optimizers(self) -> optim.Optimizer: + return optim.Adam(self.parameters(), lr=self.learning_rate) + + +@torch.no_grad() +def super_resolve_volume_3d( + generator_3d: nn.Module, + low_volume: np.ndarray, + scale_factor: int = 2, + patch_size_lr: int = 24, + overlap_lr: int = 4, + device: torch.device | str | None = None, +) -> np.ndarray: + """Sliding-window 3D inference with raised-cosine feathering to eliminate tile seams.""" + vol = np.asarray(low_volume, dtype=np.float32) + if vol.ndim != 3: + raise ValueError(f"Expected 3D LR volume, got shape {vol.shape}.") + + if device is None: + try: + dev = next(generator_3d.parameters()).device + except StopIteration: + dev = torch.device("cpu") + else: + dev = torch.device(device) + + was_training = generator_3d.training + generator_3d.eval() + + d, h, w = vol.shape + pd, ph, pw = min(patch_size_lr, d), min(patch_size_lr, h), min(patch_size_lr, w) + step_d = max(1, pd - overlap_lr) + step_h = max(1, ph - overlap_lr) + step_w = max(1, pw - overlap_lr) + + s = int(scale_factor) + out_sum = np.zeros((d * s, h * s, w * s), dtype=np.float32) + weight_sum = np.zeros((d * s, h * s, w * s), dtype=np.float32) + + def _window_1d(n: int) -> np.ndarray: + if n <= 2: + return np.ones(n, dtype=np.float32) + return (0.5 - 0.5 * np.cos(2.0 * np.pi * np.arange(n) / (n - 1))).astype(np.float32) + 1e-3 + + win_3d = ( + _window_1d(pd * s)[:, None, None] + * _window_1d(ph * s)[None, :, None] + * _window_1d(pw * s)[None, None, :] + ) + + def _starts(total: int, patch: int, step: int) -> list[int]: + if total <= patch: + return [0] + pts = list(range(0, total - patch, step)) + if not pts or pts[-1] != total - patch: + pts.append(total - patch) + return pts + + try: + for z0 in _starts(d, pd, step_d): + for y0 in _starts(h, ph, step_h): + for x0 in _starts(w, pw, step_w): + patch = vol[z0 : z0 + pd, y0 : y0 + ph, x0 : x0 + pw] + inp = torch.from_numpy(patch).unsqueeze(0).unsqueeze(0).to(dev) + pred = F.pad(inp, (0, 0, 0, 0, 0, 0)) # keep contiguous tensor + sr_patch = generator_3d(pred)[0, 0].to(torch.float32).cpu().numpy() + out_sum[ + z0 * s : (z0 + pd) * s, + y0 * s : (y0 + ph) * s, + x0 * s : (x0 + pw) * s, + ] += sr_patch * win_3d + weight_sum[ + z0 * s : (z0 + pd) * s, + y0 * s : (y0 + ph) * s, + x0 * s : (x0 + pw) * s, + ] += win_3d + finally: + if was_training: + generator_3d.train() + + return np.clip(out_sum / np.maximum(weight_sum, 1e-8), 0.0, 1.0) diff --git a/src/openlsrx/models/srgan.py b/src/openlsrx/models/srgan.py index 763d1a4..0bbf262 100644 --- a/src/openlsrx/models/srgan.py +++ b/src/openlsrx/models/srgan.py @@ -15,8 +15,12 @@ from ..losses import VGGLoss from ..metrics.image import compute_mae, compute_psnr, compute_rmse, compute_ssim from .blocks import NormType +from .rrdb import RRDBGenerator from .submodels import Discriminator, Generator, OutputActivation +BackboneType = Literal["srgan", "rrdb"] +GANLossType = Literal["standard", "ragan"] + def _ensure_4d(x: Tensor) -> Tensor: """Convert ``(B, H, W)`` or ``(B, 1, H, W)`` tensors to ``(B, 1, H, W)``.""" @@ -30,10 +34,12 @@ def _ensure_4d(x: Tensor) -> Tensor: class SRGAN(pl.LightningModule): - """Learnable Super-Resolution GAN (and SRResNet ablation) for XCT data. + """Learnable Super-Resolution GAN (SRGAN / RRDB-ESRGAN / SRResNet) for XCT data. Supports: - - Adversarial + perceptual + pixelwise training (SRGAN) + - Classical ``Generator`` (``backbone='srgan'``) or normalization-free ``RRDBGenerator`` + with arbitrary beamline magnification ratios (``backbone='rrdb'``) + - Standard BCE adversarial loss or Relativistic Average GAN (``gan_loss_type='ragan'``) - Generator-only MSE/L1 pretraining or full ablation (``use_adversarial_loss=False``) - Optional generator warmup epochs (``pretrain_epochs > 0``) prior to GAN updates - Built-in PSNR, SSIM, MAE, and RMSE logging on validation and test splits @@ -45,9 +51,12 @@ def __init__( learning_rate: float = 1e-4, adaptive_size: int = 40, accumulate_n_batches: int = 1, - super_res_factor: int = 2, + super_res_factor: int | float = 2, num_blocks: int = 10, out_channels: int = 64, + growth_channels: int = 32, + backbone: BackboneType = "srgan", + gan_loss_type: GANLossType = "standard", norm_type: NormType = "pixel", output_activation: OutputActivation = "custom_sigmoid", sigmoid_k: float = 1.0, @@ -77,14 +86,27 @@ def __init__( raise ValueError( f"Unsupported lr_scheduler_type={lr_scheduler_type!r}. Expected 'constant' or 'CAWR'." ) + if backbone not in ("srgan", "rrdb"): + raise ValueError(f"Unsupported backbone={backbone!r}. Expected 'srgan' or 'rrdb'.") + if gan_loss_type not in ("standard", "ragan"): + raise ValueError( + f"Unsupported gan_loss_type={gan_loss_type!r}. Expected 'standard' or 'ragan'." + ) self.sample_id = str(sample_id) self.learning_rate = float(learning_rate) self.adaptive_size = int(adaptive_size) self.accumulate_n_batches = int(accumulate_n_batches) - self.super_res_factor = int(super_res_factor) + self.super_res_factor: int | float = ( + int(super_res_factor) + if float(super_res_factor).is_integer() + else float(super_res_factor) + ) self.num_blocks = int(num_blocks) self.out_channels = int(out_channels) + self.growth_channels = int(growth_channels) + self.backbone: BackboneType = backbone + self.gan_loss_type: GANLossType = gan_loss_type self.norm_type: NormType = norm_type self.output_activation: OutputActivation = output_activation self.sigmoid_k = float(sigmoid_k) @@ -105,15 +127,31 @@ def __init__( self.max_val_images_saved = int(max_val_images_saved) self.strict_loading = bool(strict_loading) - self.gen = Generator( - in_channels=1, - out_channels=self.out_channels, - num_blocks=self.num_blocks, - super_res_factor=self.super_res_factor, - sigmoid_k=self.sigmoid_k, - norm_type=self.norm_type, - output_activation=self.output_activation, - ) + if self.backbone == "rrdb": + self.gen: nn.Module = RRDBGenerator( + in_channels=1, + out_channels=self.out_channels, + growth_channels=self.growth_channels, + num_blocks=self.num_blocks, + super_res_factor=self.super_res_factor, + sigmoid_k=self.sigmoid_k, + output_activation=self.output_activation, + ) + else: + if not isinstance(self.super_res_factor, int): + raise ValueError( + f"backbone='srgan' requires integer power-of-2 super_res_factor, " + f"got {self.super_res_factor}. Use backbone='rrdb' for fractional scales." + ) + self.gen = Generator( + in_channels=1, + out_channels=self.out_channels, + num_blocks=self.num_blocks, + super_res_factor=self.super_res_factor, + sigmoid_k=self.sigmoid_k, + norm_type=self.norm_type, + output_activation=self.output_activation, + ) self.disc = Discriminator( in_channels=1, features=disc_features, @@ -179,11 +217,18 @@ def training_step(self, train_batch: tuple[Tensor, Tensor], batch_idx: int) -> N disc_real = self.disc(high_res) disc_fake = self.disc(fake_detached) - # One-sided label smoothing on real targets in [0.9, 1.0] - smooth_real = torch.ones_like(disc_real) - 0.1 * torch.rand_like(disc_real) - disc_loss_real = self.bce(disc_real, smooth_real) - disc_loss_fake = self.bce(disc_fake, torch.zeros_like(disc_fake)) - disc_loss = disc_loss_real + disc_loss_fake + if self.gan_loss_type == "ragan": + rel_real = disc_real - disc_fake.mean(dim=0, keepdim=True) + rel_fake = disc_fake - disc_real.mean(dim=0, keepdim=True) + disc_loss_real = self.bce(rel_real, torch.ones_like(rel_real)) + disc_loss_fake = self.bce(rel_fake, torch.zeros_like(rel_fake)) + disc_loss = 0.5 * (disc_loss_real + disc_loss_fake) + else: + # One-sided label smoothing on real targets in [0.9, 1.0] + smooth_real = torch.ones_like(disc_real) - 0.1 * torch.rand_like(disc_real) + disc_loss_real = self.bce(disc_real, smooth_real) + disc_loss_fake = self.bce(disc_fake, torch.zeros_like(disc_fake)) + disc_loss = disc_loss_real + disc_loss_fake disc_loss_val = disc_loss.detach() self.manual_backward(disc_loss / self.accumulate_n_batches) @@ -200,9 +245,20 @@ def training_step(self, train_batch: tuple[Tensor, Tensor], batch_idx: int) -> N if adv_active: disc_fake_for_gen = self.disc(fake) - adv_loss = self.adversarial_weight * self.bce( - disc_fake_for_gen, torch.ones_like(disc_fake_for_gen) - ) + if self.gan_loss_type == "ragan": + with torch.no_grad(): + disc_real_for_gen = self.disc(high_res) + rel_fake_g = disc_fake_for_gen - disc_real_for_gen.mean(dim=0, keepdim=True) + rel_real_g = disc_real_for_gen - disc_fake_for_gen.mean(dim=0, keepdim=True) + raw_adv = 0.5 * ( + self.bce(rel_fake_g, torch.ones_like(rel_fake_g)) + + self.bce(rel_real_g, torch.zeros_like(rel_real_g)) + ) + adv_loss = self.adversarial_weight * raw_adv + else: + adv_loss = self.adversarial_weight * self.bce( + disc_fake_for_gen, torch.ones_like(disc_fake_for_gen) + ) else: adv_loss = torch.zeros((), device=self.device, dtype=pixel_loss.dtype) diff --git a/tests/test_stage2_synchrotron_and_transport.py b/tests/test_stage2_synchrotron_and_transport.py new file mode 100644 index 0000000..a27a972 --- /dev/null +++ b/tests/test_stage2_synchrotron_and_transport.py @@ -0,0 +1,369 @@ +"""Tests for Stage 2: Zn-13X synchrotron aligner, RRDB/RaGAN, SR3DResNet, transport physics, and OpenImpala.""" + +from __future__ import annotations + +import csv +import importlib.util +import json +from pathlib import Path + +import h5py +import numpy as np +import pytest +import pytorch_lightning as pl +import tifffile +import torch + +from openlsrx.cli import main as cli_main +from openlsrx.data import ( + ZN13X_I13_RESOLUTIONS, + PairedVolumeDataModule, + extract_aligned_slab_from_h5, + prepare_zn13x_benchmark, + register_multires_pair_3d, + synthesize_zn13x_phantom_pair, +) +from openlsrx.inference import super_resolve_volume_orthogonal +from openlsrx.io import export_for_openimpala +from openlsrx.metrics import ( + compute_directional_tortuosity, + compute_drainage_curve, + compute_drainage_rmse, + evaluate_transport_metrics, + solve_steady_state_diffusion_axis, +) +from openlsrx.models import ( + SRGAN, + RRDBGenerator, + SR3DResNet, + build_3d_subcube_dataloaders, + super_resolve_volume_3d, +) + + +def test_analytical_straight_and_constricted_tortuosity() -> None: + # 1) Straight duct along axis 0 (z): cross-section 4x4 out of 8x8 -> phi = 0.25, tau_z = 1.0 + duct = np.zeros((12, 8, 8), dtype=bool) + duct[:, 2:6, 2:6] = True + res = compute_directional_tortuosity(duct, tol=1e-8) + + assert res["porosity"] == pytest.approx(0.25) + assert res["d_eff_z"] == pytest.approx(0.25, rel=1e-4) + assert res["tortuosity_z"] == pytest.approx(1.0, rel=1e-4) + assert res["d_eff_y"] == 0.0 + assert res["d_eff_x"] == 0.0 + assert not np.isfinite(res["tortuosity_y"]) + assert res["percolating_axes_count"] == 1.0 + + # 2) Solid block -> zero diffusivity + solid = np.zeros((8, 8, 8), dtype=bool) + solid_res = solve_steady_state_diffusion_axis(solid, axis=0) + assert solid_res["d_eff"] == 0.0 + assert solid_res["percolates"] == 0.0 + + # 3) Constricted / baffled duct must have tortuosity strictly > 1.0 + baffled = duct.copy() + baffled[4, 2:5, 2:6] = False # leave narrow 1x4 slit + baffled[8, 3:6, 2:6] = False + baf_z = solve_steady_state_diffusion_axis(baffled, axis=0, tol=1e-8) + assert baf_z["percolates"] == 1.0 + assert 0.0 < baf_z["d_eff"] < baf_z["porosity"] + assert baf_z["tortuosity"] > 1.05 + + +def test_capillary_drainage_and_evaluate_transport_metrics() -> None: + _, hr = synthesize_zn13x_phantom_pair(pair="2x", lr_shape=(12, 12, 12), seed=7) + mask = hr < 0.45 + + pc_grid, snwp = compute_drainage_curve(mask, n_steps=8, voxel_size=0.8125) + assert pc_grid.shape == (8,) + assert snwp.shape == (8,) + assert np.all(np.diff(snwp) >= -1e-12) + assert np.all((snwp >= 0.0) & (snwp <= 1.0)) + + assert compute_drainage_rmse(mask, mask, n_steps=8) == pytest.approx(0.0, abs=1e-12) + + identical = evaluate_transport_metrics(hr, hr, pore_is_low=True, drainage_steps=8) + assert identical["d_eff_mae"] == pytest.approx(0.0, abs=1e-7) + assert identical["drainage_snwp_rmse"] == pytest.approx(0.0, abs=1e-9) + + +def test_openimpala_export_and_cli_integration(tmp_path: Path) -> None: + _, hr = synthesize_zn13x_phantom_pair(pair="2x", lr_shape=(10, 10, 10), seed=11) + pred = np.clip(hr + 0.02, 0.0, 1.0).astype(np.float32) + + tif_p, inp_p = export_for_openimpala( + pred, + output_dir=tmp_path / "impala_direct", + name="zn13x_sample", + direction="z", + ) + loaded = tifffile.imread(tif_p) + assert loaded.dtype == np.uint8 + assert loaded.shape == pred.shape + assert set(np.unique(loaded)).issubset({0, 1}) + + inp_txt = inp_p.read_text(encoding="utf-8") + assert 'eb2.geom_type = "tiff"' in inp_txt + assert 'torteff.direction = "z"' in inp_txt + assert f"amr.n_cell = {pred.shape[2]} {pred.shape[1]} {pred.shape[0]}" in inp_txt + + # CLI evaluate --include-transport --export-openimpala-dir + standalone export-openimpala + pred_file = tmp_path / "pred.tif" + hr_file = tmp_path / "hr.tif" + out_json = tmp_path / "eval_transport.json" + tifffile.imwrite(pred_file, pred) + tifffile.imwrite(hr_file, hr) + + with pytest.raises(SystemExit) as exc_eval: + cli_main( + [ + "evaluate", + "--pred", + str(pred_file), + "--target", + str(hr_file), + "--include-transport", + "--export-openimpala-dir", + str(tmp_path / "impala_from_eval"), + "--output", + str(out_json), + ] + ) + assert exc_eval.value.code == 0 + payload = json.loads(out_json.read_text(encoding="utf-8")) + assert "transport_metrics" in payload + assert "openimpala_export" in payload + assert Path(payload["openimpala_export"]["inputs_path"]).is_file() + + with pytest.raises(SystemExit) as exc_exp: + cli_main( + [ + "export-openimpala", + "--volume", + str(pred_file), + "--output-dir", + str(tmp_path / "impala_cli"), + "--name", + "cli_rock", + ] + ) + assert exc_exp.value.code == 0 + assert (tmp_path / "impala_cli" / "cli_rock_openimpala.inputs").is_file() + + +def test_zn13x_registration_h5_extraction_and_cli_download(tmp_path: Path) -> None: + lr_clean, hr_clean = synthesize_zn13x_phantom_pair(pair="2x", lr_shape=(20, 20, 20), seed=21) + + # Apply known 3D translation to LR by padding/cropping so HR aligns at offset (2, 1, 2) + lr_shifted = np.pad(lr_clean, ((2, 0), (1, 0), (2, 0)), mode="reflect")[:20, :20, :20] + aligned_lr, aligned_hr, info = register_multires_pair_3d(lr_shifted, hr_clean, scale_factor=2) + assert aligned_hr.shape == tuple(s * 2 for s in aligned_lr.shape) + assert float(info["coarse_mid_slice_ssim"]) > 0.65 # type: ignore[arg-type] + + # Mock Savu HDF5 files with minimal ROI coverage around modified test specs + lo_spec = ZN13X_I13_RESOLUTIONS["1p625um"] + hi_spec = ZN13X_I13_RESOLUTIONS["0p8125um"] + lo_h5 = tmp_path / lo_spec.recon_filename + hi_h5 = tmp_path / hi_spec.recon_filename + + assert lo_spec.roi_1040um_yx is not None and hi_spec.roi_1040um_yx is not None + lo_shape = ( + lo_spec.roi_1040um_yx[1][0] + 16, + lo_spec.anchor_z + 10, + lo_spec.roi_1040um_yx[0][0] + 16, + ) + hi_shape = ( + hi_spec.roi_1040um_yx[1][0] + 32, + hi_spec.anchor_z + 20, + hi_spec.roi_1040um_yx[0][0] + 32, + ) + with h5py.File(lo_h5, "w") as f_lo: + dset = f_lo.create_dataset("4-TomopyRecon-tomo/data", shape=lo_shape, dtype="f4") + dset[ + lo_spec.roi_1040um_yx[1][0] : lo_spec.roi_1040um_yx[1][0] + 16, + lo_spec.anchor_z - 8 : lo_spec.anchor_z + 8, + lo_spec.roi_1040um_yx[0][0] : lo_spec.roi_1040um_yx[0][0] + 16, + ] = 0.0002 + with h5py.File(hi_h5, "w") as f_hi: + dset = f_hi.create_dataset("4-TomopyRecon-tomo/data", shape=hi_shape, dtype="f4") + dset[ + hi_spec.roi_1040um_yx[1][0] : hi_spec.roi_1040um_yx[1][0] + 32, + hi_spec.anchor_z - 16 : lo_spec.anchor_z + 16, + hi_spec.roi_1040um_yx[0][0] : hi_spec.roi_1040um_yx[0][0] + 32, + ] = 0.0002 + + sub_lo, sub_hi, _ = extract_aligned_slab_from_h5( + lo_h5, hi_h5, pair="2x", lr_slab_depth=16, lr_crop_yx=16 + ) + assert sub_hi.shape == tuple(s * 2 for s in sub_lo.shape) + + # CLI download-benchmarks for zn13x_2x and zn13x_8x (offline fallback / cache) + bench_dir = tmp_path / "benchmarks" + lr_8x, hr_8x, man_8x = prepare_zn13x_benchmark( + pair="8x", dest_dir=bench_dir, tier="full", lr_size=10, seed=5 + ) + assert lr_8x.is_file() and hr_8x.is_file() + assert man_8x["scale_factor"] == 8 + + with pytest.raises(SystemExit) as exc_dl: + cli_main( + [ + "download-benchmarks", + "--datasets", + "zn13x_2x", + "--dest", + str(bench_dir), + "--size", + "16", + ] + ) + assert exc_dl.value.code == 0 + assert (bench_dir / "zn13x_2x_hr.tif").is_file() + + +def test_rrdb_arbitrary_scale_ragan_and_sr3d_end_to_end(tmp_path: Path) -> None: + # 1) Arbitrary fractional scale 2.5x (0.8125 -> 0.325 um) & 5.0x (1.625 -> 0.325 um) + gen_2p5 = RRDBGenerator(out_channels=12, growth_channels=8, num_blocks=2, super_res_factor=2.5) + x = torch.rand(2, 1, 12, 12) + y = gen_2p5(x) + assert y.shape == (2, 1, 30, 30) + + gen_5x = RRDBGenerator(out_channels=12, growth_channels=8, num_blocks=1, super_res_factor=5.0) + assert gen_5x(torch.rand(1, 1, 8, 8)).shape == (1, 1, 40, 40) + + vol_lr = np.random.default_rng(0).random((12, 12, 12), dtype=np.float32) + fused_2p5 = super_resolve_volume_orthogonal(gen_2p5, vol_lr, scale_factor=2.5)["fused"] + assert fused_2p5.shape == (30, 30, 30) + + # 2) End-to-end RRDB + RaGAN Lightning fit (warmup epoch 0 + active RaGAN epoch 1) + lr_p, hr_p, _ = prepare_zn13x_benchmark( + pair="2x", dest_dir=tmp_path / "zn", tier="full", lr_size=16, seed=9 + ) + dm = PairedVolumeDataModule( + low_path=str(lr_p), + high_path=str(hr_p), + scale_factor=2, + norm_mode="none", + batch_size=2, + eval_batch_size=2, + patch_size_lr=12, + seed=9, + ) + dm.setup("fit") + model = SRGAN( + learning_rate=2e-4, + adaptive_size=4, + super_res_factor=2, + num_blocks=2, + out_channels=12, + growth_channels=8, + backbone="rrdb", + gan_loss_type="ragan", + output_activation="clamp", + pixel_loss_type="l1", + disc_features=[12, 24], + pretrain_epochs=1, + use_adversarial_loss=True, + use_vgg_loss=False, + output_dir=str(tmp_path / "rrdb_run"), + save_val_images=False, + ) + trainer = pl.Trainer( + default_root_dir=str(tmp_path / "rrdb_run"), + max_epochs=2, + accelerator="cpu", + devices=1, + logger=False, + enable_checkpointing=False, + enable_progress_bar=False, + enable_model_summary=False, + ) + trainer.fit(model, datamodule=dm) + + # 3) End-to-end SR3DResNet + seamless raised-cosine 3D sliding-window inference + lr_tr, hr_tr = dm.splits_3d["train"] + lr_va, hr_va = dm.splits_3d["val"] + tr_3d, va_3d = build_3d_subcube_dataloaders( + lr_tr, + hr_tr, + lr_va, + hr_va, + scale_factor=2, + patch_size_lr=min(8, int(min(lr_tr.shape))), + train_samples=4, + val_samples=2, + batch_size=2, + ) + sr3d = SR3DResNet(super_res_factor=2, out_channels=8, num_blocks=1) + trainer_3d = pl.Trainer( + default_root_dir=str(tmp_path / "sr3d_run"), + max_epochs=1, + accelerator="cpu", + devices=1, + logger=False, + enable_checkpointing=False, + enable_progress_bar=False, + enable_model_summary=False, + ) + trainer_3d.fit(sr3d, train_dataloaders=tr_3d, val_dataloaders=va_3d) + sr_vol_3d = super_resolve_volume_3d( + sr3d.gen, + np.zeros((14, 14, 14), dtype=np.float32), + scale_factor=2, + patch_size_lr=8, + overlap_lr=2, + ) + assert sr_vol_3d.shape == (28, 28, 28) + assert np.all((sr_vol_3d >= 0.0) & (sr_vol_3d <= 1.0)) + + +def test_campaign_aggregator_generates_latex_markdown_and_plot(tmp_path: Path) -> None: + spec = importlib.util.spec_from_file_location( + "aggregate_campaign", + Path(__file__).resolve().parents[1] / "scripts" / "benchmarks" / "aggregate_campaign.py", + ) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + + csv_file = tmp_path / "zn13x_synchrotron_results.csv" + with csv_file.open("w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter( + f, + fieldnames=[ + "dataset", + "seed", + "method", + "psnr", + "ssim_3d_triplanar", + "pore_dice", + "porosity_abs_error", + "through_plane_anisotropy_error", + "d_eff_mae", + "drainage_snwp_rmse", + ], + ) + writer.writeheader() + for seed, psnr in ((42, 24.1), (43, 24.5)): + writer.writerow( + { + "dataset": "zn13x_2x", + "seed": seed, + "method": "OpenLSR-X-RRDB-RaGAN-3Axis-Fourier", + "psnr": psnr, + "ssim_3d_triplanar": 0.88, + "pore_dice": 0.91, + "porosity_abs_error": 0.008, + "through_plane_anisotropy_error": 0.005, + "d_eff_mae": 0.012, + "drainage_snwp_rmse": 0.031, + } + ) + + out_dir = tmp_path / "agg" + info = mod.aggregate_csv_files([csv_file], out_dir) + assert Path(info["markdown_path"]).is_file() + assert "±" in Path(info["markdown_path"]).read_text(encoding="utf-8") + assert "\\toprule" in Path(info["latex_path"]).read_text(encoding="utf-8") + assert Path(info["figure_path"]).is_file()